From 2e416ca00e27744eb67b4982be633ab524b16bd8 Mon Sep 17 00:00:00 2001 From: Josh Creek <8179928+jcreek@users.noreply.github.com> Date: Sat, 26 Apr 2025 20:43:07 +0100 Subject: [PATCH] Add work in progress for refactoring to avoid 10+ second API calls that don't work on Netlify --- src/lib/server/ai.ts | 46 +------ src/routes/api/grade/+server.ts | 8 +- src/routes/api/grade/status/+server.ts | 102 ++++++++++++++ src/routes/upload/+page.svelte | 175 ++++++++++++------------- 4 files changed, 196 insertions(+), 135 deletions(-) create mode 100644 src/routes/api/grade/status/+server.ts diff --git a/src/lib/server/ai.ts b/src/lib/server/ai.ts index ff424b4..54c225d 100644 --- a/src/lib/server/ai.ts +++ b/src/lib/server/ai.ts @@ -283,12 +283,14 @@ async function processRunStream( throw new Error('Stream ended without completion'); } +export { client, agent }; + export async function gradeSubmissionWithAgent( submission: string, task: string, roomId: string, hitl: boolean = false -): Promise { +): Promise<{ threadId: string; runId: string }> { if (!client || !agent) { throw new Error('Agent not available'); } @@ -301,52 +303,16 @@ export async function gradeSubmissionWithAgent( : buildGradingPrompt(submission, task) }); - let initialStream: AsyncIterable; try { - const runInvoker = client.agents.createRun(thread.id, agent.id, { + const run = await client.agents.createRun(thread.id, agent.id, { parallelToolCalls: false }); - initialStream = await runInvoker.stream(); + return { threadId: thread.id, runId: run.id }; } catch (err) { logger.error(`Failed to start run on thread ${thread.id}:`, err); logger.info('Task:', task); logger.info('Submission', submission); - return fallbackGrade(submission, task); - } - - // process all tool calls - let finalRun: ThreadRunOutput; - try { - finalRun = await processRunStream(thread.id, initialStream, roomId); - } catch (err) { - logger.error(`Agent streaming failed on thread ${thread.id}:`, err); - return fallbackGrade(submission, task); - } - - const msgs = await client.agents.listMessages(thread.id); - const assistant = msgs.data.find((m) => m.role === 'assistant'); - const raw = assistant ? extractTextFromMessage(assistant) : ''; - - // strip anything before the JSON object - const match = raw.match(/\{[\s\S]*\}$/); - const jsonText = match ? match[0] : raw; - - try { - const parsed = JSON.parse(jsonText) as OpenAIResponse; - if (parsed.grade === 'HUMAN_REVIEW_REQUIRED') { - return { - ...parsed, - success: true, - threadId: thread.id, - runId: finalRun?.id ?? undefined, - hitlContext: { threadId: thread.id, runId: finalRun?.id ?? undefined } - }; - } - return parsed; - } catch (parseErr) { - logger.error(`JSON parse failed, returning fallback on thread ${thread.id}:`, parseErr); - logger.info('JSON', jsonText); - return { ...fallbackGrade(submission, task), reasoning: raw }; + throw new Error('Failed to start grading run.'); } } diff --git a/src/routes/api/grade/+server.ts b/src/routes/api/grade/+server.ts index 79b0bdd..1e380db 100644 --- a/src/routes/api/grade/+server.ts +++ b/src/routes/api/grade/+server.ts @@ -35,15 +35,15 @@ export const POST: RequestHandler = async (event: RequestEvent) => { return new Response(JSON.stringify({ error: 'No valid input provided.' }), { status: 400 }); } - // Call the AI grading agent with threading and fallback - const aiResult = await gradeSubmissionWithAgent(submission, task.trim(), roomId ?? '', hitl); + // Call the AI grading agent asynchronously + const { threadId, runId } = await gradeSubmissionWithAgent(submission, task.trim(), roomId ?? '', hitl); return new Response( JSON.stringify({ success: true, task: task.trim(), - feedback: `Suggested Grade: ${aiResult.grade}\nStrengths: ${aiResult.strengths}\nAreas for Improvement: ${aiResult.areasForImprovement}\nIndividualized Activity: ${aiResult.individualizedActivity}\nSpelling and Grammar: ${aiResult.spellingAndGrammar}\nReasoning: ${aiResult.reasoning}`, - ...aiResult + threadId, + runId }), { status: 200, diff --git a/src/routes/api/grade/status/+server.ts b/src/routes/api/grade/status/+server.ts new file mode 100644 index 0000000..0ce151e --- /dev/null +++ b/src/routes/api/grade/status/+server.ts @@ -0,0 +1,102 @@ +import type { RequestHandler } from '@sveltejs/kit'; +import type { RequestEvent } from '@sveltejs/kit'; +import { client, agent } from '$lib/server/ai'; + +// GET /api/grade/status?threadId=...&runId=... +export const GET: RequestHandler = async (event: RequestEvent) => { + const url = new URL(event.request.url); + const threadId = url.searchParams.get('threadId'); + const runId = url.searchParams.get('runId'); + + if (!client || !agent) { + return new Response(JSON.stringify({ error: 'Agent not available' }), { status: 500 }); + } + if (!threadId || !runId) { + return new Response(JSON.stringify({ error: 'Missing threadId or runId' }), { status: 400 }); + } + + try { + const run = await client.agents.getRun(threadId, runId); + // Enhanced: handle all important statuses for polling + let toolEvents = []; + if (run.tool_calls) { + toolEvents = run.tool_calls; + } + + let result = null; + let parsed = null; + let humanReviewRequired = false; + let parseError = null; + + // Handle agent run statuses for polling, mirroring processRunStream + if (run.status === 'requires_action' && run.requiredAction) { + return new Response( + JSON.stringify({ + success: true, + status: 'requires_action', + requiredAction: run.requiredAction, + toolCalls: run.requiredAction.submitToolOutputs?.toolCalls || [], + toolEvents, + message: 'Agent requires tool outputs to proceed.' + }), + { status: 200, headers: { 'Content-Type': 'application/json' } } + ); + } + if (run.status === 'failed' || run.status === 'cancelling' || run.status === 'cancelled') { + return new Response( + JSON.stringify({ + success: false, + status: run.status, + toolEvents, + error: run.lastError || 'Run failed or was cancelled.' + }), + { status: 500, headers: { 'Content-Type': 'application/json' } } + ); + } + if (run.status === 'in_progress' || run.status === 'queued' || run.status === 'starting') { + return new Response( + JSON.stringify({ + success: true, + status: run.status, + toolEvents, + message: 'Grading in progress.' + }), + { status: 200, headers: { 'Content-Type': 'application/json' } } + ); + } + + // Completed and fallback handled below as before + if (run.status === 'completed') { + const msgs = await client.agents.listMessages(threadId); + const assistant = msgs.data.find((m: any) => m.role === 'assistant'); + const raw = assistant ? (typeof assistant.content === 'string' ? assistant.content : (assistant.content?.[0]?.text || '')) : ''; + // strip anything before the JSON object + const match = raw.match(/\{[\s\S]*\}$/); + const jsonText = match ? match[0] : raw; + try { + parsed = JSON.parse(jsonText); + if (parsed.grade === 'HUMAN_REVIEW_REQUIRED') { + humanReviewRequired = true; + } + result = parsed; + } catch (err: any) { + parseError = err.message || 'Failed to parse AI response.'; + result = { raw }; + } + } + + return new Response( + JSON.stringify({ + success: true, + status: run.status, + toolEvents, + result, + humanReviewRequired, + parseError + }), + { status: 200, headers: { 'Content-Type': 'application/json' } } + ); + } catch (err: any) { + return new Response(JSON.stringify({ error: err.message || 'Failed to get run status' }), { status: 500 }); + } +}; diff --git a/src/routes/upload/+page.svelte b/src/routes/upload/+page.svelte index 29b050f..ef29fba 100644 --- a/src/routes/upload/+page.svelte +++ b/src/routes/upload/+page.svelte @@ -40,111 +40,104 @@ } async function handleSubmit(event: Event) { - event.preventDefault(); - submitting = true; - successMsg = ''; - errorMsg = ''; - const THINKING_EVENT = { tool: 'thinking', time: new Date().toISOString() }; - function ensureThinkingStep(events) { - // Remove any existing 'thinking' event - const filtered = events.filter((e) => e.tool !== 'thinking'); - return [THINKING_EVENT, ...filtered]; - } - toolEvents = [THINKING_EVENT]; + event.preventDefault(); + submitting = true; + successMsg = ''; + errorMsg = ''; + const THINKING_EVENT = { tool: 'thinking', time: new Date().toISOString() }; + function ensureThinkingStep(events) { + // Remove any existing 'thinking' event + const filtered = events.filter((e) => e.tool !== 'thinking'); + return [THINKING_EVENT, ...filtered]; + } + toolEvents = [THINKING_EVENT]; - // Generate a unique roomId for this submission - roomId = crypto.randomUUID(); - // Connect to PartyKit for this session - if (ws) ws.close(); - ws = new WebSocket(`${PARTYKIT_BASE_URL}/party/tool-usage-server-${roomId}`); - ws.onmessage = (event) => { - try { - const data = JSON.parse(event.data); - toolEvents = ensureThinkingStep([...toolEvents, data]); - } catch {} - }; + // Generate a unique roomId for this submission + roomId = crypto.randomUUID(); + // Connect to PartyKit for this session + if (ws) ws.close(); + ws = new WebSocket(`${PARTYKIT_BASE_URL}/party/tool-usage-server-${roomId}`); + ws.onmessage = (event) => { + try { + const data = JSON.parse(event.data); + toolEvents = ensureThinkingStep([...toolEvents, data]); + } catch {} + }; - if (!taskDescription.trim()) { - errorMsg = 'Please provide a description of the task or assignment.'; + if (!taskDescription.trim()) { + errorMsg = 'Please provide a description of the task or assignment.'; + submitting = false; + return; + } + const formData = new FormData(); + formData.append('task', taskDescription.trim()); + + if (file) { + formData.append('file', file); + } else if (textInput.trim().length > 0) { + formData.append('text', textInput.trim()); + } + formData.append('roomId', roomId); + formData.append('hitl', hitl ? 'true' : 'false'); + + try { + const response = await fetch('/api/grade', { + method: 'POST', + body: formData + }); + const data = await response.json(); + if (!data.success) { + errorMsg = data.error || 'Failed to start grading.'; submitting = false; return; } - const formData = new FormData(); - formData.append('task', taskDescription.trim()); + const { threadId, runId } = data; + await pollForStatus(threadId, runId); + } catch (err: any) { + errorMsg = err.message || 'Failed to start grading.'; + submitting = false; + } +} - if (file) { - formData.append('file', file); - } else if (textInput.trim().length > 0) { - formData.append('text', textInput.trim()); - } - formData.append('roomId', roomId); - formData.append('hitl', hitl ? 'true' : 'false'); - - errorMsg = ''; +async function pollForStatus(threadId: string, runId: string) { + humanReviewRequired = false; + aiReasoning = ''; + hitlContext = null; + let polling = true; + while (polling) { + await new Promise((resolve) => setTimeout(resolve, 1500)); try { - const response = await fetch('/api/grade', { - method: 'POST', - body: formData - }); - - let result; - try { - result = await response.json(); - } catch (err) { - console.error('Error parsing /api/grade response:', err); - errorMsg = 'Received an invalid response from the server.'; + const res = await fetch(`/api/grade/status?threadId=${encodeURIComponent(threadId)}&runId=${encodeURIComponent(runId)}`); + const statusData = await res.json(); + if (!statusData.success) { + errorMsg = statusData.error || 'Unknown error during grading.'; submitting = false; return; } - if (response.ok && result.success) { - if (result.grade === 'HUMAN_REVIEW_REQUIRED') { - humanReviewRequired = true; - aiReasoning = result.reasoning; - hitlContext = result.hitlContext || { threadId: result.threadId, runId: result.runId }; - submitting = false; - return; - } - - try { - // Save to localStorage history - let prev = JSON.parse(localStorage.getItem('assessmentHistory') || '[]'); - const entry = { - timestamp: Date.now(), - ...result, - submission: result.submission || textInput || '', - task: taskDescription - }; - prev.unshift(entry); - localStorage.setItem('assessmentHistory', JSON.stringify(prev.slice(0, 50))); - } catch {} - - // Redirect to /results if HITL is not needed - window.location.assign('/results'); - - file = null; - textInput = ''; - taskDescription = ''; - return; - } else { - console.error('Grade API error or non-success:', response, result); - if (result && result.error) { - errorMsg = `Sorry, something went wrong: ${result.error}`; - } else if (response.status === 413) { - errorMsg = 'The uploaded file is too large. Please upload a smaller file.'; - } else if (response.status >= 500) { - errorMsg = 'The server encountered an error. Please try again later.'; - } else { - errorMsg = 'An unknown error occurred. Please check your input and try again.'; + toolEvents = statusData.toolEvents || toolEvents; + if (statusData.status === 'completed') { + polling = false; + submitting = false; + if (statusData.result) { + const result = statusData.result; + aiReasoning = result.reasoning || ''; + if (statusData.humanReviewRequired) { + humanReviewRequired = true; + hitlContext = { threadId, runId }; + } else { + successMsg = `Suggested Grade: ${result.grade}\nStrengths: ${result.strengths}\nAreas for Improvement: ${result.areasForImprovement}\nIndividualized Activity: ${result.individualizedActivity}\nSpelling and Grammar: ${result.spellingAndGrammar}\nReasoning: ${result.reasoning}`; + } + } else if (statusData.parseError) { + errorMsg = statusData.parseError; } } - } catch (err) { - console.error('Error submitting grade:', err); - errorMsg = 'Network or server error. Please check your connection and try again.'; - } finally { - // Disabled to make the transition to the results page smoother - // submitting = false; + } catch (err: any) { + errorMsg = err.message || 'Failed to poll grading status.'; + submitting = false; + return; } } +} async function submitHumanReview() { submitting = true;