diff --git a/src/lib/agent/services/agentService.ts b/src/lib/agent/services/agentService.ts index bad422b..3b45718 100644 --- a/src/lib/agent/services/agentService.ts +++ b/src/lib/agent/services/agentService.ts @@ -1,42 +1,42 @@ import { DefaultAzureCredential } from '@azure/identity'; import { AIProjectsClient, ToolUtility } from '@azure/ai-projects'; import { - rubricMatcherTool, - essayAnalyzerTool, - conceptVerifierTool, - feedbackGeneratorTool, - metacognitiveReflectionPromptTool, - selfAssessmentTool, - spellingAndGrammarCheckerTool + rubricMatcherTool, + essayAnalyzerTool, + conceptVerifierTool, + feedbackGeneratorTool, + metacognitiveReflectionPromptTool, + selfAssessmentTool, + spellingAndGrammarCheckerTool } from '../tools/index'; import { AI_FOUNDRY_PROJECT_CONNECTION_STRING, AI_MODEL } from '$env/static/private'; if (!AI_FOUNDRY_PROJECT_CONNECTION_STRING) { - throw new Error('AI_FOUNDRY_PROJECT_CONNECTION_STRING is not set'); + throw new Error('AI_FOUNDRY_PROJECT_CONNECTION_STRING is not set'); } if (!AI_MODEL) { - throw new Error('AI_MODEL is not set'); + throw new Error('AI_MODEL is not set'); } export const client = AIProjectsClient.fromConnectionString( - AI_FOUNDRY_PROJECT_CONNECTION_STRING, - new DefaultAzureCredential() + AI_FOUNDRY_PROJECT_CONNECTION_STRING, + new DefaultAzureCredential() ); // const codeInterpreterTool = ToolUtility.createCodeInterpreterTool([]); const allTools = [ - // codeInterpreterTool, - rubricMatcherTool, - essayAnalyzerTool, - conceptVerifierTool, - feedbackGeneratorTool, - metacognitiveReflectionPromptTool, - selfAssessmentTool, - spellingAndGrammarCheckerTool + // codeInterpreterTool, + rubricMatcherTool, + essayAnalyzerTool, + conceptVerifierTool, + feedbackGeneratorTool, + metacognitiveReflectionPromptTool, + selfAssessmentTool, + spellingAndGrammarCheckerTool ]; -const tools = allTools.map(t => t.definition); +const tools = allTools.map((t) => t.definition); const instructions = ` You are a helpful agent that can assist with providing feedback on a student's work for a teacher. @@ -44,50 +44,49 @@ You are familiar with modern pedagogy around summative and formative assessment Whenever you need to: • check the submission against the rubric → call "${rubricMatcherTool.definition.name}" • verify which rubric concepts appear or are missing → call "${conceptVerifierTool.definition.name}" - • analyze essay structure → call "${essayAnalyzerTool.definition.name}" + • analyze essay structure → call "${essayAnalyzerTool.definition.name}" • generate targeted feedback → call "${feedbackGeneratorTool.definition.name}" • craft a metacognitive prompt → call "${metacognitiveReflectionPromptTool.definition.name}" • guide self-assessment → call "${selfAssessmentTool.definition.name}" • check spelling & grammar → call "${spellingAndGrammarCheckerTool.definition.name}" + + If a user message begins with [HUMAN REVIEW]:, treat it as authoritative teacher feedback and use it as the basis for grading. Do not escalate to human review again. `; export const toolResources = allTools.reduce>((map, t) => { - // Try all plausible locations for the tool name - const name = - t.definition.name || - t.definition.function?.name || - Object.keys(t.resources ?? {})[0]; + // Try all plausible locations for the tool name + const name = + t.definition.name || t.definition.function?.name || Object.keys(t.resources ?? {})[0]; - if (!name) { - console.warn('Tool missing name:', t); - return map; - } - map[name] = t; - return map; + if (!name) { + console.warn('Tool missing name:', t); + return map; + } + map[name] = t; + return map; }, {}); export async function getOrCreateAgent() { - - const agentName = `assessment-feedback-agent`; - // List all agents (may need pagination for large numbers) - const agentsResponse = await client.agents.listAgents(); - const agents = Array.isArray(agentsResponse.data) ? agentsResponse.data : []; - const existing = agents.find((a: any) => a.name === agentName); - if (existing) { - return existing; - } - // Otherwise, create new agent - const agent = await client.agents.createAgent(AI_MODEL, { - name: agentName, - instructions, - temperature: 0.5, - tools, - toolResources, - requestOptions: { - headers: { 'x-ms-enable-preview': 'true' } - } - }); - return agent; + const agentName = `assessment-feedback-agent`; + // List all agents (may need pagination for large numbers) + const agentsResponse = await client.agents.listAgents(); + const agents = Array.isArray(agentsResponse.data) ? agentsResponse.data : []; + const existing = agents.find((a: any) => a.name === agentName); + if (existing) { + return existing; + } + // Otherwise, create new agent + const agent = await client.agents.createAgent(AI_MODEL, { + name: agentName, + instructions, + temperature: 0.5, + tools, + toolResources, + requestOptions: { + headers: { 'x-ms-enable-preview': 'true' } + } + }); + return agent; } export const agent = await getOrCreateAgent(); @@ -97,29 +96,29 @@ export const agent = await getOrCreateAgent(); * Use with caution! This will remove ALL agents. */ export async function deleteAllAgents() { - try { - let allAgents: any[] = []; - let after: string | undefined = undefined; - let hasMore = true; + try { + let allAgents: any[] = []; + let after: string | undefined = undefined; + let hasMore = true; - while (hasMore) { - const response = await client.agents.listAgents(after ? { after } : {}); - const agents = Array.isArray(response.data) ? response.data : []; - allAgents = allAgents.concat(agents); - hasMore = response.hasMore; - after = response.lastId; - } + while (hasMore) { + const response = await client.agents.listAgents(after ? { after } : {}); + const agents = Array.isArray(response.data) ? response.data : []; + allAgents = allAgents.concat(agents); + hasMore = response.hasMore; + after = response.lastId; + } - for (const agent of allAgents) { - await client.agents.deleteAgent(agent.id, { - requestOptions: { - headers: { 'x-ms-enable-preview': 'true' } - } - }); - console.info(`Agent ${agent.id} (${agent.name}) deleted.`); - } - console.info('All agents deleted successfully.'); - } catch (err) { - console.warn('Failed to delete all agents:', err); - } + for (const agent of allAgents) { + await client.agents.deleteAgent(agent.id, { + requestOptions: { + headers: { 'x-ms-enable-preview': 'true' } + } + }); + console.info(`Agent ${agent.id} (${agent.name}) deleted.`); + } + console.info('All agents deleted successfully.'); + } catch (err) { + console.warn('Failed to delete all agents:', err); + } } diff --git a/src/lib/server/ai.ts b/src/lib/server/ai.ts index ff0b83b..fbf0f9e 100644 --- a/src/lib/server/ai.ts +++ b/src/lib/server/ai.ts @@ -40,7 +40,10 @@ RESPONSE FORMAT (respond with a single JSON object, no extra text): Respond ONLY with the JSON object, with no preamble or explanation.`; } -export function buildGradingPromptWithSupportForHumanInTheLoop(submission: string, task: string): string { +export function buildGradingPromptWithSupportForHumanInTheLoop( + submission: string, + task: string +): string { return `You are an expert secondary school teacher and AI assessment agent. Assess the following student submission in the context of the assignment/task provided. TASK/ASSIGNMENT: @@ -128,7 +131,6 @@ EXAMPLES: - Reasoning: The task and submission are both clear and evaluatable without a rubric. Respond ONLY with the JSON object, with no preamble or explanation.`; - } function fallbackGrade(submission: string, task: string): OpenAIResponse { @@ -223,7 +225,8 @@ async function processRunStream( export async function gradeSubmissionWithAgent( submission: string, task: string, - roomId: string + roomId: string, + hitl: boolean = false ): Promise { if (!client || !agent) { throw new Error('Agent not available'); @@ -232,7 +235,9 @@ export async function gradeSubmissionWithAgent( const thread = await client.agents.createThread(); await client.agents.createMessage(thread.id, { role: 'user', - content: buildGradingPrompt(submission, task) + content: hitl + ? buildGradingPromptWithSupportForHumanInTheLoop(submission, task) + : buildGradingPrompt(submission, task) }); let initialStream: AsyncIterable; @@ -264,9 +269,65 @@ export async function gradeSubmissionWithAgent( const jsonText = match ? match[0] : raw; try { - return JSON.parse(jsonText) as OpenAIResponse; + const parsed = JSON.parse(jsonText) as OpenAIResponse; + if (parsed.grade === 'HUMAN_REVIEW_REQUIRED') { + return { + ...parsed, + success: true, + threadId: thread.id, + runId: finalRun?.id ?? undefined, + hitlContext: { threadId: thread.id, runId: finalRun?.id ?? undefined } + }; + } + return parsed; } catch (parseErr) { console.error('JSON parse failed, returning fallback:', parseErr); return { ...fallbackGrade(submission, task), reasoning: raw }; } } + +export async function resumeAgentWithHumanReview( + humanReview: string, + context: { threadId: string; runId?: string; roomId?: string; [key: string]: any } +): Promise { + const { threadId, roomId = '' } = context; + await client.agents.createMessage(threadId, { + role: 'user', + content: `[HUMAN REVIEW]: ${humanReview}` + }); + + let stream: AsyncIterable; + try { + const runInvoker = client.agents.createRun(threadId, agent.id, { + parallelToolCalls: false + }); + stream = await runInvoker.stream(); + } catch (err) { + console.error('Failed to start run:', err); + return fallbackGrade('', ''); + } + + let finalRun: ThreadRunOutput; + try { + finalRun = await processRunStream(threadId, stream, roomId); + } catch (err) { + console.error('Agent streaming failed:', err); + return fallbackGrade('', ''); + } + + const msgs = await client.agents.listMessages(threadId); + const assistant = msgs.data.find((m) => m.role === 'assistant'); + const raw = assistant ? extractTextFromMessage(assistant) : ''; + + // strip anything before the JSON object + const match = raw.match(/\{[\s\S]*\}$/); + const jsonText = match ? match[0] : raw; + + try { + const parsed = JSON.parse(jsonText) as OpenAIResponse; + return parsed; + } catch (parseErr) { + console.error('JSON parse failed, returning fallback:', parseErr); + return { ...fallbackGrade('', ''), reasoning: raw }; + } +} diff --git a/src/routes/api/grade/+server.ts b/src/routes/api/grade/+server.ts index 00be56a..dc32e74 100644 --- a/src/routes/api/grade/+server.ts +++ b/src/routes/api/grade/+server.ts @@ -3,45 +3,56 @@ import type { RequestEvent } from '@sveltejs/kit'; import { gradeSubmissionWithAgent } from '$lib/server/ai'; export const POST: RequestHandler = async (event: RequestEvent) => { - try { - const data = await event.request.formData(); - const file = data.get('file'); - const text = data.get('text'); - const task = data.get('task'); - const roomId = data.get('roomId') as string | undefined; - let submission = ''; + try { + const data = await event.request.formData(); + const file = data.get('file'); + const text = data.get('text'); + const task = data.get('task'); + const roomId = data.get('roomId') as string | undefined; + const hitl = data.get('hitl') === 'true'; + let submission = ''; - if (!task || typeof task !== 'string' || !task.trim()) { - return new Response(JSON.stringify({ error: 'Task/assignment description is required.' }), { status: 400 }); - } + if (!task || typeof task !== 'string' || !task.trim()) { + return new Response(JSON.stringify({ error: 'Task/assignment description is required.' }), { + status: 400 + }); + } - if (file && typeof file === 'object' && 'arrayBuffer' in file && 'name' in file) { - // Only support .txt for now for hackathon speed - const fileObj = file as File; - if (!fileObj.name.endsWith('.txt')) { - return new Response(JSON.stringify({ error: 'Only .txt files are supported at this time.' }), { status: 400 }); - } - const buffer = await fileObj.arrayBuffer(); - submission = new TextDecoder('utf-8').decode(buffer); - } else if (typeof text === 'string' && text.trim().length > 0) { - submission = text.trim(); - } else { - return new Response(JSON.stringify({ error: 'No valid input provided.' }), { status: 400 }); - } + if (file && typeof file === 'object' && 'arrayBuffer' in file && 'name' in file) { + // Only support .txt for now for hackathon speed + const fileObj = file as File; + if (!fileObj.name.endsWith('.txt')) { + return new Response( + JSON.stringify({ error: 'Only .txt files are supported at this time.' }), + { status: 400 } + ); + } + const buffer = await fileObj.arrayBuffer(); + submission = new TextDecoder('utf-8').decode(buffer); + } else if (typeof text === 'string' && text.trim().length > 0) { + submission = text.trim(); + } else { + return new Response(JSON.stringify({ error: 'No valid input provided.' }), { status: 400 }); + } - // Call the AI grading agent with threading and fallback - const aiResult = await gradeSubmissionWithAgent(submission, task.trim(), roomId); + // Call the AI grading agent with threading and fallback + const aiResult = await gradeSubmissionWithAgent(submission, task.trim(), roomId ?? '', hitl); - return new Response(JSON.stringify({ - success: true, - task: task.trim(), - feedback: `Grade: ${aiResult.grade}\nStrengths: ${aiResult.strengths}\nAreas for Improvement: ${aiResult.areasForImprovement}\nIndividualized Activity: ${aiResult.individualizedActivity}\nSpelling and Grammar: ${aiResult.spellingAndGrammar}\nReasoning: ${aiResult.reasoning}`, - ...aiResult - }), { - status: 200, - headers: { 'Content-Type': 'application/json' } - }); - } catch (err: any) { - return new Response(JSON.stringify({ error: err.message || 'Internal server error.' }), { status: 500 }); - } + return new Response( + JSON.stringify({ + success: true, + task: task.trim(), + feedback: `Grade: ${aiResult.grade}\nStrengths: ${aiResult.strengths}\nAreas for Improvement: ${aiResult.areasForImprovement}\nIndividualized Activity: ${aiResult.individualizedActivity}\nSpelling and Grammar: ${aiResult.spellingAndGrammar}\nReasoning: ${aiResult.reasoning}`, + ...aiResult + }), + { + status: 200, + headers: { 'Content-Type': 'application/json' } + } + ); + } catch (err: any) { + return new Response(JSON.stringify({ error: err.message || 'Internal server error.' }), { + status: 500 + }); + } }; diff --git a/src/routes/api/hitl-review/+server.ts b/src/routes/api/hitl-review/+server.ts new file mode 100644 index 0000000..44f7406 --- /dev/null +++ b/src/routes/api/hitl-review/+server.ts @@ -0,0 +1,23 @@ +import type { RequestHandler } from '@sveltejs/kit'; +import { resumeAgentWithHumanReview } from '$lib/server/ai'; + +export const POST: RequestHandler = async (event) => { + try { + const { humanReview, context } = await event.request.json(); + const aiResult = await resumeAgentWithHumanReview(humanReview, context); + return new Response( + JSON.stringify({ + success: true, + ...aiResult + }), + { + status: 200, + headers: { 'Content-Type': 'application/json' } + } + ); + } catch (err: any) { + return new Response(JSON.stringify({ error: err.message || 'Internal server error.' }), { + status: 500 + }); + } +}; diff --git a/src/routes/upload/+page.svelte b/src/routes/upload/+page.svelte index 311238e..29b050f 100644 --- a/src/routes/upload/+page.svelte +++ b/src/routes/upload/+page.svelte @@ -9,9 +9,16 @@ let errorMsg = ''; let toolEvents: Array<{ tool: string; time: string }> = []; let roomId: string = ''; + let hitl = false; // Human-in-the-Loop toggle let ws: WebSocket | null = null; const PARTYKIT_BASE_URL = import.meta.env.VITE_PARTYKIT_BASE_URL; + // HITL state + let humanReviewRequired = false; + let aiReasoning = ''; + let humanReviewText = ''; + let hitlContext: any = null; + function handleFileChange(event: Event) { const target = event.target as HTMLInputElement; if (target.files && target.files.length > 0) { @@ -40,7 +47,7 @@ const THINKING_EVENT = { tool: 'thinking', time: new Date().toISOString() }; function ensureThinkingStep(events) { // Remove any existing 'thinking' event - const filtered = events.filter(e => e.tool !== 'thinking'); + const filtered = events.filter((e) => e.tool !== 'thinking'); return [THINKING_EVENT, ...filtered]; } toolEvents = [THINKING_EVENT]; @@ -64,13 +71,14 @@ } const formData = new FormData(); formData.append('task', taskDescription.trim()); - + if (file) { formData.append('file', file); } else if (textInput.trim().length > 0) { formData.append('text', textInput.trim()); } formData.append('roomId', roomId); + formData.append('hitl', hitl ? 'true' : 'false'); errorMsg = ''; try { @@ -78,17 +86,27 @@ method: 'POST', body: formData }); + let result; try { result = await response.json(); - } catch { + } catch (err) { + console.error('Error parsing /api/grade response:', err); errorMsg = 'Received an invalid response from the server.'; submitting = false; return; } if (response.ok && result.success) { - // Save to localStorage history + if (result.grade === 'HUMAN_REVIEW_REQUIRED') { + humanReviewRequired = true; + aiReasoning = result.reasoning; + hitlContext = result.hitlContext || { threadId: result.threadId, runId: result.runId }; + submitting = false; + return; + } + try { + // Save to localStorage history let prev = JSON.parse(localStorage.getItem('assessmentHistory') || '[]'); const entry = { timestamp: Date.now(), @@ -100,7 +118,7 @@ localStorage.setItem('assessmentHistory', JSON.stringify(prev.slice(0, 50))); } catch {} - // Redirect to /results + // Redirect to /results if HITL is not needed window.location.assign('/results'); file = null; @@ -108,6 +126,7 @@ taskDescription = ''; return; } else { + console.error('Grade API error or non-success:', response, result); if (result && result.error) { errorMsg = `Sorry, something went wrong: ${result.error}`; } else if (response.status === 413) { @@ -119,12 +138,51 @@ } } } catch (err) { + console.error('Error submitting grade:', err); errorMsg = 'Network or server error. Please check your connection and try again.'; } finally { - // Disabled to make the transition to the results page smoother + // Disabled to make the transition to the results page smoother // submitting = false; } } + + async function submitHumanReview() { + submitting = true; + errorMsg = ''; + try { + const response = await fetch('/api/hitl-review', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + humanReview: humanReviewText, + context: hitlContext + }) + }); + + const result = await response.json(); + + if (response.ok && result.success) { + let prev = JSON.parse(localStorage.getItem('assessmentHistory') || '[]'); + const entry = { + timestamp: Date.now(), + ...result, + submission: result.submission || textInput || '', + task: taskDescription + }; + prev.unshift(entry); + localStorage.setItem('assessmentHistory', JSON.stringify(prev.slice(0, 50))); + window.location.assign('/results'); + } else { + console.error('HITL review API error or non-success:', response, result); + errorMsg = result.error || 'Unknown error during HITL review.'; + } + } catch (err) { + console.error('Error submitting HITL review:', err); + errorMsg = 'Network or server error during HITL review.'; + } finally { + submitting = false; + } + }
@@ -138,13 +196,31 @@ {errorMsg} {/if} - {#if submitting} + + {#if humanReviewRequired} +
+

Human Review Needed

+

{aiReasoning}

+ + +
+ {:else if submitting} {:else}

Upload Student Submissions

Upload student assignments for instant AI-powered assessment and feedback.

-
+ +
+ +
+ If enabled, the AI can ask for human help on unclear or unevaluable submissions. +
+
File upload is disabled for this hackathon demo.
- +