feat(*): Enable HITL functionality

src/lib/agent/services/agentService.ts

@@ -1,42 +1,42 @@
 import { DefaultAzureCredential } from '@azure/identity';
 import { AIProjectsClient, ToolUtility } from '@azure/ai-projects';
 import {
-  rubricMatcherTool,
-  essayAnalyzerTool,
-  conceptVerifierTool,
-  feedbackGeneratorTool,
-  metacognitiveReflectionPromptTool,
-  selfAssessmentTool,
-  spellingAndGrammarCheckerTool
+	rubricMatcherTool,
+	essayAnalyzerTool,
+	conceptVerifierTool,
+	feedbackGeneratorTool,
+	metacognitiveReflectionPromptTool,
+	selfAssessmentTool,
+	spellingAndGrammarCheckerTool
 } from '../tools/index';
 import { AI_FOUNDRY_PROJECT_CONNECTION_STRING, AI_MODEL } from '$env/static/private';
 
 if (!AI_FOUNDRY_PROJECT_CONNECTION_STRING) {
-  throw new Error('AI_FOUNDRY_PROJECT_CONNECTION_STRING is not set');
+	throw new Error('AI_FOUNDRY_PROJECT_CONNECTION_STRING is not set');
 }
 if (!AI_MODEL) {
-  throw new Error('AI_MODEL is not set');
+	throw new Error('AI_MODEL is not set');
 }
 
 export const client = AIProjectsClient.fromConnectionString(
-  AI_FOUNDRY_PROJECT_CONNECTION_STRING,
-  new DefaultAzureCredential()
+	AI_FOUNDRY_PROJECT_CONNECTION_STRING,
+	new DefaultAzureCredential()
 );
 
 // const codeInterpreterTool = ToolUtility.createCodeInterpreterTool([]);
 
 const allTools = [
-  // codeInterpreterTool,
-  rubricMatcherTool,
-  essayAnalyzerTool,
-  conceptVerifierTool,
-  feedbackGeneratorTool,
-  metacognitiveReflectionPromptTool,
-  selfAssessmentTool,
-  spellingAndGrammarCheckerTool
+	// codeInterpreterTool,
+	rubricMatcherTool,
+	essayAnalyzerTool,
+	conceptVerifierTool,
+	feedbackGeneratorTool,
+	metacognitiveReflectionPromptTool,
+	selfAssessmentTool,
+	spellingAndGrammarCheckerTool
 ];
 
-const tools = allTools.map(t => t.definition);
+const tools = allTools.map((t) => t.definition);
 
 const instructions = `
 You are a helpful agent that can assist with providing feedback on a student's work for a teacher.
@@ -49,45 +49,44 @@ Whenever you need to:
  • craft a metacognitive prompt → call "${metacognitiveReflectionPromptTool.definition.name}"  
  • guide self-assessment → call "${selfAssessmentTool.definition.name}"  
  • check spelling & grammar → call "${spellingAndGrammarCheckerTool.definition.name}"  
+
+ If a user message begins with [HUMAN REVIEW]:, treat it as authoritative teacher feedback and use it as the basis for grading. Do not escalate to human review again.
 `;
 
 export const toolResources = allTools.reduce<Record<string, any>>((map, t) => {
-  // Try all plausible locations for the tool name
-  const name =
-    t.definition.name ||
-    t.definition.function?.name ||
-    Object.keys(t.resources ?? {})[0];
+	// Try all plausible locations for the tool name
+	const name =
+		t.definition.name || t.definition.function?.name || Object.keys(t.resources ?? {})[0];
 
-  if (!name) {
-    console.warn('Tool missing name:', t);
-    return map;
-  }
-  map[name] = t;
-  return map;
+	if (!name) {
+		console.warn('Tool missing name:', t);
+		return map;
+	}
+	map[name] = t;
+	return map;
 }, {});
 
 export async function getOrCreateAgent() {
-  
-  const agentName = `assessment-feedback-agent`;
-  // List all agents (may need pagination for large numbers)
-  const agentsResponse = await client.agents.listAgents();
-  const agents = Array.isArray(agentsResponse.data) ? agentsResponse.data : [];
-  const existing = agents.find((a: any) => a.name === agentName);
-  if (existing) {
-    return existing;
-  }
-  // Otherwise, create new agent
-  const agent = await client.agents.createAgent(AI_MODEL, {
-    name: agentName,
-    instructions,
-    temperature: 0.5,
-    tools,
-    toolResources,
-    requestOptions: {
-      headers: { 'x-ms-enable-preview': 'true' }
-    }
-  });
-  return agent;
+	const agentName = `assessment-feedback-agent`;
+	// List all agents (may need pagination for large numbers)
+	const agentsResponse = await client.agents.listAgents();
+	const agents = Array.isArray(agentsResponse.data) ? agentsResponse.data : [];
+	const existing = agents.find((a: any) => a.name === agentName);
+	if (existing) {
+		return existing;
+	}
+	// Otherwise, create new agent
+	const agent = await client.agents.createAgent(AI_MODEL, {
+		name: agentName,
+		instructions,
+		temperature: 0.5,
+		tools,
+		toolResources,
+		requestOptions: {
+			headers: { 'x-ms-enable-preview': 'true' }
+		}
+	});
+	return agent;
 }
 
 export const agent = await getOrCreateAgent();
@@ -97,29 +96,29 @@ export const agent = await getOrCreateAgent();
  * Use with caution! This will remove ALL agents.
  */
 export async function deleteAllAgents() {
-  try {
-    let allAgents: any[] = [];
-    let after: string | undefined = undefined;
-    let hasMore = true;
+	try {
+		let allAgents: any[] = [];
+		let after: string | undefined = undefined;
+		let hasMore = true;
 
-    while (hasMore) {
-      const response = await client.agents.listAgents(after ? { after } : {});
-      const agents = Array.isArray(response.data) ? response.data : [];
-      allAgents = allAgents.concat(agents);
-      hasMore = response.hasMore;
-      after = response.lastId;
-    }
+		while (hasMore) {
+			const response = await client.agents.listAgents(after ? { after } : {});
+			const agents = Array.isArray(response.data) ? response.data : [];
+			allAgents = allAgents.concat(agents);
+			hasMore = response.hasMore;
+			after = response.lastId;
+		}
 
-    for (const agent of allAgents) {
-      await client.agents.deleteAgent(agent.id, {
-        requestOptions: {
-          headers: { 'x-ms-enable-preview': 'true' }
-        }
-      });
-      console.info(`Agent ${agent.id} (${agent.name}) deleted.`);
-    }
-    console.info('All agents deleted successfully.');
-  } catch (err) {
-    console.warn('Failed to delete all agents:', err);
-  }
+		for (const agent of allAgents) {
+			await client.agents.deleteAgent(agent.id, {
+				requestOptions: {
+					headers: { 'x-ms-enable-preview': 'true' }
+				}
+			});
+			console.info(`Agent ${agent.id} (${agent.name}) deleted.`);
+		}
+		console.info('All agents deleted successfully.');
+	} catch (err) {
+		console.warn('Failed to delete all agents:', err);
+	}
 }

src/lib/server/ai.ts

@@ -40,7 +40,10 @@ RESPONSE FORMAT (respond with a single JSON object, no extra text):
 Respond ONLY with the JSON object, with no preamble or explanation.`;
 }
 
-export function buildGradingPromptWithSupportForHumanInTheLoop(submission: string, task: string): string {
+export function buildGradingPromptWithSupportForHumanInTheLoop(
+	submission: string,
+	task: string
+): string {
 	return `You are an expert secondary school teacher and AI assessment agent. Assess the following student submission in the context of the assignment/task provided.
 
 TASK/ASSIGNMENT:
@@ -128,7 +131,6 @@ EXAMPLES:
 - Reasoning: The task and submission are both clear and evaluatable without a rubric.
 
 Respond ONLY with the JSON object, with no preamble or explanation.`;
-
 }
 
 function fallbackGrade(submission: string, task: string): OpenAIResponse {
@@ -223,7 +225,8 @@ async function processRunStream(
 export async function gradeSubmissionWithAgent(
 	submission: string,
 	task: string,
-	roomId: string
+	roomId: string,
+	hitl: boolean = false
 ): Promise<OpenAIResponse> {
 	if (!client || !agent) {
 		throw new Error('Agent not available');
@@ -232,7 +235,9 @@ export async function gradeSubmissionWithAgent(
 	const thread = await client.agents.createThread();
 	await client.agents.createMessage(thread.id, {
 		role: 'user',
-		content: buildGradingPrompt(submission, task)
+		content: hitl
+			? buildGradingPromptWithSupportForHumanInTheLoop(submission, task)
+			: buildGradingPrompt(submission, task)
 	});
 
 	let initialStream: AsyncIterable<RunStreamEvent>;
@@ -264,9 +269,65 @@ export async function gradeSubmissionWithAgent(
 	const jsonText = match ? match[0] : raw;
 
 	try {
-		return JSON.parse(jsonText) as OpenAIResponse;
+		const parsed = JSON.parse(jsonText) as OpenAIResponse;
+		if (parsed.grade === 'HUMAN_REVIEW_REQUIRED') {
+			return {
+				...parsed,
+				success: true,
+				threadId: thread.id,
+				runId: finalRun?.id ?? undefined,
+				hitlContext: { threadId: thread.id, runId: finalRun?.id ?? undefined }
+			};
+		}
+		return parsed;
 	} catch (parseErr) {
 		console.error('JSON parse failed, returning fallback:', parseErr);
 		return { ...fallbackGrade(submission, task), reasoning: raw };
 	}
 }
+
+export async function resumeAgentWithHumanReview(
+	humanReview: string,
+	context: { threadId: string; runId?: string; roomId?: string; [key: string]: any }
+): Promise<OpenAIResponse> {
+	const { threadId, roomId = '' } = context;
+	await client.agents.createMessage(threadId, {
+		role: 'user',
+		content: `[HUMAN REVIEW]: ${humanReview}`
+	});
+
+	let stream: AsyncIterable<RunStreamEvent>;
+	try {
+		const runInvoker = client.agents.createRun(threadId, agent.id, {
+			parallelToolCalls: false
+		});
+		stream = await runInvoker.stream();
+	} catch (err) {
+		console.error('Failed to start run:', err);
+		return fallbackGrade('', '');
+	}
+
+	let finalRun: ThreadRunOutput;
+	try {
+		finalRun = await processRunStream(threadId, stream, roomId);
+	} catch (err) {
+		console.error('Agent streaming failed:', err);
+		return fallbackGrade('', '');
+	}
+
+	const msgs = await client.agents.listMessages(threadId);
+	const assistant = msgs.data.find((m) => m.role === 'assistant');
+	const raw = assistant ? extractTextFromMessage(assistant) : '';
+
+	// strip anything before the JSON object
+	const match = raw.match(/\{[\s\S]*\}$/);
+	const jsonText = match ? match[0] : raw;
+
+	try {
+		const parsed = JSON.parse(jsonText) as OpenAIResponse;
+		return parsed;
+	} catch (parseErr) {
+		console.error('JSON parse failed, returning fallback:', parseErr);
+		return { ...fallbackGrade('', ''), reasoning: raw };
+	}
+}

src/routes/api/grade/+server.ts

@@ -3,45 +3,56 @@ import type { RequestEvent } from '@sveltejs/kit';
 import { gradeSubmissionWithAgent } from '$lib/server/ai';
 
 export const POST: RequestHandler = async (event: RequestEvent) => {
-  try {
-    const data = await event.request.formData();
-    const file = data.get('file');
-    const text = data.get('text');
-    const task = data.get('task');
-    const roomId = data.get('roomId') as string | undefined;
-    let submission = '';
+	try {
+		const data = await event.request.formData();
+		const file = data.get('file');
+		const text = data.get('text');
+		const task = data.get('task');
+		const roomId = data.get('roomId') as string | undefined;
+		const hitl = data.get('hitl') === 'true';
+		let submission = '';
 
-    if (!task || typeof task !== 'string' || !task.trim()) {
-      return new Response(JSON.stringify({ error: 'Task/assignment description is required.' }), { status: 400 });
-    }
+		if (!task || typeof task !== 'string' || !task.trim()) {
+			return new Response(JSON.stringify({ error: 'Task/assignment description is required.' }), {
+				status: 400
+			});
+		}
 
-    if (file && typeof file === 'object' && 'arrayBuffer' in file && 'name' in file) {
-      // Only support .txt for now for hackathon speed
-      const fileObj = file as File;
-      if (!fileObj.name.endsWith('.txt')) {
-        return new Response(JSON.stringify({ error: 'Only .txt files are supported at this time.' }), { status: 400 });
-      }
-      const buffer = await fileObj.arrayBuffer();
-      submission = new TextDecoder('utf-8').decode(buffer);
-    } else if (typeof text === 'string' && text.trim().length > 0) {
-      submission = text.trim();
-    } else {
-      return new Response(JSON.stringify({ error: 'No valid input provided.' }), { status: 400 });
-    }
+		if (file && typeof file === 'object' && 'arrayBuffer' in file && 'name' in file) {
+			// Only support .txt for now for hackathon speed
+			const fileObj = file as File;
+			if (!fileObj.name.endsWith('.txt')) {
+				return new Response(
+					JSON.stringify({ error: 'Only .txt files are supported at this time.' }),
+					{ status: 400 }
+				);
+			}
+			const buffer = await fileObj.arrayBuffer();
+			submission = new TextDecoder('utf-8').decode(buffer);
+		} else if (typeof text === 'string' && text.trim().length > 0) {
+			submission = text.trim();
+		} else {
+			return new Response(JSON.stringify({ error: 'No valid input provided.' }), { status: 400 });
+		}
 
-    // Call the AI grading agent with threading and fallback
-    const aiResult = await gradeSubmissionWithAgent(submission, task.trim(), roomId);
+		// Call the AI grading agent with threading and fallback
+		const aiResult = await gradeSubmissionWithAgent(submission, task.trim(), roomId ?? '', hitl);
 
-    return new Response(JSON.stringify({
-      success: true,
-      task: task.trim(),
-      feedback: `Grade: ${aiResult.grade}\nStrengths: ${aiResult.strengths}\nAreas for Improvement: ${aiResult.areasForImprovement}\nIndividualized Activity: ${aiResult.individualizedActivity}\nSpelling and Grammar: ${aiResult.spellingAndGrammar}\nReasoning: ${aiResult.reasoning}`,
-      ...aiResult
-    }), {
-      status: 200,
-      headers: { 'Content-Type': 'application/json' }
-    });
-  } catch (err: any) {
-    return new Response(JSON.stringify({ error: err.message || 'Internal server error.' }), { status: 500 });
-  }
+		return new Response(
+			JSON.stringify({
+				success: true,
+				task: task.trim(),
+				feedback: `Grade: ${aiResult.grade}\nStrengths: ${aiResult.strengths}\nAreas for Improvement: ${aiResult.areasForImprovement}\nIndividualized Activity: ${aiResult.individualizedActivity}\nSpelling and Grammar: ${aiResult.spellingAndGrammar}\nReasoning: ${aiResult.reasoning}`,
+				...aiResult
+			}),
+			{
+				status: 200,
+				headers: { 'Content-Type': 'application/json' }
+			}
+		);
+	} catch (err: any) {
+		return new Response(JSON.stringify({ error: err.message || 'Internal server error.' }), {
+			status: 500
+		});
+	}
 };

src/routes/api/hitl-review/+server.ts

@@ -0,0 +1,23 @@
+import type { RequestHandler } from '@sveltejs/kit';
+import { resumeAgentWithHumanReview } from '$lib/server/ai';
+
+export const POST: RequestHandler = async (event) => {
+	try {
+		const { humanReview, context } = await event.request.json();
+		const aiResult = await resumeAgentWithHumanReview(humanReview, context);
+		return new Response(
+			JSON.stringify({
+				success: true,
+				...aiResult
+			}),
+			{
+				status: 200,
+				headers: { 'Content-Type': 'application/json' }
+			}
+		);
+	} catch (err: any) {
+		return new Response(JSON.stringify({ error: err.message || 'Internal server error.' }), {
+			status: 500
+		});
+	}
+};

src/routes/upload/+page.svelte

@@ -9,9 +9,16 @@
 	let errorMsg = '';
 	let toolEvents: Array<{ tool: string; time: string }> = [];
 	let roomId: string = '';
+	let hitl = false; // Human-in-the-Loop toggle
 	let ws: WebSocket | null = null;
 	const PARTYKIT_BASE_URL = import.meta.env.VITE_PARTYKIT_BASE_URL;
 
+	// HITL state
+	let humanReviewRequired = false;
+	let aiReasoning = '';
+	let humanReviewText = '';
+	let hitlContext: any = null;
+
 	function handleFileChange(event: Event) {
 		const target = event.target as HTMLInputElement;
 		if (target.files && target.files.length > 0) {
@@ -40,7 +47,7 @@
 		const THINKING_EVENT = { tool: 'thinking', time: new Date().toISOString() };
 		function ensureThinkingStep(events) {
 			// Remove any existing 'thinking' event
-			const filtered = events.filter(e => e.tool !== 'thinking');
+			const filtered = events.filter((e) => e.tool !== 'thinking');
 			return [THINKING_EVENT, ...filtered];
 		}
 		toolEvents = [THINKING_EVENT];
@@ -71,6 +78,7 @@
 			formData.append('text', textInput.trim());
 		}
 		formData.append('roomId', roomId);
+		formData.append('hitl', hitl ? 'true' : 'false');
 
 		errorMsg = '';
 		try {
@@ -78,17 +86,27 @@
 				method: 'POST',
 				body: formData
 			});
+
 			let result;
 			try {
 				result = await response.json();
-			} catch {
+			} catch (err) {
+				console.error('Error parsing /api/grade response:', err);
 				errorMsg = 'Received an invalid response from the server.';
 				submitting = false;
 				return;
 			}
 			if (response.ok && result.success) {
-				// Save to localStorage history
+				if (result.grade === 'HUMAN_REVIEW_REQUIRED') {
+					humanReviewRequired = true;
+					aiReasoning = result.reasoning;
+					hitlContext = result.hitlContext || { threadId: result.threadId, runId: result.runId };
+					submitting = false;
+					return;
+				}
+
 				try {
+					// Save to localStorage history
 					let prev = JSON.parse(localStorage.getItem('assessmentHistory') || '[]');
 					const entry = {
 						timestamp: Date.now(),
@@ -100,7 +118,7 @@
 					localStorage.setItem('assessmentHistory', JSON.stringify(prev.slice(0, 50)));
 				} catch {}
 
-				// Redirect to /results
+				// Redirect to /results if HITL is not needed
 				window.location.assign('/results');
 
 				file = null;
@@ -108,6 +126,7 @@
 				taskDescription = '';
 				return;
 			} else {
+				console.error('Grade API error or non-success:', response, result);
 				if (result && result.error) {
 					errorMsg = `Sorry, something went wrong: ${result.error}`;
 				} else if (response.status === 413) {
@@ -119,12 +138,51 @@
 				}
 			}
 		} catch (err) {
+			console.error('Error submitting grade:', err);
 			errorMsg = 'Network or server error. Please check your connection and try again.';
 		} finally {
-      // Disabled to make the transition to the results page smoother
+			// Disabled to make the transition to the results page smoother
 			// submitting = false;
 		}
 	}
+
+	async function submitHumanReview() {
+		submitting = true;
+		errorMsg = '';
+		try {
+			const response = await fetch('/api/hitl-review', {
+				method: 'POST',
+				headers: { 'Content-Type': 'application/json' },
+				body: JSON.stringify({
+					humanReview: humanReviewText,
+					context: hitlContext
+				})
+			});
+
+			const result = await response.json();
+
+			if (response.ok && result.success) {
+				let prev = JSON.parse(localStorage.getItem('assessmentHistory') || '[]');
+				const entry = {
+					timestamp: Date.now(),
+					...result,
+					submission: result.submission || textInput || '',
+					task: taskDescription
+				};
+				prev.unshift(entry);
+				localStorage.setItem('assessmentHistory', JSON.stringify(prev.slice(0, 50)));
+				window.location.assign('/results');
+			} else {
+				console.error('HITL review API error or non-success:', response, result);
+				errorMsg = result.error || 'Unknown error during HITL review.';
+			}
+		} catch (err) {
+			console.error('Error submitting HITL review:', err);
+			errorMsg = 'Network or server error during HITL review.';
+		} finally {
+			submitting = false;
+		}
+	}
 </script>
 
 <section class="relative mx-auto max-w-4xl py-12">
@@ -138,13 +196,31 @@
 			{errorMsg}
 		</div>
 	{/if}
-	{#if submitting}
+
+	{#if humanReviewRequired}
+		<div class="mb-6 rounded border-l-4 border-yellow-400 bg-yellow-50 p-4">
+			<h2 class="mb-2 font-bold text-yellow-800">Human Review Needed</h2>
+			<p class="mb-3 text-gray-800">{aiReasoning}</p>
+			<textarea
+				bind:value={humanReviewText}
+				rows="4"
+				class="mb-3 w-full rounded border p-2"
+				placeholder="Enter your feedback, grade, or comments..."
+			></textarea>
+			<button
+				class="rounded bg-blue-600 px-4 py-2 text-white hover:bg-blue-700"
+				on:click|preventDefault={submitHumanReview}
+				disabled={submitting || !humanReviewText.trim()}
+			>
+				Submit Review
+			</button>
+		</div>
+	{:else if submitting}
 		<AgenticProgress {toolEvents} />
 	{:else}
 		<h1 class="mb-4 text-2xl font-bold">Upload Student Submissions</h1>
 		<p class="mb-6">Upload student assignments for instant AI-powered assessment and feedback.</p>
 		<form class="space-y-6" on:submit|preventDefault={handleSubmit}>
-
 			<div class="mb-4">
 				<label class="mb-1 block font-semibold" for="task"
 					>Task/Assignment Description <span class="text-red-500">*</span></label
@@ -161,20 +237,48 @@
 					data-testid="task-input"
 				></textarea>
 			</div>
+
+			<div class="mb-4">
+				<label class="flex cursor-pointer items-center gap-3 select-none" for="hitl-toggle">
+					<span class="relative inline-block h-7 w-12 align-middle select-none">
+						<input
+							type="checkbox"
+							id="hitl-toggle"
+							class="peer absolute h-0 w-0 opacity-0"
+							bind:checked={hitl}
+							disabled={submitting}
+						/>
+						<span
+							class="block h-7 w-12 rounded-full bg-gray-300 transition-colors peer-checked:bg-blue-600"
+						></span>
+						<span
+							class="dot absolute top-1 left-1 h-5 w-5 rounded-full bg-white shadow transition-transform peer-checked:translate-x-5"
+						></span>
+					</span>
+					<span class="font-medium text-gray-900"
+						>EXPERIMENTAL: Let the agent request human review if needed (Human-in-the-Loop)</span
+					>
+				</label>
+				<div class="mt-1 ml-1 text-sm text-gray-500">
+					If enabled, the AI can ask for human help on unclear or unevaluable submissions.
+				</div>
+			</div>
 			<div class="mb-4">
 				<label class="mb-1 block font-semibold" for="file">Upload a file (PDF, DOCX, TXT):</label>
 				<input
 					id="file"
 					type="file"
 					accept=".pdf,.doc,.docx,.txt"
-					class="block w-full rounded border p-2 bg-gray-100 cursor-not-allowed"
+					class="block w-full cursor-not-allowed rounded border bg-gray-100 p-2"
 					on:change={handleFileChange}
 					disabled
 				/>
 				<span class="text-sm text-red-600">File upload is disabled for this hackathon demo.</span>
 			</div>
 			<div class="mb-4">
-				<label class="mb-1 block font-semibold" for="textInput">Paste assignment text: <span class="text-red-500">*</span></label>
+				<label class="mb-1 block font-semibold" for="textInput"
+					>Paste assignment text: <span class="text-red-500">*</span></label
+				>
 				<textarea
 					id="textInput"
 					rows="5"