mirror of
https://github.com/jcreek/AutomatedAssessmentFeedbackAgent.git
synced 2026-07-12 18:43:49 +00:00
Add work in progress for refactoring to avoid 10+ second API calls that don't work on Netlify
This commit is contained in:
Josh Creek
+6
-40
@@ -283,12 +283,14 @@ async function processRunStream(
|
||||
throw new Error('Stream ended without completion');
|
||||
}
|
||||
|
||||
export { client, agent };
|
||||
|
||||
export async function gradeSubmissionWithAgent(
|
||||
submission: string,
|
||||
task: string,
|
||||
roomId: string,
|
||||
hitl: boolean = false
|
||||
): Promise<OpenAIResponse> {
|
||||
): Promise<{ threadId: string; runId: string }> {
|
||||
if (!client || !agent) {
|
||||
throw new Error('Agent not available');
|
||||
}
|
||||
@@ -301,52 +303,16 @@ export async function gradeSubmissionWithAgent(
|
||||
: buildGradingPrompt(submission, task)
|
||||
});
|
||||
|
||||
let initialStream: AsyncIterable<RunStreamEvent>;
|
||||
try {
|
||||
const runInvoker = client.agents.createRun(thread.id, agent.id, {
|
||||
const run = await client.agents.createRun(thread.id, agent.id, {
|
||||
parallelToolCalls: false
|
||||
});
|
||||
initialStream = await runInvoker.stream();
|
||||
return { threadId: thread.id, runId: run.id };
|
||||
} catch (err) {
|
||||
logger.error(`Failed to start run on thread ${thread.id}:`, err);
|
||||
logger.info('Task:', task);
|
||||
logger.info('Submission', submission);
|
||||
return fallbackGrade(submission, task);
|
||||
}
|
||||
|
||||
// process all tool calls
|
||||
let finalRun: ThreadRunOutput;
|
||||
try {
|
||||
finalRun = await processRunStream(thread.id, initialStream, roomId);
|
||||
} catch (err) {
|
||||
logger.error(`Agent streaming failed on thread ${thread.id}:`, err);
|
||||
return fallbackGrade(submission, task);
|
||||
}
|
||||
|
||||
const msgs = await client.agents.listMessages(thread.id);
|
||||
const assistant = msgs.data.find((m) => m.role === 'assistant');
|
||||
const raw = assistant ? extractTextFromMessage(assistant) : '';
|
||||
|
||||
// strip anything before the JSON object
|
||||
const match = raw.match(/\{[\s\S]*\}$/);
|
||||
const jsonText = match ? match[0] : raw;
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(jsonText) as OpenAIResponse;
|
||||
if (parsed.grade === 'HUMAN_REVIEW_REQUIRED') {
|
||||
return {
|
||||
...parsed,
|
||||
success: true,
|
||||
threadId: thread.id,
|
||||
runId: finalRun?.id ?? undefined,
|
||||
hitlContext: { threadId: thread.id, runId: finalRun?.id ?? undefined }
|
||||
};
|
||||
}
|
||||
return parsed;
|
||||
} catch (parseErr) {
|
||||
logger.error(`JSON parse failed, returning fallback on thread ${thread.id}:`, parseErr);
|
||||
logger.info('JSON', jsonText);
|
||||
return { ...fallbackGrade(submission, task), reasoning: raw };
|
||||
throw new Error('Failed to start grading run.');
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -35,15 +35,15 @@ export const POST: RequestHandler = async (event: RequestEvent) => {
|
||||
return new Response(JSON.stringify({ error: 'No valid input provided.' }), { status: 400 });
|
||||
}
|
||||
|
||||
// Call the AI grading agent with threading and fallback
|
||||
const aiResult = await gradeSubmissionWithAgent(submission, task.trim(), roomId ?? '', hitl);
|
||||
// Call the AI grading agent asynchronously
|
||||
const { threadId, runId } = await gradeSubmissionWithAgent(submission, task.trim(), roomId ?? '', hitl);
|
||||
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
success: true,
|
||||
task: task.trim(),
|
||||
feedback: `Suggested Grade: ${aiResult.grade}\nStrengths: ${aiResult.strengths}\nAreas for Improvement: ${aiResult.areasForImprovement}\nIndividualized Activity: ${aiResult.individualizedActivity}\nSpelling and Grammar: ${aiResult.spellingAndGrammar}\nReasoning: ${aiResult.reasoning}`,
|
||||
...aiResult
|
||||
threadId,
|
||||
runId
|
||||
}),
|
||||
{
|
||||
status: 200,
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
import type { RequestHandler } from '@sveltejs/kit';
|
||||
import type { RequestEvent } from '@sveltejs/kit';
|
||||
import { client, agent } from '$lib/server/ai';
|
||||
|
||||
// GET /api/grade/status?threadId=...&runId=...
|
||||
export const GET: RequestHandler = async (event: RequestEvent) => {
|
||||
const url = new URL(event.request.url);
|
||||
const threadId = url.searchParams.get('threadId');
|
||||
const runId = url.searchParams.get('runId');
|
||||
|
||||
if (!client || !agent) {
|
||||
return new Response(JSON.stringify({ error: 'Agent not available' }), { status: 500 });
|
||||
}
|
||||
if (!threadId || !runId) {
|
||||
return new Response(JSON.stringify({ error: 'Missing threadId or runId' }), { status: 400 });
|
||||
}
|
||||
|
||||
try {
|
||||
const run = await client.agents.getRun(threadId, runId);
|
||||
// Enhanced: handle all important statuses for polling
|
||||
let toolEvents = [];
|
||||
if (run.tool_calls) {
|
||||
toolEvents = run.tool_calls;
|
||||
}
|
||||
|
||||
let result = null;
|
||||
let parsed = null;
|
||||
let humanReviewRequired = false;
|
||||
let parseError = null;
|
||||
|
||||
// Handle agent run statuses for polling, mirroring processRunStream
|
||||
if (run.status === 'requires_action' && run.requiredAction) {
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
success: true,
|
||||
status: 'requires_action',
|
||||
requiredAction: run.requiredAction,
|
||||
toolCalls: run.requiredAction.submitToolOutputs?.toolCalls || [],
|
||||
toolEvents,
|
||||
message: 'Agent requires tool outputs to proceed.'
|
||||
}),
|
||||
{ status: 200, headers: { 'Content-Type': 'application/json' } }
|
||||
);
|
||||
}
|
||||
if (run.status === 'failed' || run.status === 'cancelling' || run.status === 'cancelled') {
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
success: false,
|
||||
status: run.status,
|
||||
toolEvents,
|
||||
error: run.lastError || 'Run failed or was cancelled.'
|
||||
}),
|
||||
{ status: 500, headers: { 'Content-Type': 'application/json' } }
|
||||
);
|
||||
}
|
||||
if (run.status === 'in_progress' || run.status === 'queued' || run.status === 'starting') {
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
success: true,
|
||||
status: run.status,
|
||||
toolEvents,
|
||||
message: 'Grading in progress.'
|
||||
}),
|
||||
{ status: 200, headers: { 'Content-Type': 'application/json' } }
|
||||
);
|
||||
}
|
||||
|
||||
// Completed and fallback handled below as before
|
||||
if (run.status === 'completed') {
|
||||
const msgs = await client.agents.listMessages(threadId);
|
||||
const assistant = msgs.data.find((m: any) => m.role === 'assistant');
|
||||
const raw = assistant ? (typeof assistant.content === 'string' ? assistant.content : (assistant.content?.[0]?.text || '')) : '';
|
||||
// strip anything before the JSON object
|
||||
const match = raw.match(/\{[\s\S]*\}$/);
|
||||
const jsonText = match ? match[0] : raw;
|
||||
try {
|
||||
parsed = JSON.parse(jsonText);
|
||||
if (parsed.grade === 'HUMAN_REVIEW_REQUIRED') {
|
||||
humanReviewRequired = true;
|
||||
}
|
||||
result = parsed;
|
||||
} catch (err: any) {
|
||||
parseError = err.message || 'Failed to parse AI response.';
|
||||
result = { raw };
|
||||
}
|
||||
}
|
||||
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
success: true,
|
||||
status: run.status,
|
||||
toolEvents,
|
||||
result,
|
||||
humanReviewRequired,
|
||||
parseError
|
||||
}),
|
||||
{ status: 200, headers: { 'Content-Type': 'application/json' } }
|
||||
);
|
||||
} catch (err: any) {
|
||||
return new Response(JSON.stringify({ error: err.message || 'Failed to get run status' }), { status: 500 });
|
||||
}
|
||||
};
|
||||
@@ -40,111 +40,104 @@
|
||||
}
|
||||
|
||||
async function handleSubmit(event: Event) {
|
||||
event.preventDefault();
|
||||
submitting = true;
|
||||
successMsg = '';
|
||||
errorMsg = '';
|
||||
const THINKING_EVENT = { tool: 'thinking', time: new Date().toISOString() };
|
||||
function ensureThinkingStep(events) {
|
||||
// Remove any existing 'thinking' event
|
||||
const filtered = events.filter((e) => e.tool !== 'thinking');
|
||||
return [THINKING_EVENT, ...filtered];
|
||||
}
|
||||
toolEvents = [THINKING_EVENT];
|
||||
event.preventDefault();
|
||||
submitting = true;
|
||||
successMsg = '';
|
||||
errorMsg = '';
|
||||
const THINKING_EVENT = { tool: 'thinking', time: new Date().toISOString() };
|
||||
function ensureThinkingStep(events) {
|
||||
// Remove any existing 'thinking' event
|
||||
const filtered = events.filter((e) => e.tool !== 'thinking');
|
||||
return [THINKING_EVENT, ...filtered];
|
||||
}
|
||||
toolEvents = [THINKING_EVENT];
|
||||
|
||||
// Generate a unique roomId for this submission
|
||||
roomId = crypto.randomUUID();
|
||||
// Connect to PartyKit for this session
|
||||
if (ws) ws.close();
|
||||
ws = new WebSocket(`${PARTYKIT_BASE_URL}/party/tool-usage-server-${roomId}`);
|
||||
ws.onmessage = (event) => {
|
||||
try {
|
||||
const data = JSON.parse(event.data);
|
||||
toolEvents = ensureThinkingStep([...toolEvents, data]);
|
||||
} catch {}
|
||||
};
|
||||
// Generate a unique roomId for this submission
|
||||
roomId = crypto.randomUUID();
|
||||
// Connect to PartyKit for this session
|
||||
if (ws) ws.close();
|
||||
ws = new WebSocket(`${PARTYKIT_BASE_URL}/party/tool-usage-server-${roomId}`);
|
||||
ws.onmessage = (event) => {
|
||||
try {
|
||||
const data = JSON.parse(event.data);
|
||||
toolEvents = ensureThinkingStep([...toolEvents, data]);
|
||||
} catch {}
|
||||
};
|
||||
|
||||
if (!taskDescription.trim()) {
|
||||
errorMsg = 'Please provide a description of the task or assignment.';
|
||||
if (!taskDescription.trim()) {
|
||||
errorMsg = 'Please provide a description of the task or assignment.';
|
||||
submitting = false;
|
||||
return;
|
||||
}
|
||||
const formData = new FormData();
|
||||
formData.append('task', taskDescription.trim());
|
||||
|
||||
if (file) {
|
||||
formData.append('file', file);
|
||||
} else if (textInput.trim().length > 0) {
|
||||
formData.append('text', textInput.trim());
|
||||
}
|
||||
formData.append('roomId', roomId);
|
||||
formData.append('hitl', hitl ? 'true' : 'false');
|
||||
|
||||
try {
|
||||
const response = await fetch('/api/grade', {
|
||||
method: 'POST',
|
||||
body: formData
|
||||
});
|
||||
const data = await response.json();
|
||||
if (!data.success) {
|
||||
errorMsg = data.error || 'Failed to start grading.';
|
||||
submitting = false;
|
||||
return;
|
||||
}
|
||||
const formData = new FormData();
|
||||
formData.append('task', taskDescription.trim());
|
||||
const { threadId, runId } = data;
|
||||
await pollForStatus(threadId, runId);
|
||||
} catch (err: any) {
|
||||
errorMsg = err.message || 'Failed to start grading.';
|
||||
submitting = false;
|
||||
}
|
||||
}
|
||||
|
||||
if (file) {
|
||||
formData.append('file', file);
|
||||
} else if (textInput.trim().length > 0) {
|
||||
formData.append('text', textInput.trim());
|
||||
}
|
||||
formData.append('roomId', roomId);
|
||||
formData.append('hitl', hitl ? 'true' : 'false');
|
||||
|
||||
errorMsg = '';
|
||||
async function pollForStatus(threadId: string, runId: string) {
|
||||
humanReviewRequired = false;
|
||||
aiReasoning = '';
|
||||
hitlContext = null;
|
||||
let polling = true;
|
||||
while (polling) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 1500));
|
||||
try {
|
||||
const response = await fetch('/api/grade', {
|
||||
method: 'POST',
|
||||
body: formData
|
||||
});
|
||||
|
||||
let result;
|
||||
try {
|
||||
result = await response.json();
|
||||
} catch (err) {
|
||||
console.error('Error parsing /api/grade response:', err);
|
||||
errorMsg = 'Received an invalid response from the server.';
|
||||
const res = await fetch(`/api/grade/status?threadId=${encodeURIComponent(threadId)}&runId=${encodeURIComponent(runId)}`);
|
||||
const statusData = await res.json();
|
||||
if (!statusData.success) {
|
||||
errorMsg = statusData.error || 'Unknown error during grading.';
|
||||
submitting = false;
|
||||
return;
|
||||
}
|
||||
if (response.ok && result.success) {
|
||||
if (result.grade === 'HUMAN_REVIEW_REQUIRED') {
|
||||
humanReviewRequired = true;
|
||||
aiReasoning = result.reasoning;
|
||||
hitlContext = result.hitlContext || { threadId: result.threadId, runId: result.runId };
|
||||
submitting = false;
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
// Save to localStorage history
|
||||
let prev = JSON.parse(localStorage.getItem('assessmentHistory') || '[]');
|
||||
const entry = {
|
||||
timestamp: Date.now(),
|
||||
...result,
|
||||
submission: result.submission || textInput || '',
|
||||
task: taskDescription
|
||||
};
|
||||
prev.unshift(entry);
|
||||
localStorage.setItem('assessmentHistory', JSON.stringify(prev.slice(0, 50)));
|
||||
} catch {}
|
||||
|
||||
// Redirect to /results if HITL is not needed
|
||||
window.location.assign('/results');
|
||||
|
||||
file = null;
|
||||
textInput = '';
|
||||
taskDescription = '';
|
||||
return;
|
||||
} else {
|
||||
console.error('Grade API error or non-success:', response, result);
|
||||
if (result && result.error) {
|
||||
errorMsg = `Sorry, something went wrong: ${result.error}`;
|
||||
} else if (response.status === 413) {
|
||||
errorMsg = 'The uploaded file is too large. Please upload a smaller file.';
|
||||
} else if (response.status >= 500) {
|
||||
errorMsg = 'The server encountered an error. Please try again later.';
|
||||
} else {
|
||||
errorMsg = 'An unknown error occurred. Please check your input and try again.';
|
||||
toolEvents = statusData.toolEvents || toolEvents;
|
||||
if (statusData.status === 'completed') {
|
||||
polling = false;
|
||||
submitting = false;
|
||||
if (statusData.result) {
|
||||
const result = statusData.result;
|
||||
aiReasoning = result.reasoning || '';
|
||||
if (statusData.humanReviewRequired) {
|
||||
humanReviewRequired = true;
|
||||
hitlContext = { threadId, runId };
|
||||
} else {
|
||||
successMsg = `Suggested Grade: ${result.grade}\nStrengths: ${result.strengths}\nAreas for Improvement: ${result.areasForImprovement}\nIndividualized Activity: ${result.individualizedActivity}\nSpelling and Grammar: ${result.spellingAndGrammar}\nReasoning: ${result.reasoning}`;
|
||||
}
|
||||
} else if (statusData.parseError) {
|
||||
errorMsg = statusData.parseError;
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Error submitting grade:', err);
|
||||
errorMsg = 'Network or server error. Please check your connection and try again.';
|
||||
} finally {
|
||||
// Disabled to make the transition to the results page smoother
|
||||
// submitting = false;
|
||||
} catch (err: any) {
|
||||
errorMsg = err.message || 'Failed to poll grading status.';
|
||||
submitting = false;
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function submitHumanReview() {
|
||||
submitting = true;
|
||||
|
||||
Reference in New Issue
Block a user