mirror of
https://github.com/jcreek/AutomatedAssessmentFeedbackAgent.git
synced 2026-07-13 02:53:49 +00:00
refactor(*): Change CI to use secrets instead of fallbacks
This commit is contained in:
Josh Creek
+115
-131
@@ -1,14 +1,10 @@
|
||||
import { client, agent, toolResources } from '../agent/services/agentService';
|
||||
import {
|
||||
RunStreamEvent,
|
||||
ErrorEvent,
|
||||
type ThreadRunOutput
|
||||
} from '@azure/ai-projects';
|
||||
import { RunStreamEvent, ErrorEvent, type ThreadRunOutput } from '@azure/ai-projects';
|
||||
import { PARTYKIT_BASE_URL } from '$env/static/private';
|
||||
import type { OpenAIResponse } from './types';
|
||||
|
||||
export function buildGradingPrompt(submission: string, task: string): string {
|
||||
return `You are an expert secondary school teacher and AI assessment agent. Assess the following student submission in the context of the assignment/task provided.
|
||||
return `You are an expert secondary school teacher and AI assessment agent. Assess the following student submission in the context of the assignment/task provided.
|
||||
|
||||
TASK/ASSIGNMENT:
|
||||
${task}
|
||||
@@ -45,153 +41,141 @@ Respond ONLY with the JSON object, with no preamble or explanation.`;
|
||||
}
|
||||
|
||||
function fallbackGrade(submission: string, task: string): OpenAIResponse {
|
||||
console.warn('Using fallback grade logic');
|
||||
return {
|
||||
grade: 'EXAMPLE',
|
||||
strengths: `Clear argument and good evidence (task: ${task}).`,
|
||||
areasForImprovement: 'Needs deeper analysis.',
|
||||
individualizedActivity: 'Add two more supporting examples.',
|
||||
reflectionQuestion: 'Which part was hardest?',
|
||||
teacherSuggestion: 'Model a paragraph with evidence.',
|
||||
spellingAndGrammar: 'No errors detected.',
|
||||
reasoning: 'Standard fallback reasoning.'
|
||||
};
|
||||
console.warn('Using fallback grade logic');
|
||||
return {
|
||||
grade: 'EXAMPLE',
|
||||
strengths: `Clear argument and good evidence (task: ${task}).`,
|
||||
areasForImprovement: 'Needs deeper analysis.',
|
||||
individualizedActivity: 'Add two more supporting examples.',
|
||||
reflectionQuestion: 'Which part was hardest?',
|
||||
teacherSuggestion: 'Model a paragraph with evidence.',
|
||||
spellingAndGrammar: 'No errors detected.',
|
||||
reasoning: 'Standard fallback reasoning.'
|
||||
};
|
||||
}
|
||||
|
||||
function extractTextFromMessage(msg: { content: any[] }): string {
|
||||
return msg.content
|
||||
.filter(c => c.type === 'text')
|
||||
.map(c => (typeof c.text === 'string' ? c.text : c.text.value ?? ''))
|
||||
.join('\n')
|
||||
.trim();
|
||||
return msg.content
|
||||
.filter((c) => c.type === 'text')
|
||||
.map((c) => (typeof c.text === 'string' ? c.text : (c.text.value ?? '')))
|
||||
.join('\n')
|
||||
.trim();
|
||||
}
|
||||
|
||||
// Recursively process each run stream, invoke the tools & notify PartyKit
|
||||
async function processRunStream(
|
||||
threadId: string,
|
||||
stream: AsyncIterable<RunStreamEvent>,
|
||||
roomId: string
|
||||
threadId: string,
|
||||
stream: AsyncIterable<RunStreamEvent>,
|
||||
roomId: string
|
||||
): Promise<ThreadRunOutput> {
|
||||
for await (const evt of stream) {
|
||||
// if (!evt.event.includes('delta')) {
|
||||
// console.log('▶️ Stream event:', evt.event);
|
||||
// }
|
||||
for await (const evt of stream) {
|
||||
// if (!evt.event.includes('delta')) {
|
||||
// console.log('▶️ Stream event:', evt.event);
|
||||
// }
|
||||
|
||||
if (evt.event === RunStreamEvent.ThreadRunRequiresAction) {
|
||||
const runOutput = evt.data as ThreadRunOutput;
|
||||
const calls = runOutput.requiredAction!.submitToolOutputs!.toolCalls!;
|
||||
if (evt.event === RunStreamEvent.ThreadRunRequiresAction) {
|
||||
const runOutput = evt.data as ThreadRunOutput;
|
||||
const calls = runOutput.requiredAction!.submitToolOutputs!.toolCalls!;
|
||||
|
||||
// notify PartyKit
|
||||
await Promise.all(
|
||||
calls.map(async call => {
|
||||
if (!PARTYKIT_BASE_URL) return;
|
||||
const wsCtor = typeof WebSocket !== 'undefined'
|
||||
? WebSocket
|
||||
: (await import('ws')).default;
|
||||
const ws = new wsCtor(
|
||||
`${PARTYKIT_BASE_URL}/party/tool-usage-server-${roomId}`
|
||||
);
|
||||
await new Promise<void>((res, rej) => {
|
||||
ws.onopen = () => res();
|
||||
ws.onerror = e => rej(e);
|
||||
});
|
||||
ws.send(JSON.stringify({ tool: call.function.name, time: new Date().toISOString() }));
|
||||
ws.close();
|
||||
})
|
||||
);
|
||||
// notify PartyKit
|
||||
await Promise.all(
|
||||
calls.map(async (call) => {
|
||||
if (!PARTYKIT_BASE_URL) return;
|
||||
const wsCtor =
|
||||
typeof WebSocket !== 'undefined' ? WebSocket : (await import('ws')).default;
|
||||
const ws = new wsCtor(`${PARTYKIT_BASE_URL}/party/tool-usage-server-${roomId}`);
|
||||
await new Promise<void>((res, rej) => {
|
||||
ws.onopen = () => res();
|
||||
ws.onerror = (e) => rej(e);
|
||||
});
|
||||
ws.send(JSON.stringify({ tool: call.function.name, time: new Date().toISOString() }));
|
||||
ws.close();
|
||||
})
|
||||
);
|
||||
|
||||
// invoke the tools
|
||||
const toolOutputs = await Promise.all(
|
||||
calls.map(async call => {
|
||||
const name = call.function.name;
|
||||
const args = call.arguments!;
|
||||
try {
|
||||
const fn = toolResources[name];
|
||||
if (typeof fn !== 'function') {
|
||||
throw new Error(`No tool implementation for "${name}"`);
|
||||
}
|
||||
const result = await fn(args);
|
||||
return { toolCallId: call.id, output: result };
|
||||
} catch (err: any) {
|
||||
return { toolCallId: call.id, output: `Tool error: ${err.message}` };
|
||||
}
|
||||
})
|
||||
);
|
||||
// invoke the tools
|
||||
const toolOutputs = await Promise.all(
|
||||
calls.map(async (call) => {
|
||||
const name = call.function.name;
|
||||
const args = call.arguments!;
|
||||
try {
|
||||
const fn = toolResources[name];
|
||||
if (typeof fn !== 'function') {
|
||||
throw new Error(`No tool implementation for "${name}"`);
|
||||
}
|
||||
const result = await fn(args);
|
||||
return { toolCallId: call.id, output: result };
|
||||
} catch (err: any) {
|
||||
return { toolCallId: call.id, output: `Tool error: ${err.message}` };
|
||||
}
|
||||
})
|
||||
);
|
||||
|
||||
// resume the run
|
||||
const resumed = client.agents.submitToolOutputsToRun(
|
||||
threadId,
|
||||
runOutput.id,
|
||||
toolOutputs
|
||||
);
|
||||
const nextStream = await resumed.stream();
|
||||
return processRunStream(threadId, nextStream, roomId);
|
||||
}
|
||||
// resume the run
|
||||
const resumed = client.agents.submitToolOutputsToRun(threadId, runOutput.id, toolOutputs);
|
||||
const nextStream = await resumed.stream();
|
||||
return processRunStream(threadId, nextStream, roomId);
|
||||
}
|
||||
|
||||
if (evt.event === RunStreamEvent.ThreadRunCompleted) {
|
||||
return evt.data as ThreadRunOutput;
|
||||
}
|
||||
if (evt.event === RunStreamEvent.ThreadRunCompleted) {
|
||||
return evt.data as ThreadRunOutput;
|
||||
}
|
||||
|
||||
if (evt.event === ErrorEvent.Error) {
|
||||
throw new Error(`Agent error: ${JSON.stringify(evt.data)}`);
|
||||
}
|
||||
}
|
||||
if (evt.event === ErrorEvent.Error) {
|
||||
throw new Error(`Agent error: ${JSON.stringify(evt.data)}`);
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error('Stream ended without completion');
|
||||
throw new Error('Stream ended without completion');
|
||||
}
|
||||
|
||||
export async function gradeSubmissionWithAgent(
|
||||
submission: string,
|
||||
task: string,
|
||||
roomId: string
|
||||
submission: string,
|
||||
task: string,
|
||||
roomId: string
|
||||
): Promise<OpenAIResponse> {
|
||||
if (!client) {
|
||||
// In test/CI mode, return fallback data
|
||||
return fallbackGrade(submission, task);
|
||||
}
|
||||
if (!client || !agent) {
|
||||
throw new Error('Agent not available');
|
||||
}
|
||||
|
||||
if (!agent) {
|
||||
throw new Error('Agent not available');
|
||||
}
|
||||
const thread = await client.agents.createThread();
|
||||
await client.agents.createMessage(thread.id, {
|
||||
role: 'user',
|
||||
content: buildGradingPrompt(submission, task)
|
||||
});
|
||||
|
||||
const thread = await client.agents.createThread();
|
||||
await client.agents.createMessage(thread.id, {
|
||||
role: 'user',
|
||||
content: buildGradingPrompt(submission, task)
|
||||
});
|
||||
let initialStream: AsyncIterable<RunStreamEvent>;
|
||||
try {
|
||||
const runInvoker = client.agents.createRun(thread.id, agent.id, {
|
||||
parallelToolCalls: false
|
||||
});
|
||||
initialStream = await runInvoker.stream();
|
||||
} catch (err) {
|
||||
console.error('Failed to start run:', err);
|
||||
return fallbackGrade(submission, task);
|
||||
}
|
||||
|
||||
let initialStream: AsyncIterable<RunStreamEvent>;
|
||||
try {
|
||||
const runInvoker = client.agents.createRun(thread.id, agent.id, {
|
||||
parallelToolCalls: false
|
||||
});
|
||||
initialStream = await runInvoker.stream();
|
||||
} catch (err) {
|
||||
console.error('Failed to start run:', err);
|
||||
return fallbackGrade(submission, task);
|
||||
}
|
||||
// process all tool calls
|
||||
let finalRun: ThreadRunOutput;
|
||||
try {
|
||||
finalRun = await processRunStream(thread.id, initialStream, roomId);
|
||||
} catch (err) {
|
||||
console.error('Agent streaming failed:', err);
|
||||
return fallbackGrade(submission, task);
|
||||
}
|
||||
|
||||
// process all tool calls
|
||||
let finalRun: ThreadRunOutput;
|
||||
try {
|
||||
finalRun = await processRunStream(thread.id, initialStream, roomId);
|
||||
} catch (err) {
|
||||
console.error('Agent streaming failed:', err);
|
||||
return fallbackGrade(submission, task);
|
||||
}
|
||||
const msgs = await client.agents.listMessages(thread.id);
|
||||
const assistant = msgs.data.find((m) => m.role === 'assistant');
|
||||
const raw = assistant ? extractTextFromMessage(assistant) : '';
|
||||
|
||||
const msgs = await client.agents.listMessages(thread.id);
|
||||
const assistant = msgs.data.find(m => m.role === 'assistant');
|
||||
const raw = assistant ? extractTextFromMessage(assistant) : '';
|
||||
// strip anything before the JSON object
|
||||
const match = raw.match(/\{[\s\S]*\}$/);
|
||||
const jsonText = match ? match[0] : raw;
|
||||
|
||||
// strip anything before the JSON object
|
||||
const match = raw.match(/\{[\s\S]*\}$/);
|
||||
const jsonText = match ? match[0] : raw;
|
||||
|
||||
try {
|
||||
return JSON.parse(jsonText) as OpenAIResponse;
|
||||
} catch (parseErr) {
|
||||
console.error('JSON parse failed, returning fallback:', parseErr);
|
||||
return { ...fallbackGrade(submission, task), reasoning: raw };
|
||||
}
|
||||
}
|
||||
try {
|
||||
return JSON.parse(jsonText) as OpenAIResponse;
|
||||
} catch (parseErr) {
|
||||
console.error('JSON parse failed, returning fallback:', parseErr);
|
||||
return { ...fallbackGrade(submission, task), reasoning: raw };
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user