mirror of
https://github.com/jcreek/AutomatedAssessmentFeedbackAgent.git
synced 2026-07-12 18:43:49 +00:00
refactor(*): Use a stream for handling the agent communication
This commit is contained in:
Josh Creek
@@ -1,69 +1,72 @@
|
||||
import { DefaultAzureCredential } from '@azure/identity';
|
||||
import { AIProjectsClient } from '@azure/ai-projects';
|
||||
import { ToolUtility } from '@azure/ai-projects';
|
||||
import { AIProjectsClient, ToolUtility } from '@azure/ai-projects';
|
||||
import {
|
||||
rubricMatcherTool,
|
||||
essayAnalyzerTool,
|
||||
conceptVerifierTool,
|
||||
feedbackGeneratorTool,
|
||||
metacognitiveReflectionPromptTool,
|
||||
selfAssessmentTool,
|
||||
spellingAndGrammarCheckerTool
|
||||
} from '../tools/index';
|
||||
import { AI_FOUNDRY_PROJECT_CONNECTION_STRING, AI_MODEL } from '$env/static/private';
|
||||
|
||||
import {
|
||||
rubricMatcherTool,
|
||||
essayAnalyzerTool,
|
||||
conceptVerifierTool,
|
||||
feedbackGeneratorTool,
|
||||
metacognitiveReflectionPromptTool,
|
||||
selfAssessmentTool,
|
||||
spellingAndGrammarCheckerTool
|
||||
} from '../tools/index';
|
||||
|
||||
const connectionString = AI_FOUNDRY_PROJECT_CONNECTION_STRING;
|
||||
if (!connectionString) {
|
||||
throw new Error('AI_FOUNDRY_PROJECT_CONNECTION_STRING is not set in environment variables');
|
||||
if (!AI_FOUNDRY_PROJECT_CONNECTION_STRING) {
|
||||
throw new Error('AI_FOUNDRY_PROJECT_CONNECTION_STRING is not set');
|
||||
}
|
||||
if (!AI_MODEL) {
|
||||
throw new Error('AI_MODEL is not set');
|
||||
}
|
||||
|
||||
const aiModel = AI_MODEL || '';
|
||||
if (!connectionString) {
|
||||
throw new Error('AI_MODEL is not set in environment variables');
|
||||
}
|
||||
|
||||
const client = AIProjectsClient.fromConnectionString(
|
||||
connectionString,
|
||||
export const client = AIProjectsClient.fromConnectionString(
|
||||
AI_FOUNDRY_PROJECT_CONNECTION_STRING,
|
||||
new DefaultAzureCredential()
|
||||
);
|
||||
|
||||
const codeInterpreterTool = ToolUtility.createCodeInterpreterTool([]);
|
||||
|
||||
const tools = [
|
||||
codeInterpreterTool.definition,
|
||||
rubricMatcherTool.definition,
|
||||
essayAnalyzerTool.definition,
|
||||
conceptVerifierTool.definition,
|
||||
feedbackGeneratorTool.definition,
|
||||
metacognitiveReflectionPromptTool.definition,
|
||||
selfAssessmentTool.definition,
|
||||
spellingAndGrammarCheckerTool.definition
|
||||
const allTools = [
|
||||
codeInterpreterTool,
|
||||
rubricMatcherTool,
|
||||
essayAnalyzerTool,
|
||||
conceptVerifierTool,
|
||||
feedbackGeneratorTool,
|
||||
metacognitiveReflectionPromptTool,
|
||||
selfAssessmentTool,
|
||||
spellingAndGrammarCheckerTool
|
||||
];
|
||||
|
||||
const instructions = "You are a helpful agent that can assist with providing feedback on a student's work for a teacher. You are familiar with all modern pedagogy around summative and formative assessment, and K12 education, and primary and secondary education.";
|
||||
const tools = allTools.map(t => t.definition);
|
||||
|
||||
const toolResources = {
|
||||
...codeInterpreterTool.resources,
|
||||
matchRubric: {},
|
||||
analyzeEssay: {},
|
||||
conceptVerifier: {},
|
||||
feedbackGenerator: {},
|
||||
metacognitiveReflectionPrompt: {},
|
||||
selfAssessment: {},
|
||||
spellingAndGrammarChecker: {}
|
||||
};
|
||||
const instructions = `
|
||||
You are a helpful agent that can assist with providing feedback on a student's work for a teacher.
|
||||
You are familiar with modern pedagogy around summative and formative assessment in K-12 education.
|
||||
Whenever you need to:
|
||||
• check the submission against the rubric → call "${rubricMatcherTool.definition.name}"
|
||||
• verify which rubric concepts appear or are missing → call "${conceptVerifierTool.definition.name}"
|
||||
• analyze essay structure → call "${essayAnalyzerTool.definition.name}"
|
||||
• generate targeted feedback → call "${feedbackGeneratorTool.definition.name}"
|
||||
• craft a metacognitive prompt → call "${metacognitiveReflectionPromptTool.definition.name}"
|
||||
• guide self-assessment → call "${selfAssessmentTool.definition.name}"
|
||||
• check spelling & grammar → call "${spellingAndGrammarCheckerTool.definition.name}"
|
||||
`;
|
||||
|
||||
const agent = await client.agents.createAgent(aiModel, {
|
||||
const toolResources = allTools.reduce<Record<string, any>>((map, t) => {
|
||||
if (typeof t === 'function') {
|
||||
map[t.definition.name] = t;
|
||||
} else {
|
||||
map[t.definition.name] = t.resources;
|
||||
}
|
||||
return map;
|
||||
}, {});
|
||||
|
||||
export const agent = await client.agents.createAgent(AI_MODEL, {
|
||||
name: `assessment-feedback-agent`,
|
||||
instructions,
|
||||
temperature: 0.5,
|
||||
tools,
|
||||
toolResources,
|
||||
|
||||
requestOptions: {
|
||||
headers: { "x-ms-enable-preview": "true" },
|
||||
},
|
||||
});
|
||||
|
||||
export { client, agent };
|
||||
headers: { 'x-ms-enable-preview': 'true' }
|
||||
}
|
||||
});
|
||||
@@ -8,7 +8,7 @@ export const spellingAndGrammarCheckerToolMeta = {
|
||||
|
||||
export const spellingAndGrammarCheckerTool = ToolUtility.createFunctionTool({
|
||||
name: spellingAndGrammarCheckerToolMeta.key,
|
||||
description: 'Identifies spelling, punctuation, and syntax issues in the submission and suggests corrections.',
|
||||
description: 'Identifies spelling, punctuation, and syntax issues in the submission and suggests corrections. This should result in a clear set of pairs, e.g. "shud" - "should"',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
|
||||
+136
-141
@@ -1,10 +1,13 @@
|
||||
import { client, agent } from '../agent/services/agentService';
|
||||
import type { MessageTextContentOutput, ToolOutput } from '@azure/ai-projects';
|
||||
import { client, agent, toolResources } from '../agent/services/agentService';
|
||||
import {
|
||||
RunStreamEvent,
|
||||
ErrorEvent,
|
||||
type ThreadRunOutput
|
||||
} from '@azure/ai-projects';
|
||||
import { PARTYKIT_BASE_URL } from '$env/static/private';
|
||||
import type { OpenAIResponse } from './types';
|
||||
|
||||
// Prompt template for grading and feedback
|
||||
export function buildGradingPrompt(submission: string, task: string) {
|
||||
export function buildGradingPrompt(submission: string, task: string): string {
|
||||
return `You are an expert secondary school teacher and AI assessment agent. Assess the following student submission in the context of the assignment/task provided.
|
||||
|
||||
TASK/ASSIGNMENT:
|
||||
@@ -14,7 +17,7 @@ STUDENT SUBMISSION:
|
||||
${submission}
|
||||
|
||||
Follow these steps:
|
||||
1. Grade the work out of 10, using clear, objective criteria.
|
||||
1. Grade the work with a letter (A+ is best, E- is worst), using clear, objective criteria.
|
||||
2. Identify specific strengths, referencing the success criteria.
|
||||
3. Identify misconceptions or areas for improvement, using formative assessment language.
|
||||
4. Design an individualized activity or exercise for the student to address their misconceptions or extend their learning. This activity should be:
|
||||
@@ -28,7 +31,7 @@ Follow these steps:
|
||||
RESPONSE FORMAT (respond with a single JSON object, no extra text):
|
||||
|
||||
{
|
||||
"grade": "<number or string, e.g. A, B, C+, etc>",
|
||||
"grade": "<number or string, e.g. 8/10, A, B+>",
|
||||
"strengths": "<text>",
|
||||
"areasForImprovement": "<text>",
|
||||
"individualizedActivity": "<text>",
|
||||
@@ -38,160 +41,152 @@ RESPONSE FORMAT (respond with a single JSON object, no extra text):
|
||||
"reasoning": "<step-by-step explanation>"
|
||||
}
|
||||
|
||||
Respond ONLY with the JSON object, with no preamble or explanation.
|
||||
`;
|
||||
Respond ONLY with the JSON object, with no preamble or explanation.`;
|
||||
}
|
||||
|
||||
function fallbackGrade(submission: string, task: string): OpenAIResponse {
|
||||
console.log('using fallback');
|
||||
console.warn('Using fallback grade logic');
|
||||
return {
|
||||
grade: '7/10',
|
||||
strengths: 'Clear argument and good evidence (in response to the task: ' + task + ").",
|
||||
areasForImprovement: 'Needs deeper analysis and more examples.',
|
||||
individualizedActivity: 'Write a paragraph expanding on your main point and provide two additional examples to support your argument.',
|
||||
reflectionQuestion: 'What was the most challenging part of this assignment for you, and why?',
|
||||
teacherSuggestion: 'In the next lesson, review how to develop arguments with supporting evidence and provide a model answer for comparison.',
|
||||
spellingAndGrammar: 'Everything was spelled correctly',
|
||||
reasoning: 'The submission demonstrates a good structure and evidence, but lacks depth and breadth of analysis. The activity and suggestions are designed to target this gap.'
|
||||
grade: 'EXAMPLE',
|
||||
strengths: `Clear argument and good evidence (task: ${task}).`,
|
||||
areasForImprovement: 'Needs deeper analysis.',
|
||||
individualizedActivity: 'Add two more supporting examples.',
|
||||
reflectionQuestion: 'Which part was hardest?',
|
||||
teacherSuggestion: 'Model a paragraph with evidence.',
|
||||
spellingAndGrammar: 'No errors detected.',
|
||||
reasoning: 'Standard fallback reasoning.'
|
||||
};
|
||||
}
|
||||
|
||||
export async function gradeSubmissionWithAgent(submission: string, task: string, roomId?: string): Promise<OpenAIResponse> {
|
||||
try {
|
||||
if (!client || !agent) throw new Error('Agent not available');
|
||||
function extractTextFromMessage(msg: { content: any[] }): string {
|
||||
return msg.content
|
||||
.filter(c => c.type === 'text')
|
||||
.map(c => (typeof c.text === 'string' ? c.text : c.text.value ?? ''))
|
||||
.join('\n')
|
||||
.trim();
|
||||
}
|
||||
|
||||
// 1. Create a new thread
|
||||
const thread = await client.agents.createThread();
|
||||
// Recursively process each run stream, invoke the tools & notify PartyKit
|
||||
async function processRunStream(
|
||||
threadId: string,
|
||||
stream: AsyncIterable<RunStreamEvent>,
|
||||
roomId: string
|
||||
): Promise<ThreadRunOutput> {
|
||||
for await (const evt of stream) {
|
||||
// if (!evt.event.includes('delta')) {
|
||||
// console.log('▶️ Stream event:', evt.event);
|
||||
// }
|
||||
|
||||
// 2. Add a message to the thread
|
||||
await client.agents.createMessage(thread.id, {
|
||||
role: 'user',
|
||||
content: buildGradingPrompt(submission, task)
|
||||
});
|
||||
if (evt.event === RunStreamEvent.ThreadRunRequiresAction) {
|
||||
const runOutput = evt.data as ThreadRunOutput;
|
||||
const calls = runOutput.requiredAction!.submitToolOutputs!.toolCalls!;
|
||||
|
||||
// 3. Run the agent
|
||||
const run = await client.agents.createRun(thread.id, agent.id, {});
|
||||
// notify PartyKit
|
||||
await Promise.all(
|
||||
calls.map(async call => {
|
||||
if (!PARTYKIT_BASE_URL) return;
|
||||
const wsCtor = typeof WebSocket !== 'undefined'
|
||||
? WebSocket
|
||||
: (await import('ws')).default;
|
||||
const ws = new wsCtor(
|
||||
`${PARTYKIT_BASE_URL}/party/tool-usage-server-${roomId}`
|
||||
);
|
||||
await new Promise<void>((res, rej) => {
|
||||
ws.onopen = () => res();
|
||||
ws.onerror = e => rej(e);
|
||||
});
|
||||
ws.send(JSON.stringify({ tool: call.function.name, time: new Date().toISOString() }));
|
||||
ws.close();
|
||||
})
|
||||
);
|
||||
|
||||
// 4. Poll for run completion
|
||||
let runResult;
|
||||
for (let i = 0; i < 60; i++) { // up to ~60 seconds
|
||||
runResult = await client.agents.getRun(thread.id, run.id);
|
||||
console.log('Agent run status:', runResult.status);
|
||||
|
||||
if (runResult.status === 'completed') break;
|
||||
if (runResult.status === 'failed') {
|
||||
const messages = await client.agents.listMessages(thread.id);
|
||||
const agentMessage = messages.data.find((m: { role: string; content: any[] }) => m.role === 'agent');
|
||||
let errorText = '';
|
||||
if (agentMessage && Array.isArray(agentMessage.content) && agentMessage.content.length > 0) {
|
||||
const contentItem = agentMessage.content[0];
|
||||
if (contentItem.type === 'text' && contentItem.text && typeof contentItem.text.value === 'string') {
|
||||
errorText = contentItem.text.value;
|
||||
} else if (typeof contentItem.value === 'string') {
|
||||
errorText = contentItem.value;
|
||||
}
|
||||
}
|
||||
throw new Error(`Agent run failed. Status: failed. Message: ${errorText}`);
|
||||
}
|
||||
|
||||
// Handle tool calls if agent requires action
|
||||
if (runResult.status === 'requires_action' && runResult.requiredAction?.submitToolOutputs.toolCalls) {
|
||||
const toolCalls = runResult.requiredAction.submitToolOutputs.toolCalls;
|
||||
const toolOutputs: ToolOutput[] = await Promise.all(toolCalls.map(async (call: any) => {
|
||||
let output = '';
|
||||
const args = JSON.parse(call.function.arguments);
|
||||
|
||||
if (roomId) {
|
||||
try {
|
||||
const ws = new (typeof WebSocket !== 'undefined' ? WebSocket : (await import('ws')).default)(`${PARTYKIT_BASE_URL}/party/tool-usage-server-${roomId}`);
|
||||
|
||||
// Wait for connection to open
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
ws.onopen = () => resolve();
|
||||
ws.onerror = (err) => reject(err);
|
||||
});
|
||||
|
||||
ws.send(JSON.stringify({ tool: call.function.name, time: new Date().toISOString() }));
|
||||
|
||||
ws.close();
|
||||
} catch (err) {
|
||||
console.error('Failed to send tool usage event to PartyKit:', err);
|
||||
// invoke the tools
|
||||
const toolOutputs = await Promise.all(
|
||||
calls.map(async call => {
|
||||
const name = call.function.name;
|
||||
const args = call.arguments!;
|
||||
try {
|
||||
const fn = toolResources[name];
|
||||
if (typeof fn !== 'function') {
|
||||
throw new Error(`No tool implementation for "${name}"`);
|
||||
}
|
||||
const result = await fn(args);
|
||||
return { toolCallId: call.id, output: result };
|
||||
} catch (err: any) {
|
||||
return { toolCallId: call.id, output: `Tool error: ${err.message}` };
|
||||
}
|
||||
})
|
||||
);
|
||||
|
||||
if (
|
||||
call.function.name === 'matchRubric' ||
|
||||
call.function.name === 'analyzeEssay'
|
||||
) {
|
||||
// LLM will handle the tool logic, just submit empty output
|
||||
output = '';
|
||||
} else {
|
||||
output = `Unknown tool: ${call.function.name}`;
|
||||
}
|
||||
|
||||
return {
|
||||
toolCallId: call.id,
|
||||
output: typeof output === 'string' ? output : JSON.stringify(output)
|
||||
};
|
||||
}));
|
||||
await client.agents.submitToolOutputsToRun(thread.id, run.id, toolOutputs);
|
||||
}
|
||||
|
||||
await new Promise(res => setTimeout(res, 1000));
|
||||
}
|
||||
if (!runResult || runResult.status !== 'completed') {
|
||||
const messages = await client.agents.listMessages(thread.id);
|
||||
const agentMessage = messages.data.find((m: { role: string; content: any[] }) => m.role === 'agent');
|
||||
let errorText = '';
|
||||
if (agentMessage && Array.isArray(agentMessage.content) && agentMessage.content.length > 0) {
|
||||
const contentItem = agentMessage.content[0];
|
||||
if (contentItem.type === 'text' && contentItem.text && typeof contentItem.text.value === 'string') {
|
||||
errorText = contentItem.text.value;
|
||||
} else if (typeof contentItem.value === 'string') {
|
||||
errorText = contentItem.value;
|
||||
}
|
||||
}
|
||||
throw new Error(`Agent run did not complete in time. Last status: ${runResult?.status}. Last message: ${errorText}`);
|
||||
// resume the run
|
||||
const resumed = client.agents.submitToolOutputsToRun(
|
||||
threadId,
|
||||
runOutput.id,
|
||||
toolOutputs
|
||||
);
|
||||
const nextStream = await resumed.stream();
|
||||
return processRunStream(threadId, nextStream, roomId);
|
||||
}
|
||||
|
||||
// 5. Get agent's feedback message
|
||||
const messages = await client.agents.listMessages(thread.id);
|
||||
const agentMessage = messages.data.find((m: { role: string; content: any[] }) => m.role === 'assistant');
|
||||
// Safely extract text content from agent message
|
||||
let text = '';
|
||||
if (agentMessage && Array.isArray(agentMessage.content) && agentMessage.content.length > 0) {
|
||||
const contentItem = agentMessage.content[0];
|
||||
if (contentItem.type === 'text' && 'text' in contentItem) {
|
||||
text = (contentItem as MessageTextContentOutput).text.value;
|
||||
} else if ('value' in contentItem && typeof (contentItem as any).value === 'string') {
|
||||
text = (contentItem as any).value;
|
||||
}
|
||||
if (evt.event === RunStreamEvent.ThreadRunCompleted) {
|
||||
return evt.data as ThreadRunOutput;
|
||||
}
|
||||
|
||||
// 6. Parse agent's response into OpenAIResponse format
|
||||
let parsed: OpenAIResponse | null = null;
|
||||
try {
|
||||
parsed = JSON.parse(text);
|
||||
} catch (err) {
|
||||
console.error(err);
|
||||
if (evt.event === ErrorEvent.Error) {
|
||||
throw new Error(`Agent error: ${JSON.stringify(evt.data)}`);
|
||||
}
|
||||
if (parsed && typeof parsed === 'object' && parsed.grade) {
|
||||
return parsed;
|
||||
}
|
||||
// If parsing fails, return raw text in reasoning
|
||||
return {
|
||||
grade: '',
|
||||
strengths: '',
|
||||
areasForImprovement: '',
|
||||
individualizedActivity: '',
|
||||
reflectionQuestion: '',
|
||||
teacherSuggestion: '',
|
||||
spellingAndGrammar: '',
|
||||
reasoning: text || 'No feedback generated.'
|
||||
};
|
||||
}
|
||||
|
||||
throw new Error('Stream ended without completion');
|
||||
}
|
||||
|
||||
export async function gradeSubmissionWithAgent(
|
||||
submission: string,
|
||||
task: string,
|
||||
roomId: string
|
||||
): Promise<OpenAIResponse> {
|
||||
if (!client || !agent) {
|
||||
throw new Error('Agent not available');
|
||||
}
|
||||
|
||||
const thread = await client.agents.createThread();
|
||||
await client.agents.createMessage(thread.id, {
|
||||
role: 'user',
|
||||
content: buildGradingPrompt(submission, task)
|
||||
});
|
||||
|
||||
let initialStream: AsyncIterable<RunStreamEvent>;
|
||||
try {
|
||||
const runInvoker = client.agents.createRun(thread.id, agent.id, {
|
||||
parallelToolCalls: false
|
||||
});
|
||||
initialStream = await runInvoker.stream();
|
||||
} catch (err) {
|
||||
console.error(err);
|
||||
// Fallback to local/demo logic
|
||||
console.error('Failed to start run:', err);
|
||||
return fallbackGrade(submission, task);
|
||||
}
|
||||
}
|
||||
|
||||
// process all tool calls
|
||||
let finalRun: ThreadRunOutput;
|
||||
try {
|
||||
finalRun = await processRunStream(thread.id, initialStream, roomId);
|
||||
} catch (err) {
|
||||
console.error('Agent streaming failed:', err);
|
||||
return fallbackGrade(submission, task);
|
||||
}
|
||||
|
||||
const msgs = await client.agents.listMessages(thread.id);
|
||||
const assistant = msgs.data.find(m => m.role === 'assistant');
|
||||
const raw = assistant ? extractTextFromMessage(assistant) : '';
|
||||
|
||||
// strip anything before the JSON object
|
||||
const match = raw.match(/\{[\s\S]*\}$/);
|
||||
const jsonText = match ? match[0] : raw;
|
||||
|
||||
try {
|
||||
return JSON.parse(jsonText) as OpenAIResponse;
|
||||
} catch (parseErr) {
|
||||
console.error('JSON parse failed, returning fallback:', parseErr);
|
||||
return { ...fallbackGrade(submission, task), reasoning: raw };
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user