jcreek/AutomatedAssessmentFeedbackAgent
1
0
Fork 0
mirror of https://github.com/jcreek/AutomatedAssessmentFeedbackAgent.git synced 2026-07-13 02:53:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(*): Enable HITL functionality

Browse Source
This commit is contained in:
Josh Creek
2025-04-25 23:08:02 +01:00
parent 00ef1476d9
commit 8394a24ac1
5 changed files with 326 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/agent/services/agentService.ts
+75 -76
View File
@@ -1,42 +1,42 @@
import { DefaultAzureCredential } from '@azure/identity';
import { AIProjectsClient, ToolUtility } from '@azure/ai-projects';
import {
rubricMatcherTool,
essayAnalyzerTool,
conceptVerifierTool,
feedbackGeneratorTool,
metacognitiveReflectionPromptTool,
selfAssessmentTool,
spellingAndGrammarCheckerTool
rubricMatcherTool,
essayAnalyzerTool,
conceptVerifierTool,
feedbackGeneratorTool,
metacognitiveReflectionPromptTool,
selfAssessmentTool,
spellingAndGrammarCheckerTool
} from '../tools/index';
import { AI_FOUNDRY_PROJECT_CONNECTION_STRING, AI_MODEL } from '$env/static/private';
if (!AI_FOUNDRY_PROJECT_CONNECTION_STRING) {
throw new Error('AI_FOUNDRY_PROJECT_CONNECTION_STRING is not set');
throw new Error('AI_FOUNDRY_PROJECT_CONNECTION_STRING is not set');
}
if (!AI_MODEL) {
throw new Error('AI_MODEL is not set');
throw new Error('AI_MODEL is not set');
}
export const client = AIProjectsClient.fromConnectionString(
AI_FOUNDRY_PROJECT_CONNECTION_STRING,
new DefaultAzureCredential()
AI_FOUNDRY_PROJECT_CONNECTION_STRING,
new DefaultAzureCredential()
);
// const codeInterpreterTool = ToolUtility.createCodeInterpreterTool([]);
const allTools = [
// codeInterpreterTool,
rubricMatcherTool,
essayAnalyzerTool,
conceptVerifierTool,
feedbackGeneratorTool,
metacognitiveReflectionPromptTool,
selfAssessmentTool,
spellingAndGrammarCheckerTool
// codeInterpreterTool,
rubricMatcherTool,
essayAnalyzerTool,
conceptVerifierTool,
feedbackGeneratorTool,
metacognitiveReflectionPromptTool,
selfAssessmentTool,
spellingAndGrammarCheckerTool
];
const tools = allTools.map(t => t.definition);
const tools = allTools.map((t) => t.definition);
const instructions = `
You are a helpful agent that can assist with providing feedback on a student's work for a teacher.
@@ -44,50 +44,49 @@ You are familiar with modern pedagogy around summative and formative assessment
Whenever you need to:
• check the submission against the rubric → call "${rubricMatcherTool.definition.name}"
• verify which rubric concepts appear or are missing → call "${conceptVerifierTool.definition.name}"
• analyze essay structure → call "${essayAnalyzerTool.definition.name}"
• analyze essay structure → call "${essayAnalyzerTool.definition.name}"
• generate targeted feedback → call "${feedbackGeneratorTool.definition.name}"
• craft a metacognitive prompt → call "${metacognitiveReflectionPromptTool.definition.name}"
• guide self-assessment → call "${selfAssessmentTool.definition.name}"
• check spelling & grammar → call "${spellingAndGrammarCheckerTool.definition.name}"
If a user message begins with [HUMAN REVIEW]:, treat it as authoritative teacher feedback and use it as the basis for grading. Do not escalate to human review again.
`;
export const toolResources = allTools.reduce<Record<string, any>>((map, t) => {
// Try all plausible locations for the tool name
const name =
t.definition.name ||
t.definition.function?.name ||
Object.keys(t.resources ?? {})[0];
// Try all plausible locations for the tool name
const name =
t.definition.name || t.definition.function?.name || Object.keys(t.resources ?? {})[0];
if (!name) {
console.warn('Tool missing name:', t);
return map;
}
map[name] = t;
return map;
if (!name) {
console.warn('Tool missing name:', t);
return map;
}
map[name] = t;
return map;
}, {});
export async function getOrCreateAgent() {
const agentName = `assessment-feedback-agent`;
// List all agents (may need pagination for large numbers)
const agentsResponse = await client.agents.listAgents();
const agents = Array.isArray(agentsResponse.data) ? agentsResponse.data : [];
const existing = agents.find((a: any) => a.name === agentName);
if (existing) {
return existing;
}
// Otherwise, create new agent
const agent = await client.agents.createAgent(AI_MODEL, {
name: agentName,
instructions,
temperature: 0.5,
tools,
toolResources,
requestOptions: {
headers: { 'x-ms-enable-preview': 'true' }
}
});
return agent;
const agentName = `assessment-feedback-agent`;
// List all agents (may need pagination for large numbers)
const agentsResponse = await client.agents.listAgents();
const agents = Array.isArray(agentsResponse.data) ? agentsResponse.data : [];
const existing = agents.find((a: any) => a.name === agentName);
if (existing) {
return existing;
}
// Otherwise, create new agent
const agent = await client.agents.createAgent(AI_MODEL, {
name: agentName,
instructions,
temperature: 0.5,
tools,
toolResources,
requestOptions: {
headers: { 'x-ms-enable-preview': 'true' }
}
});
return agent;
}
export const agent = await getOrCreateAgent();
@@ -97,29 +96,29 @@ export const agent = await getOrCreateAgent();
* Use with caution! This will remove ALL agents.
*/
export async function deleteAllAgents() {
try {
let allAgents: any[] = [];
let after: string | undefined = undefined;
let hasMore = true;
try {
let allAgents: any[] = [];
let after: string | undefined = undefined;
let hasMore = true;
while (hasMore) {
const response = await client.agents.listAgents(after ? { after } : {});
const agents = Array.isArray(response.data) ? response.data : [];
allAgents = allAgents.concat(agents);
hasMore = response.hasMore;
after = response.lastId;
}
while (hasMore) {
const response = await client.agents.listAgents(after ? { after } : {});
const agents = Array.isArray(response.data) ? response.data : [];
allAgents = allAgents.concat(agents);
hasMore = response.hasMore;
after = response.lastId;
}
for (const agent of allAgents) {
await client.agents.deleteAgent(agent.id, {
requestOptions: {
headers: { 'x-ms-enable-preview': 'true' }
}
});
console.info(`Agent ${agent.id} (${agent.name}) deleted.`);
}
console.info('All agents deleted successfully.');
} catch (err) {
console.warn('Failed to delete all agents:', err);
}
for (const agent of allAgents) {
await client.agents.deleteAgent(agent.id, {
requestOptions: {
headers: { 'x-ms-enable-preview': 'true' }
}
});
console.info(`Agent ${agent.id} (${agent.name}) deleted.`);
}
console.info('All agents deleted successfully.');
} catch (err) {
console.warn('Failed to delete all agents:', err);
}
}
src/lib/server/ai.ts
+66 -5
View File
@@ -40,7 +40,10 @@ RESPONSE FORMAT (respond with a single JSON object, no extra text):
Respond ONLY with the JSON object, with no preamble or explanation.`;
}
export function buildGradingPromptWithSupportForHumanInTheLoop(submission: string, task: string): string {
export function buildGradingPromptWithSupportForHumanInTheLoop(
submission: string,
task: string
): string {
return `You are an expert secondary school teacher and AI assessment agent. Assess the following student submission in the context of the assignment/task provided.
TASK/ASSIGNMENT:
@@ -128,7 +131,6 @@ EXAMPLES:
- Reasoning: The task and submission are both clear and evaluatable without a rubric.
Respond ONLY with the JSON object, with no preamble or explanation.`;
}
function fallbackGrade(submission: string, task: string): OpenAIResponse {
@@ -223,7 +225,8 @@ async function processRunStream(
export async function gradeSubmissionWithAgent(
submission: string,
task: string,
roomId: string
roomId: string,
hitl: boolean = false
): Promise<OpenAIResponse> {
if (!client || !agent) {
throw new Error('Agent not available');
@@ -232,7 +235,9 @@ export async function gradeSubmissionWithAgent(
const thread = await client.agents.createThread();
await client.agents.createMessage(thread.id, {
role: 'user',
content: buildGradingPrompt(submission, task)
content: hitl
? buildGradingPromptWithSupportForHumanInTheLoop(submission, task)
: buildGradingPrompt(submission, task)
});
let initialStream: AsyncIterable<RunStreamEvent>;
@@ -264,9 +269,65 @@ export async function gradeSubmissionWithAgent(
const jsonText = match ? match[0] : raw;
try {
return JSON.parse(jsonText) as OpenAIResponse;
const parsed = JSON.parse(jsonText) as OpenAIResponse;
if (parsed.grade === 'HUMAN_REVIEW_REQUIRED') {
return {
...parsed,
success: true,
threadId: thread.id,
runId: finalRun?.id ?? undefined,
hitlContext: { threadId: thread.id, runId: finalRun?.id ?? undefined }
};
}
return parsed;
} catch (parseErr) {
console.error('JSON parse failed, returning fallback:', parseErr);
return { ...fallbackGrade(submission, task), reasoning: raw };
}
}
export async function resumeAgentWithHumanReview(
humanReview: string,
context: { threadId: string; runId?: string; roomId?: string; [key: string]: any }
): Promise<OpenAIResponse> {
const { threadId, roomId = '' } = context;
await client.agents.createMessage(threadId, {
role: 'user',
content: `[HUMAN REVIEW]: ${humanReview}`
});
let stream: AsyncIterable<RunStreamEvent>;
try {
const runInvoker = client.agents.createRun(threadId, agent.id, {
parallelToolCalls: false
});
stream = await runInvoker.stream();
} catch (err) {
console.error('Failed to start run:', err);
return fallbackGrade('', '');
}
let finalRun: ThreadRunOutput;
try {
finalRun = await processRunStream(threadId, stream, roomId);
} catch (err) {
console.error('Agent streaming failed:', err);
return fallbackGrade('', '');
}
const msgs = await client.agents.listMessages(threadId);
const assistant = msgs.data.find((m) => m.role === 'assistant');
const raw = assistant ? extractTextFromMessage(assistant) : '';
// strip anything before the JSON object
const match = raw.match(/\{[\s\S]*\}$/);
const jsonText = match ? match[0] : raw;
try {
const parsed = JSON.parse(jsonText) as OpenAIResponse;
return parsed;
} catch (parseErr) {
console.error('JSON parse failed, returning fallback:', parseErr);
return { ...fallbackGrade('', ''), reasoning: raw };
}
}