jcreek/AutomatedAssessmentFeedbackAgent
1
0
Fork 0
mirror of https://github.com/jcreek/AutomatedAssessmentFeedbackAgent.git synced 2026-07-12 18:43:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(*): Change CI to use secrets instead of fallbacks

Browse Source
This commit is contained in:
Josh Creek
2025-04-24 21:08:13 +01:00
parent f421a0a110
commit 426e52f765
4 changed files with 135 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/ci.yml
+4 -4
View File
@@ -9,6 +9,7 @@ on:
jobs:
test-and-deploy:
runs-on: ubuntu-latest
environment: test
steps:
- uses: actions/checkout@v4
- name: Use Node.js
@@ -24,8 +25,7 @@ jobs:
- name: Run Playwright/BDD tests
run: npm run bdd:full
env:
NODE_ENV: test
AI_FOUNDRY_PROJECT_CONNECTION_STRING: test
AI_FOUNDRY_PROJECT_CONNECTION_STRING: ${{ secrets.AI_FOUNDRY_PROJECT_CONNECTION_STRING }}
AI_MODEL: gpt-4o
VITE_PARTYKIT_BASE_URL: wss://partykit.jcreek.partykit.dev
PARTYKIT_BASE_URL: wss://partykit.jcreek.partykit.dev
VITE_PARTYKIT_BASE_URL: ${{ secrets.VITE_PARTYKIT_BASE_URL }}
PARTYKIT_BASE_URL: ${{ secrets.PARTYKIT_BASE_URL }}
src/lib/agent/services/agentService.ts
+16 -22
View File
@@ -11,21 +11,17 @@ import {
} from '../tools/index';
import { AI_FOUNDRY_PROJECT_CONNECTION_STRING, AI_MODEL } from '$env/static/private';
const isTestEnv = process.env.NODE_ENV === 'test' || AI_FOUNDRY_PROJECT_CONNECTION_STRING === 'test';
const safeConnString = AI_FOUNDRY_PROJECT_CONNECTION_STRING || (isTestEnv ? 'dummy-connection-string' : undefined);
const safeModel = AI_MODEL || (isTestEnv ? 'dummy-model' : undefined);
if (!safeConnString && !isTestEnv) {
if (!AI_FOUNDRY_PROJECT_CONNECTION_STRING) {
throw new Error('AI_FOUNDRY_PROJECT_CONNECTION_STRING is not set');
}
if (!safeModel && !isTestEnv) {
if (!AI_MODEL) {
throw new Error('AI_MODEL is not set');
}
export const client = !isTestEnv && safeConnString
? AIProjectsClient.fromConnectionString(safeConnString, new DefaultAzureCredential())
: undefined;
export const client = AIProjectsClient.fromConnectionString(
AI_FOUNDRY_PROJECT_CONNECTION_STRING,
new DefaultAzureCredential()
);
// const codeInterpreterTool = ToolUtility.createCodeInterpreterTool([]);
@@ -70,15 +66,13 @@ export const toolResources = allTools.reduce<Record<string, any>>((map, t) => {
return map;
}, {});
export const agent = client
? await client.agents.createAgent(AI_MODEL, {
name: `assessment-feedback-agent`,
instructions,
temperature: 0.5,
tools,
toolResources,
requestOptions: {
headers: { 'x-ms-enable-preview': 'true' }
}
})
: undefined;
export const agent = await client.agents.createAgent(AI_MODEL, {
name: `assessment-feedback-agent`,
instructions,
temperature: 0.5,
tools,
toolResources,
requestOptions: {
headers: { 'x-ms-enable-preview': 'true' }
}
});
src/lib/utils/ai.ts
+115 -131
View File
@@ -1,14 +1,10 @@
import { client, agent, toolResources } from '../agent/services/agentService';
import {
RunStreamEvent,
ErrorEvent,
type ThreadRunOutput
} from '@azure/ai-projects';
import { RunStreamEvent, ErrorEvent, type ThreadRunOutput } from '@azure/ai-projects';
import { PARTYKIT_BASE_URL } from '$env/static/private';
import type { OpenAIResponse } from './types';
export function buildGradingPrompt(submission: string, task: string): string {
return `You are an expert secondary school teacher and AI assessment agent. Assess the following student submission in the context of the assignment/task provided.
return `You are an expert secondary school teacher and AI assessment agent. Assess the following student submission in the context of the assignment/task provided.
TASK/ASSIGNMENT:
${task}
@@ -45,153 +41,141 @@ Respond ONLY with the JSON object, with no preamble or explanation.`;
}
function fallbackGrade(submission: string, task: string): OpenAIResponse {
console.warn('Using fallback grade logic');
return {
grade: 'EXAMPLE',
strengths: `Clear argument and good evidence (task: ${task}).`,
areasForImprovement: 'Needs deeper analysis.',
individualizedActivity: 'Add two more supporting examples.',
reflectionQuestion: 'Which part was hardest?',
teacherSuggestion: 'Model a paragraph with evidence.',
spellingAndGrammar: 'No errors detected.',
reasoning: 'Standard fallback reasoning.'
};
console.warn('Using fallback grade logic');
return {
grade: 'EXAMPLE',
strengths: `Clear argument and good evidence (task: ${task}).`,
areasForImprovement: 'Needs deeper analysis.',
individualizedActivity: 'Add two more supporting examples.',
reflectionQuestion: 'Which part was hardest?',
teacherSuggestion: 'Model a paragraph with evidence.',
spellingAndGrammar: 'No errors detected.',
reasoning: 'Standard fallback reasoning.'
};
}
function extractTextFromMessage(msg: { content: any[] }): string {
return msg.content
.filter(c => c.type === 'text')
.map(c => (typeof c.text === 'string' ? c.text : c.text.value ?? ''))
.join('\n')
.trim();
return msg.content
.filter((c) => c.type === 'text')
.map((c) => (typeof c.text === 'string' ? c.text : (c.text.value ?? '')))
.join('\n')
.trim();
}
// Recursively process each run stream, invoke the tools & notify PartyKit
async function processRunStream(
threadId: string,
stream: AsyncIterable<RunStreamEvent>,
roomId: string
threadId: string,
stream: AsyncIterable<RunStreamEvent>,
roomId: string
): Promise<ThreadRunOutput> {
for await (const evt of stream) {
// if (!evt.event.includes('delta')) {
// console.log('▶️ Stream event:', evt.event);
// }
for await (const evt of stream) {
// if (!evt.event.includes('delta')) {
// console.log('▶️ Stream event:', evt.event);
// }
if (evt.event === RunStreamEvent.ThreadRunRequiresAction) {
const runOutput = evt.data as ThreadRunOutput;
const calls = runOutput.requiredAction!.submitToolOutputs!.toolCalls!;
if (evt.event === RunStreamEvent.ThreadRunRequiresAction) {
const runOutput = evt.data as ThreadRunOutput;
const calls = runOutput.requiredAction!.submitToolOutputs!.toolCalls!;
// notify PartyKit
await Promise.all(
calls.map(async call => {
if (!PARTYKIT_BASE_URL) return;
const wsCtor = typeof WebSocket !== 'undefined'
? WebSocket
: (await import('ws')).default;
const ws = new wsCtor(
`${PARTYKIT_BASE_URL}/party/tool-usage-server-${roomId}`
);
await new Promise<void>((res, rej) => {
ws.onopen = () => res();
ws.onerror = e => rej(e);
});
ws.send(JSON.stringify({ tool: call.function.name, time: new Date().toISOString() }));
ws.close();
})
);
// notify PartyKit
await Promise.all(
calls.map(async (call) => {
if (!PARTYKIT_BASE_URL) return;
const wsCtor =
typeof WebSocket !== 'undefined' ? WebSocket : (await import('ws')).default;
const ws = new wsCtor(`${PARTYKIT_BASE_URL}/party/tool-usage-server-${roomId}`);
await new Promise<void>((res, rej) => {
ws.onopen = () => res();
ws.onerror = (e) => rej(e);
});
ws.send(JSON.stringify({ tool: call.function.name, time: new Date().toISOString() }));
ws.close();
})
);
// invoke the tools
const toolOutputs = await Promise.all(
calls.map(async call => {
const name = call.function.name;
const args = call.arguments!;
try {
const fn = toolResources[name];
if (typeof fn !== 'function') {
throw new Error(`No tool implementation for "${name}"`);
}
const result = await fn(args);
return { toolCallId: call.id, output: result };
} catch (err: any) {
return { toolCallId: call.id, output: `Tool error: ${err.message}` };
}
})
);
// invoke the tools
const toolOutputs = await Promise.all(
calls.map(async (call) => {
const name = call.function.name;
const args = call.arguments!;
try {
const fn = toolResources[name];
if (typeof fn !== 'function') {
throw new Error(`No tool implementation for "${name}"`);
}
const result = await fn(args);
return { toolCallId: call.id, output: result };
} catch (err: any) {
return { toolCallId: call.id, output: `Tool error: ${err.message}` };
}
})
);
// resume the run
const resumed = client.agents.submitToolOutputsToRun(
threadId,
runOutput.id,
toolOutputs
);
const nextStream = await resumed.stream();
return processRunStream(threadId, nextStream, roomId);
}
// resume the run
const resumed = client.agents.submitToolOutputsToRun(threadId, runOutput.id, toolOutputs);
const nextStream = await resumed.stream();
return processRunStream(threadId, nextStream, roomId);
}
if (evt.event === RunStreamEvent.ThreadRunCompleted) {
return evt.data as ThreadRunOutput;
}
if (evt.event === RunStreamEvent.ThreadRunCompleted) {
return evt.data as ThreadRunOutput;
}
if (evt.event === ErrorEvent.Error) {
throw new Error(`Agent error: ${JSON.stringify(evt.data)}`);
}
}
if (evt.event === ErrorEvent.Error) {
throw new Error(`Agent error: ${JSON.stringify(evt.data)}`);
}
}
throw new Error('Stream ended without completion');
throw new Error('Stream ended without completion');
}
export async function gradeSubmissionWithAgent(
submission: string,
task: string,
roomId: string
submission: string,
task: string,
roomId: string
): Promise<OpenAIResponse> {
if (!client) {
// In test/CI mode, return fallback data
return fallbackGrade(submission, task);
}
if (!client || !agent) {
throw new Error('Agent not available');
}
if (!agent) {
throw new Error('Agent not available');
}
const thread = await client.agents.createThread();
await client.agents.createMessage(thread.id, {
role: 'user',
content: buildGradingPrompt(submission, task)
});
const thread = await client.agents.createThread();
await client.agents.createMessage(thread.id, {
role: 'user',
content: buildGradingPrompt(submission, task)
});
let initialStream: AsyncIterable<RunStreamEvent>;
try {
const runInvoker = client.agents.createRun(thread.id, agent.id, {
parallelToolCalls: false
});
initialStream = await runInvoker.stream();
} catch (err) {
console.error('Failed to start run:', err);
return fallbackGrade(submission, task);
}
let initialStream: AsyncIterable<RunStreamEvent>;
try {
const runInvoker = client.agents.createRun(thread.id, agent.id, {
parallelToolCalls: false
});
initialStream = await runInvoker.stream();
} catch (err) {
console.error('Failed to start run:', err);
return fallbackGrade(submission, task);
}
// process all tool calls
let finalRun: ThreadRunOutput;
try {
finalRun = await processRunStream(thread.id, initialStream, roomId);
} catch (err) {
console.error('Agent streaming failed:', err);
return fallbackGrade(submission, task);
}
// process all tool calls
let finalRun: ThreadRunOutput;
try {
finalRun = await processRunStream(thread.id, initialStream, roomId);
} catch (err) {
console.error('Agent streaming failed:', err);
return fallbackGrade(submission, task);
}
const msgs = await client.agents.listMessages(thread.id);
const assistant = msgs.data.find((m) => m.role === 'assistant');
const raw = assistant ? extractTextFromMessage(assistant) : '';
const msgs = await client.agents.listMessages(thread.id);
const assistant = msgs.data.find(m => m.role === 'assistant');
const raw = assistant ? extractTextFromMessage(assistant) : '';
// strip anything before the JSON object
const match = raw.match(/\{[\s\S]*\}$/);
const jsonText = match ? match[0] : raw;
// strip anything before the JSON object
const match = raw.match(/\{[\s\S]*\}$/);
const jsonText = match ? match[0] : raw;
try {
return JSON.parse(jsonText) as OpenAIResponse;
} catch (parseErr) {
console.error('JSON parse failed, returning fallback:', parseErr);
return { ...fallbackGrade(submission, task), reasoning: raw };
}
}
try {
return JSON.parse(jsonText) as OpenAIResponse;
} catch (parseErr) {
console.error('JSON parse failed, returning fallback:', parseErr);
return { ...fallbackGrade(submission, task), reasoning: raw };
}
}
tests/bdd/steps/agentic_progress.steps.ts
-6
View File
@@ -2,12 +2,6 @@ import { Given, When, Then, Before } from '@cucumber/cucumber';
import { expect } from '@playwright/test';
import { UploadPage } from '../pages/UploadPage.ts';
Before(function () {
if (process.env.AI_FOUNDRY_PROJECT_CONNECTION_STRING === 'test') {
console.log('Skipping in CI');
return 'skipped';
}
});
Given('I have submitted an assignment for assessment', async function () {
await this.uploadPage.navigateTo();