jcreek/AutomatedAssessmentFeedbackAgent
1
0
Fork 0
mirror of https://github.com/jcreek/AutomatedAssessmentFeedbackAgent.git synced 2026-07-14 19:43:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(*): Change CI to use secrets instead of fallbacks

Browse Source
This commit is contained in:
Josh Creek
2025-04-24 21:08:13 +01:00
parent f421a0a110
commit 426e52f765
4 changed files with 135 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/ci.yml
+4 -4
View File
@@ -9,6 +9,7 @@ on:
jobs: jobs:
test-and-deploy: test-and-deploy:
runs-on: ubuntu-latest runs-on: ubuntu-latest
environment: test
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
- name: Use Node.js - name: Use Node.js
@@ -24,8 +25,7 @@ jobs:
- name: Run Playwright/BDD tests - name: Run Playwright/BDD tests
run: npm run bdd:full run: npm run bdd:full
env: env:
NODE_ENV: test AI_FOUNDRY_PROJECT_CONNECTION_STRING: ${{ secrets.AI_FOUNDRY_PROJECT_CONNECTION_STRING }}
AI_FOUNDRY_PROJECT_CONNECTION_STRING: test
AI_MODEL: gpt-4o AI_MODEL: gpt-4o
VITE_PARTYKIT_BASE_URL: wss://partykit.jcreek.partykit.dev VITE_PARTYKIT_BASE_URL: ${{ secrets.VITE_PARTYKIT_BASE_URL }}
PARTYKIT_BASE_URL: wss://partykit.jcreek.partykit.dev PARTYKIT_BASE_URL: ${{ secrets.PARTYKIT_BASE_URL }}
src/lib/agent/services/agentService.ts
+16 -22
View File
@@ -11,21 +11,17 @@ import {
} from '../tools/index'; } from '../tools/index';
import { AI_FOUNDRY_PROJECT_CONNECTION_STRING, AI_MODEL } from '$env/static/private'; import { AI_FOUNDRY_PROJECT_CONNECTION_STRING, AI_MODEL } from '$env/static/private';
const isTestEnv = process.env.NODE_ENV === 'test' || AI_FOUNDRY_PROJECT_CONNECTION_STRING === 'test'; if (!AI_FOUNDRY_PROJECT_CONNECTION_STRING) {
const safeConnString = AI_FOUNDRY_PROJECT_CONNECTION_STRING || (isTestEnv ? 'dummy-connection-string' : undefined);
const safeModel = AI_MODEL || (isTestEnv ? 'dummy-model' : undefined);
if (!safeConnString && !isTestEnv) {
throw new Error('AI_FOUNDRY_PROJECT_CONNECTION_STRING is not set'); throw new Error('AI_FOUNDRY_PROJECT_CONNECTION_STRING is not set');
} }
if (!safeModel && !isTestEnv) { if (!AI_MODEL) {
throw new Error('AI_MODEL is not set'); throw new Error('AI_MODEL is not set');
} }
export const client = !isTestEnv && safeConnString export const client = AIProjectsClient.fromConnectionString(
? AIProjectsClient.fromConnectionString(safeConnString, new DefaultAzureCredential()) AI_FOUNDRY_PROJECT_CONNECTION_STRING,
: undefined; new DefaultAzureCredential()
);
// const codeInterpreterTool = ToolUtility.createCodeInterpreterTool([]); // const codeInterpreterTool = ToolUtility.createCodeInterpreterTool([]);
@@ -70,15 +66,13 @@ export const toolResources = allTools.reduce<Record<string, any>>((map, t) => {
return map; return map;
}, {}); }, {});
export const agent = client export const agent = await client.agents.createAgent(AI_MODEL, {
? await client.agents.createAgent(AI_MODEL, { name: `assessment-feedback-agent`,
name: `assessment-feedback-agent`, instructions,
instructions, temperature: 0.5,
temperature: 0.5, tools,
tools, toolResources,
toolResources, requestOptions: {
requestOptions: { headers: { 'x-ms-enable-preview': 'true' }
headers: { 'x-ms-enable-preview': 'true' } }
} });
})
: undefined;
src/lib/utils/ai.ts
+115 -131
View File
@@ -1,14 +1,10 @@
import { client, agent, toolResources } from '../agent/services/agentService'; import { client, agent, toolResources } from '../agent/services/agentService';
import { import { RunStreamEvent, ErrorEvent, type ThreadRunOutput } from '@azure/ai-projects';
RunStreamEvent,
ErrorEvent,
type ThreadRunOutput
} from '@azure/ai-projects';
import { PARTYKIT_BASE_URL } from '$env/static/private'; import { PARTYKIT_BASE_URL } from '$env/static/private';
import type { OpenAIResponse } from './types'; import type { OpenAIResponse } from './types';
export function buildGradingPrompt(submission: string, task: string): string { export function buildGradingPrompt(submission: string, task: string): string {
return `You are an expert secondary school teacher and AI assessment agent. Assess the following student submission in the context of the assignment/task provided. return `You are an expert secondary school teacher and AI assessment agent. Assess the following student submission in the context of the assignment/task provided.
TASK/ASSIGNMENT: TASK/ASSIGNMENT:
${task} ${task}
@@ -45,153 +41,141 @@ Respond ONLY with the JSON object, with no preamble or explanation.`;
} }
function fallbackGrade(submission: string, task: string): OpenAIResponse { function fallbackGrade(submission: string, task: string): OpenAIResponse {
console.warn('Using fallback grade logic'); console.warn('Using fallback grade logic');
return { return {
grade: 'EXAMPLE', grade: 'EXAMPLE',
strengths: `Clear argument and good evidence (task: ${task}).`, strengths: `Clear argument and good evidence (task: ${task}).`,
areasForImprovement: 'Needs deeper analysis.', areasForImprovement: 'Needs deeper analysis.',
individualizedActivity: 'Add two more supporting examples.', individualizedActivity: 'Add two more supporting examples.',
reflectionQuestion: 'Which part was hardest?', reflectionQuestion: 'Which part was hardest?',
teacherSuggestion: 'Model a paragraph with evidence.', teacherSuggestion: 'Model a paragraph with evidence.',
spellingAndGrammar: 'No errors detected.', spellingAndGrammar: 'No errors detected.',
reasoning: 'Standard fallback reasoning.' reasoning: 'Standard fallback reasoning.'
}; };
} }
function extractTextFromMessage(msg: { content: any[] }): string { function extractTextFromMessage(msg: { content: any[] }): string {
return msg.content return msg.content
.filter(c => c.type === 'text') .filter((c) => c.type === 'text')
.map(c => (typeof c.text === 'string' ? c.text : c.text.value ?? '')) .map((c) => (typeof c.text === 'string' ? c.text : (c.text.value ?? '')))
.join('\n') .join('\n')
.trim(); .trim();
} }
// Recursively process each run stream, invoke the tools & notify PartyKit // Recursively process each run stream, invoke the tools & notify PartyKit
async function processRunStream( async function processRunStream(
threadId: string, threadId: string,
stream: AsyncIterable<RunStreamEvent>, stream: AsyncIterable<RunStreamEvent>,
roomId: string roomId: string
): Promise<ThreadRunOutput> { ): Promise<ThreadRunOutput> {
for await (const evt of stream) { for await (const evt of stream) {
// if (!evt.event.includes('delta')) { // if (!evt.event.includes('delta')) {
// console.log('▶️ Stream event:', evt.event); // console.log('▶️ Stream event:', evt.event);
// } // }
if (evt.event === RunStreamEvent.ThreadRunRequiresAction) { if (evt.event === RunStreamEvent.ThreadRunRequiresAction) {
const runOutput = evt.data as ThreadRunOutput; const runOutput = evt.data as ThreadRunOutput;
const calls = runOutput.requiredAction!.submitToolOutputs!.toolCalls!; const calls = runOutput.requiredAction!.submitToolOutputs!.toolCalls!;
// notify PartyKit // notify PartyKit
await Promise.all( await Promise.all(
calls.map(async call => { calls.map(async (call) => {
if (!PARTYKIT_BASE_URL) return; if (!PARTYKIT_BASE_URL) return;
const wsCtor = typeof WebSocket !== 'undefined' const wsCtor =
? WebSocket typeof WebSocket !== 'undefined' ? WebSocket : (await import('ws')).default;
: (await import('ws')).default; const ws = new wsCtor(`${PARTYKIT_BASE_URL}/party/tool-usage-server-${roomId}`);
const ws = new wsCtor( await new Promise<void>((res, rej) => {
`${PARTYKIT_BASE_URL}/party/tool-usage-server-${roomId}` ws.onopen = () => res();
); ws.onerror = (e) => rej(e);
await new Promise<void>((res, rej) => { });
ws.onopen = () => res(); ws.send(JSON.stringify({ tool: call.function.name, time: new Date().toISOString() }));
ws.onerror = e => rej(e); ws.close();
}); })
ws.send(JSON.stringify({ tool: call.function.name, time: new Date().toISOString() })); );
ws.close();
})
);
// invoke the tools // invoke the tools
const toolOutputs = await Promise.all( const toolOutputs = await Promise.all(
calls.map(async call => { calls.map(async (call) => {
const name = call.function.name; const name = call.function.name;
const args = call.arguments!; const args = call.arguments!;
try { try {
const fn = toolResources[name]; const fn = toolResources[name];
if (typeof fn !== 'function') { if (typeof fn !== 'function') {
throw new Error(`No tool implementation for "${name}"`); throw new Error(`No tool implementation for "${name}"`);
} }
const result = await fn(args); const result = await fn(args);
return { toolCallId: call.id, output: result }; return { toolCallId: call.id, output: result };
} catch (err: any) { } catch (err: any) {
return { toolCallId: call.id, output: `Tool error: ${err.message}` }; return { toolCallId: call.id, output: `Tool error: ${err.message}` };
} }
}) })
); );
// resume the run // resume the run
const resumed = client.agents.submitToolOutputsToRun( const resumed = client.agents.submitToolOutputsToRun(threadId, runOutput.id, toolOutputs);
threadId, const nextStream = await resumed.stream();
runOutput.id, return processRunStream(threadId, nextStream, roomId);
toolOutputs }
);
const nextStream = await resumed.stream();
return processRunStream(threadId, nextStream, roomId);
}
if (evt.event === RunStreamEvent.ThreadRunCompleted) { if (evt.event === RunStreamEvent.ThreadRunCompleted) {
return evt.data as ThreadRunOutput; return evt.data as ThreadRunOutput;
} }
if (evt.event === ErrorEvent.Error) { if (evt.event === ErrorEvent.Error) {
throw new Error(`Agent error: ${JSON.stringify(evt.data)}`); throw new Error(`Agent error: ${JSON.stringify(evt.data)}`);
} }
} }
throw new Error('Stream ended without completion'); throw new Error('Stream ended without completion');
} }
export async function gradeSubmissionWithAgent( export async function gradeSubmissionWithAgent(
submission: string, submission: string,
task: string, task: string,
roomId: string roomId: string
): Promise<OpenAIResponse> { ): Promise<OpenAIResponse> {
if (!client) { if (!client || !agent) {
// In test/CI mode, return fallback data throw new Error('Agent not available');
return fallbackGrade(submission, task); }
}
if (!agent) { const thread = await client.agents.createThread();
throw new Error('Agent not available'); await client.agents.createMessage(thread.id, {
} role: 'user',
content: buildGradingPrompt(submission, task)
});
const thread = await client.agents.createThread(); let initialStream: AsyncIterable<RunStreamEvent>;
await client.agents.createMessage(thread.id, { try {
role: 'user', const runInvoker = client.agents.createRun(thread.id, agent.id, {
content: buildGradingPrompt(submission, task) parallelToolCalls: false
}); });
initialStream = await runInvoker.stream();
} catch (err) {
console.error('Failed to start run:', err);
return fallbackGrade(submission, task);
}
let initialStream: AsyncIterable<RunStreamEvent>; // process all tool calls
try { let finalRun: ThreadRunOutput;
const runInvoker = client.agents.createRun(thread.id, agent.id, { try {
parallelToolCalls: false finalRun = await processRunStream(thread.id, initialStream, roomId);
}); } catch (err) {
initialStream = await runInvoker.stream(); console.error('Agent streaming failed:', err);
} catch (err) { return fallbackGrade(submission, task);
console.error('Failed to start run:', err); }
return fallbackGrade(submission, task);
}
// process all tool calls const msgs = await client.agents.listMessages(thread.id);
let finalRun: ThreadRunOutput; const assistant = msgs.data.find((m) => m.role === 'assistant');
try { const raw = assistant ? extractTextFromMessage(assistant) : '';
finalRun = await processRunStream(thread.id, initialStream, roomId);
} catch (err) {
console.error('Agent streaming failed:', err);
return fallbackGrade(submission, task);
}
const msgs = await client.agents.listMessages(thread.id); // strip anything before the JSON object
const assistant = msgs.data.find(m => m.role === 'assistant'); const match = raw.match(/\{[\s\S]*\}$/);
const raw = assistant ? extractTextFromMessage(assistant) : ''; const jsonText = match ? match[0] : raw;
// strip anything before the JSON object try {
const match = raw.match(/\{[\s\S]*\}$/); return JSON.parse(jsonText) as OpenAIResponse;
const jsonText = match ? match[0] : raw; } catch (parseErr) {
console.error('JSON parse failed, returning fallback:', parseErr);
try { return { ...fallbackGrade(submission, task), reasoning: raw };
return JSON.parse(jsonText) as OpenAIResponse; }
} catch (parseErr) { }
console.error('JSON parse failed, returning fallback:', parseErr);
return { ...fallbackGrade(submission, task), reasoning: raw };
}
}
tests/bdd/steps/agentic_progress.steps.ts
-6
View File
@@ -2,12 +2,6 @@ import { Given, When, Then, Before } from '@cucumber/cucumber';
import { expect } from '@playwright/test'; import { expect } from '@playwright/test';
import { UploadPage } from '../pages/UploadPage.ts'; import { UploadPage } from '../pages/UploadPage.ts';
Before(function () {
if (process.env.AI_FOUNDRY_PROJECT_CONNECTION_STRING === 'test') {
console.log('Skipping in CI');
return 'skipped';
}
});
Given('I have submitted an assignment for assessment', async function () { Given('I have submitted an assignment for assessment', async function () {
await this.uploadPage.navigateTo(); await this.uploadPage.navigateTo();