From 426e52f7651eea8b12d3bd53fc1b232e5dc17f9f Mon Sep 17 00:00:00 2001 From: Josh Creek <8179928+jcreek@users.noreply.github.com> Date: Thu, 24 Apr 2025 21:08:13 +0100 Subject: [PATCH] refactor(*): Change CI to use secrets instead of fallbacks --- .github/workflows/ci.yml | 8 +- src/lib/agent/services/agentService.ts | 38 ++-- src/lib/utils/ai.ts | 246 ++++++++++------------ tests/bdd/steps/agentic_progress.steps.ts | 6 - 4 files changed, 135 insertions(+), 163 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b2b4017..e7ef3e0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,6 +9,7 @@ on: jobs: test-and-deploy: runs-on: ubuntu-latest + environment: test steps: - uses: actions/checkout@v4 - name: Use Node.js @@ -24,8 +25,7 @@ jobs: - name: Run Playwright/BDD tests run: npm run bdd:full env: - NODE_ENV: test - AI_FOUNDRY_PROJECT_CONNECTION_STRING: test + AI_FOUNDRY_PROJECT_CONNECTION_STRING: ${{ secrets.AI_FOUNDRY_PROJECT_CONNECTION_STRING }} AI_MODEL: gpt-4o - VITE_PARTYKIT_BASE_URL: wss://partykit.jcreek.partykit.dev - PARTYKIT_BASE_URL: wss://partykit.jcreek.partykit.dev + VITE_PARTYKIT_BASE_URL: ${{ secrets.VITE_PARTYKIT_BASE_URL }} + PARTYKIT_BASE_URL: ${{ secrets.PARTYKIT_BASE_URL }} diff --git a/src/lib/agent/services/agentService.ts b/src/lib/agent/services/agentService.ts index bd55199..603e7ec 100644 --- a/src/lib/agent/services/agentService.ts +++ b/src/lib/agent/services/agentService.ts @@ -11,21 +11,17 @@ import { } from '../tools/index'; import { AI_FOUNDRY_PROJECT_CONNECTION_STRING, AI_MODEL } from '$env/static/private'; -const isTestEnv = process.env.NODE_ENV === 'test' || AI_FOUNDRY_PROJECT_CONNECTION_STRING === 'test'; - -const safeConnString = AI_FOUNDRY_PROJECT_CONNECTION_STRING || (isTestEnv ? 'dummy-connection-string' : undefined); -const safeModel = AI_MODEL || (isTestEnv ? 'dummy-model' : undefined); - -if (!safeConnString && !isTestEnv) { +if (!AI_FOUNDRY_PROJECT_CONNECTION_STRING) { throw new Error('AI_FOUNDRY_PROJECT_CONNECTION_STRING is not set'); } -if (!safeModel && !isTestEnv) { +if (!AI_MODEL) { throw new Error('AI_MODEL is not set'); } -export const client = !isTestEnv && safeConnString - ? AIProjectsClient.fromConnectionString(safeConnString, new DefaultAzureCredential()) - : undefined; +export const client = AIProjectsClient.fromConnectionString( + AI_FOUNDRY_PROJECT_CONNECTION_STRING, + new DefaultAzureCredential() +); // const codeInterpreterTool = ToolUtility.createCodeInterpreterTool([]); @@ -70,15 +66,13 @@ export const toolResources = allTools.reduce>((map, t) => { return map; }, {}); -export const agent = client - ? await client.agents.createAgent(AI_MODEL, { - name: `assessment-feedback-agent`, - instructions, - temperature: 0.5, - tools, - toolResources, - requestOptions: { - headers: { 'x-ms-enable-preview': 'true' } - } - }) - : undefined; \ No newline at end of file +export const agent = await client.agents.createAgent(AI_MODEL, { + name: `assessment-feedback-agent`, + instructions, + temperature: 0.5, + tools, + toolResources, + requestOptions: { + headers: { 'x-ms-enable-preview': 'true' } + } +}); \ No newline at end of file diff --git a/src/lib/utils/ai.ts b/src/lib/utils/ai.ts index b8eaf3a..13878d7 100644 --- a/src/lib/utils/ai.ts +++ b/src/lib/utils/ai.ts @@ -1,14 +1,10 @@ import { client, agent, toolResources } from '../agent/services/agentService'; -import { - RunStreamEvent, - ErrorEvent, - type ThreadRunOutput -} from '@azure/ai-projects'; +import { RunStreamEvent, ErrorEvent, type ThreadRunOutput } from '@azure/ai-projects'; import { PARTYKIT_BASE_URL } from '$env/static/private'; import type { OpenAIResponse } from './types'; export function buildGradingPrompt(submission: string, task: string): string { - return `You are an expert secondary school teacher and AI assessment agent. Assess the following student submission in the context of the assignment/task provided. + return `You are an expert secondary school teacher and AI assessment agent. Assess the following student submission in the context of the assignment/task provided. TASK/ASSIGNMENT: ${task} @@ -45,153 +41,141 @@ Respond ONLY with the JSON object, with no preamble or explanation.`; } function fallbackGrade(submission: string, task: string): OpenAIResponse { - console.warn('Using fallback grade logic'); - return { - grade: 'EXAMPLE', - strengths: `Clear argument and good evidence (task: ${task}).`, - areasForImprovement: 'Needs deeper analysis.', - individualizedActivity: 'Add two more supporting examples.', - reflectionQuestion: 'Which part was hardest?', - teacherSuggestion: 'Model a paragraph with evidence.', - spellingAndGrammar: 'No errors detected.', - reasoning: 'Standard fallback reasoning.' - }; + console.warn('Using fallback grade logic'); + return { + grade: 'EXAMPLE', + strengths: `Clear argument and good evidence (task: ${task}).`, + areasForImprovement: 'Needs deeper analysis.', + individualizedActivity: 'Add two more supporting examples.', + reflectionQuestion: 'Which part was hardest?', + teacherSuggestion: 'Model a paragraph with evidence.', + spellingAndGrammar: 'No errors detected.', + reasoning: 'Standard fallback reasoning.' + }; } function extractTextFromMessage(msg: { content: any[] }): string { - return msg.content - .filter(c => c.type === 'text') - .map(c => (typeof c.text === 'string' ? c.text : c.text.value ?? '')) - .join('\n') - .trim(); + return msg.content + .filter((c) => c.type === 'text') + .map((c) => (typeof c.text === 'string' ? c.text : (c.text.value ?? ''))) + .join('\n') + .trim(); } // Recursively process each run stream, invoke the tools & notify PartyKit async function processRunStream( - threadId: string, - stream: AsyncIterable, - roomId: string + threadId: string, + stream: AsyncIterable, + roomId: string ): Promise { - for await (const evt of stream) { - // if (!evt.event.includes('delta')) { - // console.log('▶️ Stream event:', evt.event); - // } + for await (const evt of stream) { + // if (!evt.event.includes('delta')) { + // console.log('▶️ Stream event:', evt.event); + // } - if (evt.event === RunStreamEvent.ThreadRunRequiresAction) { - const runOutput = evt.data as ThreadRunOutput; - const calls = runOutput.requiredAction!.submitToolOutputs!.toolCalls!; + if (evt.event === RunStreamEvent.ThreadRunRequiresAction) { + const runOutput = evt.data as ThreadRunOutput; + const calls = runOutput.requiredAction!.submitToolOutputs!.toolCalls!; - // notify PartyKit - await Promise.all( - calls.map(async call => { - if (!PARTYKIT_BASE_URL) return; - const wsCtor = typeof WebSocket !== 'undefined' - ? WebSocket - : (await import('ws')).default; - const ws = new wsCtor( - `${PARTYKIT_BASE_URL}/party/tool-usage-server-${roomId}` - ); - await new Promise((res, rej) => { - ws.onopen = () => res(); - ws.onerror = e => rej(e); - }); - ws.send(JSON.stringify({ tool: call.function.name, time: new Date().toISOString() })); - ws.close(); - }) - ); + // notify PartyKit + await Promise.all( + calls.map(async (call) => { + if (!PARTYKIT_BASE_URL) return; + const wsCtor = + typeof WebSocket !== 'undefined' ? WebSocket : (await import('ws')).default; + const ws = new wsCtor(`${PARTYKIT_BASE_URL}/party/tool-usage-server-${roomId}`); + await new Promise((res, rej) => { + ws.onopen = () => res(); + ws.onerror = (e) => rej(e); + }); + ws.send(JSON.stringify({ tool: call.function.name, time: new Date().toISOString() })); + ws.close(); + }) + ); - // invoke the tools - const toolOutputs = await Promise.all( - calls.map(async call => { - const name = call.function.name; - const args = call.arguments!; - try { - const fn = toolResources[name]; - if (typeof fn !== 'function') { - throw new Error(`No tool implementation for "${name}"`); - } - const result = await fn(args); - return { toolCallId: call.id, output: result }; - } catch (err: any) { - return { toolCallId: call.id, output: `Tool error: ${err.message}` }; - } - }) - ); + // invoke the tools + const toolOutputs = await Promise.all( + calls.map(async (call) => { + const name = call.function.name; + const args = call.arguments!; + try { + const fn = toolResources[name]; + if (typeof fn !== 'function') { + throw new Error(`No tool implementation for "${name}"`); + } + const result = await fn(args); + return { toolCallId: call.id, output: result }; + } catch (err: any) { + return { toolCallId: call.id, output: `Tool error: ${err.message}` }; + } + }) + ); - // resume the run - const resumed = client.agents.submitToolOutputsToRun( - threadId, - runOutput.id, - toolOutputs - ); - const nextStream = await resumed.stream(); - return processRunStream(threadId, nextStream, roomId); - } + // resume the run + const resumed = client.agents.submitToolOutputsToRun(threadId, runOutput.id, toolOutputs); + const nextStream = await resumed.stream(); + return processRunStream(threadId, nextStream, roomId); + } - if (evt.event === RunStreamEvent.ThreadRunCompleted) { - return evt.data as ThreadRunOutput; - } + if (evt.event === RunStreamEvent.ThreadRunCompleted) { + return evt.data as ThreadRunOutput; + } - if (evt.event === ErrorEvent.Error) { - throw new Error(`Agent error: ${JSON.stringify(evt.data)}`); - } - } + if (evt.event === ErrorEvent.Error) { + throw new Error(`Agent error: ${JSON.stringify(evt.data)}`); + } + } - throw new Error('Stream ended without completion'); + throw new Error('Stream ended without completion'); } export async function gradeSubmissionWithAgent( - submission: string, - task: string, - roomId: string + submission: string, + task: string, + roomId: string ): Promise { - if (!client) { - // In test/CI mode, return fallback data - return fallbackGrade(submission, task); - } + if (!client || !agent) { + throw new Error('Agent not available'); + } - if (!agent) { - throw new Error('Agent not available'); - } + const thread = await client.agents.createThread(); + await client.agents.createMessage(thread.id, { + role: 'user', + content: buildGradingPrompt(submission, task) + }); - const thread = await client.agents.createThread(); - await client.agents.createMessage(thread.id, { - role: 'user', - content: buildGradingPrompt(submission, task) - }); + let initialStream: AsyncIterable; + try { + const runInvoker = client.agents.createRun(thread.id, agent.id, { + parallelToolCalls: false + }); + initialStream = await runInvoker.stream(); + } catch (err) { + console.error('Failed to start run:', err); + return fallbackGrade(submission, task); + } - let initialStream: AsyncIterable; - try { - const runInvoker = client.agents.createRun(thread.id, agent.id, { - parallelToolCalls: false - }); - initialStream = await runInvoker.stream(); - } catch (err) { - console.error('Failed to start run:', err); - return fallbackGrade(submission, task); - } + // process all tool calls + let finalRun: ThreadRunOutput; + try { + finalRun = await processRunStream(thread.id, initialStream, roomId); + } catch (err) { + console.error('Agent streaming failed:', err); + return fallbackGrade(submission, task); + } - // process all tool calls - let finalRun: ThreadRunOutput; - try { - finalRun = await processRunStream(thread.id, initialStream, roomId); - } catch (err) { - console.error('Agent streaming failed:', err); - return fallbackGrade(submission, task); - } + const msgs = await client.agents.listMessages(thread.id); + const assistant = msgs.data.find((m) => m.role === 'assistant'); + const raw = assistant ? extractTextFromMessage(assistant) : ''; - const msgs = await client.agents.listMessages(thread.id); - const assistant = msgs.data.find(m => m.role === 'assistant'); - const raw = assistant ? extractTextFromMessage(assistant) : ''; + // strip anything before the JSON object + const match = raw.match(/\{[\s\S]*\}$/); + const jsonText = match ? match[0] : raw; - // strip anything before the JSON object - const match = raw.match(/\{[\s\S]*\}$/); - const jsonText = match ? match[0] : raw; - - try { - return JSON.parse(jsonText) as OpenAIResponse; - } catch (parseErr) { - console.error('JSON parse failed, returning fallback:', parseErr); - return { ...fallbackGrade(submission, task), reasoning: raw }; - } -} \ No newline at end of file + try { + return JSON.parse(jsonText) as OpenAIResponse; + } catch (parseErr) { + console.error('JSON parse failed, returning fallback:', parseErr); + return { ...fallbackGrade(submission, task), reasoning: raw }; + } +} diff --git a/tests/bdd/steps/agentic_progress.steps.ts b/tests/bdd/steps/agentic_progress.steps.ts index b92fd4c..d675c34 100644 --- a/tests/bdd/steps/agentic_progress.steps.ts +++ b/tests/bdd/steps/agentic_progress.steps.ts @@ -2,12 +2,6 @@ import { Given, When, Then, Before } from '@cucumber/cucumber'; import { expect } from '@playwright/test'; import { UploadPage } from '../pages/UploadPage.ts'; -Before(function () { - if (process.env.AI_FOUNDRY_PROJECT_CONNECTION_STRING === 'test') { - console.log('Skipping in CI'); - return 'skipped'; - } -}); Given('I have submitted an assignment for assessment', async function () { await this.uploadPage.navigateTo();