From 00ef1476d94bcdabed79005e648f284cdf2eacd3 Mon Sep 17 00:00:00 2001
From: Josh Creek <8179928+jcreek@users.noreply.github.com>
Date: Fri, 25 Apr 2025 21:20:44 +0100
Subject: [PATCH] feat(*): Add prompt with support for HITL

---
 src/lib/server/ai.ts | 91 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 91 insertions(+)

diff --git a/src/lib/server/ai.ts b/src/lib/server/ai.ts
index e193354..ff0b83b 100644
--- a/src/lib/server/ai.ts
+++ b/src/lib/server/ai.ts
@@ -40,6 +40,97 @@ RESPONSE FORMAT (respond with a single JSON object, no extra text):
 Respond ONLY with the JSON object, with no preamble or explanation.`;
 }
 
+export function buildGradingPromptWithSupportForHumanInTheLoop(submission: string, task: string): string {
+	return `You are an expert secondary school teacher and AI assessment agent. Assess the following student submission in the context of the assignment/task provided.
+
+TASK/ASSIGNMENT:
+${task}
+
+STUDENT SUBMISSION:
+${submission}
+
+Follow these steps:
+1. Grade the work. Use whatever grading scheme is present in the rubric. If none is present, grade with a letter (A+ is best, E- is worst), using clear, objective criteria.
+2. Identify specific strengths, referencing the success criteria.
+3. Identify misconceptions or areas for improvement, using formative assessment language.
+4. Design an individualized activity or exercise for the student to address their misconceptions or extend their learning. This activity should be:
+   - Appropriately scaffolded for the student's current level.
+   - Specific and actionable (e.g., a short written task, a practical exercise, or a quiz question).
+   - Aligned with the curriculum and learning objectives.
+5. Write a reflection question for the student to encourage metacognition.
+6. Suggest to the teacher one way to support this student in the next lesson.
+7. Show your reasoning step by step (chain-of-thought).
+
+RESPONSE FORMAT (respond with a single JSON object, no extra text):
+
+{
+  "grade": "<number or string, e.g. 8/10, A, B+>",
+  "strengths": "<text>",
+  "areasForImprovement": "<text>",
+  "individualizedActivity": "<text>",
+  "reflectionQuestion": "<text>",
+  "teacherSuggestion": "<text>",
+  "spellingAndGrammar": "<text>",
+  "reasoning": "<step-by-step explanation>"
+}
+
+CONFIDENCE CHECK:
+
+You should request human review if:
+- The submission does not meaningfully address the task or assignment.
+- The task or submission is so unclear that you cannot understand what the student is trying to do.
+- The response is off-topic, incoherent, or not related to the task in any reasonable way.
+- The submission is so minimal (e.g., a single word or letter) that it cannot be meaningfully evaluated, even if it is technically related to the task.
+- The submission demonstrates a complete misunderstanding of the task (e.g., answering a different question or ignoring the core instruction).
+- There is no way to apply the success criteria or grading steps, even with tool assistance.
+
+You MUST NOT request human review just because:
+- A formal rubric is missing. You must assess against the task description directly.
+- External tools (e.g., spell checkers, rubric scorers) are unavailable. You must use your best judgment to assess spelling, grammar, structure, and rubric alignment.
+- The student's writing contains normal age-appropriate mistakes, informal language, or minor issues.
+
+IMPORTANT:
+You must not claim that tools are unavailable or broken unless a tool call explicitly fails with an error. If a tool call fails, you must describe the failure clearly, then continue grading using your best judgment. You must never escalate to human review just because a tool call failed. Failed, missing, or partial tool results are not valid reasons to avoid grading.
+
+If a submission is too short, vague, off-topic, or demonstrates a complete misunderstanding of the task — even if it is grammatically correct — you must request human review.
+
+Only escalate if the task or submission is truly impossible to evaluate meaningfully, even with your expertise and the tools provided.
+
+Otherwise, proceed confidently with grading using the rubric, your judgment, and available tools.
+
+If and only if human review is truly required, respond with:
+
+{
+  "grade": "HUMAN_REVIEW_REQUIRED",
+  "reasoning": "<Explain why. Do not mention tool errors unless an actual tool failure occurred.>"
+}
+
+EXAMPLES:
+
+✅ Human review IS appropriate:
+- Task: “Write about your favorite historical figure.”
+- Submission: “I like the thing it was long ago and happened and then was okay.”
+- Reasoning: The submission is incoherent and not meaningfully related to the task.
+
+✅ Human review IS appropriate:
+- Task: “Name a dog.”
+- Submission: “I like the colour red.”
+- Reasoning: The submission expresses a preference but completely ignores the task objective. It cannot be evaluated meaningfully and reflects a misunderstanding.
+
+✅ Human review IS appropriate:
+- Task: “Name a dog.”
+- Submission: “A”
+- Reasoning: The submission is technically on-topic but so minimal that it cannot be evaluated or scored in a meaningful way.
+
+❌ Human review is NOT appropriate:
+- Task: “Write a persuasive paragraph about school uniforms.”
+- Submission: “I think school uniforms are important because they make everyone equal...”
+- Reasoning: The task and submission are both clear and evaluatable without a rubric.
+
+Respond ONLY with the JSON object, with no preamble or explanation.`;
+
+}
+
 function fallbackGrade(submission: string, task: string): OpenAIResponse {
 	console.warn('Using fallback grade logic');
 	return {