feat(*): Be stricter regarding HITL

src/lib/server/ai.ts

@@ -4,31 +4,31 @@ import { PARTYKIT_BASE_URL } from '$env/static/private';
 import type { OpenAIResponse } from '../utils/types';
 
 function sanitizeForPrompt(input: string): string {
-  return input
+	return input
-    .replace(/\\/g, '\\\\')
+		.replace(/\\/g, '\\\\')
-    .replace(/`/g, '\\`')
+		.replace(/`/g, '\\`')
-    .replace(/"/g, '\\"')
+		.replace(/"/g, '\\"')
-    .replace(/'/g, "\\'")
+		.replace(/'/g, "\\'")
-    .replace(/\n/g, '\\n')
+		.replace(/\n/g, '\\n')
-    .replace(/\r/g, '\\r')
+		.replace(/\r/g, '\\r')
-    .replace(/\t/g, '\\t')
+		.replace(/\t/g, '\\t')
-    .replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ''); // remove other control chars
+		.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ''); // remove other control chars
 }
 
 function buildBaseGradingPrompt({
-  task,
+	task,
-  submission,
+	submission,
-  steps,
+	steps,
-  responseFormat,
+	responseFormat,
-  extraSections = ''
+	extraSections = ''
 }: {
-  task: string;
+	task: string;
-  submission: string;
+	submission: string;
-  steps: string;
+	steps: string;
-  responseFormat: string;
+	responseFormat: string;
-  extraSections?: string;
+	extraSections?: string;
 }): string {
-  return `You are an expert school teacher and AI assessment agent. Assess the following student submission in the context of the assignment/task provided.
+	return `You are an expert school teacher and AI assessment agent. Assess the following student submission in the context of the assignment/task provided.
 
 ===== TASK/ASSIGNMENT possibly including rubric or available marks =====
 
@@ -50,9 +50,9 @@ ${extraSections}`;
 }
 
 export function buildGradingPrompt(submission: string, task: string): string {
-  const safeSubmission = sanitizeForPrompt(submission);
+	const safeSubmission = sanitizeForPrompt(submission);
-  const safeTask = sanitizeForPrompt(task);
+	const safeTask = sanitizeForPrompt(task);
-  const steps = `Follow these steps:
+	const steps = `Follow these steps:
 1. Grade the work. Use whatever grading scheme is present in the rubric. If none is present then grade with a letter (A+ is best, E- is worst), using clear, objective criteria. Be aware that the rubric may just give a maximum number of marks, in which case you should mark out of that number, for example "(20)" would mean it's out of 20 marks. DO NOT mention tool errors. 
 2. Identify specific strengths, referencing the success criteria.
 3. Identify misconceptions or areas for improvement, using formative assessment language.
@@ -63,7 +63,7 @@ export function buildGradingPrompt(submission: string, task: string): string {
 5. Write a reflection question for the student to encourage metacognition.
 6. Suggest to the teacher one way to support this student in the next lesson.
 7. Show your reasoning step by step (chain-of-thought).`;
-  const responseFormat = `RESPONSE FORMAT (respond with a single JSON object, no extra text):
+	const responseFormat = `RESPONSE FORMAT (respond with a single JSON object, no extra text):
 
 {
   "grade": "<number or string, e.g. 8/10, A, B+>",
@@ -77,105 +77,119 @@ export function buildGradingPrompt(submission: string, task: string): string {
 }
 
 Respond ONLY with the JSON object, with no preamble or explanation.`;
-  return buildBaseGradingPrompt({
+	return buildBaseGradingPrompt({
-    task: safeTask,
+		task: safeTask,
-    submission: safeSubmission,
+		submission: safeSubmission,
-    steps,
+		steps,
-    responseFormat
+		responseFormat
-  });
+	});
 }
 
 export function buildGradingPromptWithSupportForHumanInTheLoop(
-  submission: string,
+	submission: string,
-  task: string
+	task: string
 ): string {
-  const safeSubmission = sanitizeForPrompt(submission);
+	const safeSubmission = sanitizeForPrompt(submission);
-  const safeTask = sanitizeForPrompt(task);
+	const safeTask = sanitizeForPrompt(task);
-  const steps = `Follow these steps:
-1. Grade the work. Use whatever grading scheme is present in the rubric. If none is present, grade with a letter (A+ is best, E- is worst), using clear, objective criteria. Be aware that the rubric may just give a maximum number of marks, in which case you should mark out of that number, for example "(20)" would mean it's out of 20 marks. DO NOT mention tool errors. 
-2. Identify specific strengths, referencing the success criteria.
-3. Identify misconceptions or areas for improvement, using formative assessment language.
-4. Design an individualized activity or exercise for the student to address their misconceptions or extend their learning. This activity should be:
-   - Appropriately scaffolded for the student's current level.
-   - Specific and actionable (e.g., a short written task, a practical exercise, or a quiz question).
-   - Aligned with the curriculum and learning objectives.
-5. Write a reflection question for the student to encourage metacognition.
-6. Suggest to the teacher one way to support this student in the next lesson.
-7. Show your reasoning step by step (chain-of-thought).`;
-  const responseFormat = `RESPONSE FORMAT (respond with a single JSON object, no extra text):
 
-{
+	const steps = `Follow these steps:
-  "grade": "<number or string, e.g. 8/10, A, B+>",
+  1. Grade the work. Use whatever grading scheme is present in the rubric. If none is present, grade with a letter (A+ is best, E- is worst), using clear, objective criteria. Be aware that the rubric may just give a maximum number of marks, in which case you should mark out of that number, for example "(20)" would mean it's out of 20 marks. DO NOT mention tool errors. 
-  "strengths": "<text>",
+  2. Identify specific strengths, referencing the success criteria.
-  "areasForImprovement": "<text>",
+  3. Identify misconceptions or areas for improvement, using formative assessment language.
-  "individualizedActivity": "<text>",
+  4. Design an individualized activity or exercise for the student to address their misconceptions or extend their learning. This activity should be:
-  "reflectionQuestion": "<text>",
+	 - Appropriately scaffolded for the student's current level.
-  "teacherSuggestion": "<text>",
+	 - Specific and actionable (e.g., a short written task, a practical exercise, or a quiz question).
-  "spellingAndGrammar": "<text>",
+	 - Aligned with the curriculum and learning objectives.
-  "reasoning": "<step-by-step explanation>"
+  5. Write a reflection question for the student to encourage metacognition.
-}`;
+  6. Suggest to the teacher one way to support this student in the next lesson.
-  const extraSections = `
+  7. Show your reasoning step by step (chain-of-thought).`;
-CONFIDENCE CHECK:
 
-You should request human review if:
+	const responseFormat = `RESPONSE FORMAT (respond with a single JSON object, no extra text):
-- The submission does not meaningfully address the task or assignment.
+  
-- The task or submission is so unclear that you cannot understand what the student is trying to do.
+  {
-- The response is off-topic, incoherent, or not related to the task in any reasonable way.
+	"grade": "<number or string, e.g. 8/10, A, B+>",
-- The submission is so minimal (e.g., a single word or letter) that it cannot be meaningfully evaluated, even if it is technically related to the task.
+	"strengths": "<text>",
-- The submission demonstrates a complete misunderstanding of the task (e.g., answering a different question or ignoring the core instruction).
+	"areasForImprovement": "<text>",
-- There is no way to apply the success criteria or grading steps, even with tool assistance.
+	"individualizedActivity": "<text>",
+	"reflectionQuestion": "<text>",
+	"teacherSuggestion": "<text>",
+	"spellingAndGrammar": "<text>",
+	"reasoning": "<step-by-step explanation>"
+  }
+  
+  Respond ONLY with the JSON object, with no preamble or explanation.`;
 
-You MUST NOT request human review just because:
+	const extraSections = `
-- A formal rubric is missing. You must assess against the task description directly.
+  CONFIDENCE CHECK:
-- External tools (e.g., spell checkers, rubric scorers) are unavailable. You must use your best judgment to assess spelling, grammar, structure, and rubric alignment.
+  
-- The student's writing contains normal age-appropriate mistakes, informal language, or minor issues.
+  You should request human review if:
+  - The submission does not meaningfully address the task or assignment.
+  - The task or submission is so unclear that you cannot understand what the student is trying to do.
+  - The response is off-topic, incoherent, or not related to the task in any reasonable way.
+  - The submission is so minimal (e.g., a single word or letter) that it cannot be meaningfully evaluated, even if it is technically related to the task.
+  - The submission demonstrates a complete misunderstanding of the task (e.g., answering a different question or ignoring the core instruction).
+  - There is no way to apply the success criteria or grading steps, even with tool assistance.
+  
+  You MUST NOT request human review just because:
+  - A formal rubric is missing. You must assess against the task description directly.
+  - External tools (e.g., spell checkers, rubric scorers) are unavailable. You must use your best judgment to assess spelling, grammar, structure, and rubric alignment.
+  - The student's writing contains normal age-appropriate mistakes, informal language, or minor issues.
+  
+  IMPORTANT:
+  If a tool call fails, you must continue grading using your best professional judgment. Tool failures are normal and are not a valid reason to request human review.
+  
+  You must evaluate the submission manually, even if rubric analysis, spelling, or grammar checking tools are unavailable.
+  
+  You MUST NOT escalate to human review just because of uncertainty about:
+  - Spelling
+  - Grammar
+  - Alignment with specific rubric details
+  - Missing tool outputs
+  
+  If the submission is on-topic, detailed, intelligible, and attempts to respond meaningfully to the task, you must grade it manually.
+  
+  Only escalate if the submission itself is incoherent, nonsensical, off-topic, or so vague that no meaningful evaluation is possible — even using your best human-like judgment.
+  
+  Otherwise, proceed confidently with grading using the rubric, your judgment, and available tools.
+  
+  If and only if human review is truly required, respond with:
+  
+  {
+	"grade": "HUMAN_REVIEW_REQUIRED",
+	"reasoning": "<Explain why. Do not mention tool errors unless an actual tool failure occurred and it prevents understanding of the submission.>"
+  }
+  
+  EXAMPLES:
+  
+  ✅ Human review IS appropriate:
+  - Task: “Write about your favorite historical figure.”
+  - Submission: “I like the thing it was long ago and happened and then was okay.”
+  - Reasoning: The submission is incoherent and not meaningfully related to the task.
+  
+  ✅ Human review IS appropriate:
+  - Task: “Name a dog.”
+  - Submission: “I like the colour red.”
+  - Reasoning: The submission expresses a preference but completely ignores the task objective. It cannot be evaluated meaningfully and reflects a misunderstanding.
+  
+  ✅ Human review IS appropriate:
+  - Task: “Name a dog.”
+  - Submission: “A”
+  - Reasoning: The submission is technically on-topic but so minimal that it cannot be evaluated or scored in a meaningful way.
+  
+  ❌ Human review is NOT appropriate:
+  - Task: “Write a persuasive paragraph about school uniforms.”
+  - Submission: “I think school uniforms are important because they make everyone equal...”
+  - Reasoning: The task and submission are both clear and evaluatable without a rubric.
+  
+  Respond ONLY with the JSON object, with no preamble or explanation.`;
 
-IMPORTANT:
+	return buildBaseGradingPrompt({
-You must not claim that tools are unavailable or broken unless a tool call explicitly fails with an error. If a tool call fails, you must describe the failure clearly, then continue grading using your best judgment. You must never escalate to human review just because a tool call failed. Failed, missing, or partial tool results are not valid reasons to avoid grading.
+		task: safeTask,
-
+		submission: safeSubmission,
-If a submission is too short, vague, off-topic, or demonstrates a complete misunderstanding of the task — even if it is grammatically correct — you must request human review.
+		steps,
-
+		responseFormat,
-Only escalate if the task or submission is truly impossible to evaluate meaningfully, even with your expertise and the tools provided.
+		extraSections
-
+	});
-Otherwise, proceed confidently with grading using the rubric, your judgment, and available tools.
-
-If and only if human review is truly required, respond with:
-
-{
-  "grade": "HUMAN_REVIEW_REQUIRED",
-  "reasoning": "<Explain why. Do not mention tool errors unless an actual tool failure occurred.>"
-}
-
-EXAMPLES:
-
-✅ Human review IS appropriate:
-- Task: “Write about your favorite historical figure.”
-- Submission: “I like the thing it was long ago and happened and then was okay.”
-- Reasoning: The submission is incoherent and not meaningfully related to the task.
-
-✅ Human review IS appropriate:
-- Task: “Name a dog.”
-- Submission: “I like the colour red.”
-- Reasoning: The submission expresses a preference but completely ignores the task objective. It cannot be evaluated meaningfully and reflects a misunderstanding.
-
-✅ Human review IS appropriate:
-- Task: “Name a dog.”
-- Submission: “A”
-- Reasoning: The submission is technically on-topic but so minimal that it cannot be evaluated or scored in a meaningful way.
-
-❌ Human review is NOT appropriate:
-- Task: “Write a persuasive paragraph about school uniforms.”
-- Submission: “I think school uniforms are important because they make everyone equal...”
-- Reasoning: The task and submission are both clear and evaluatable without a rubric.
-
-Respond ONLY with the JSON object, with no preamble or explanation.`;
-  return buildBaseGradingPrompt({
-    task: safeTask,
-    submission: safeSubmission,
-    steps,
-    responseFormat,
-    extraSections
-  });
 }
 
 function fallbackGrade(submission: string, task: string): OpenAIResponse {