mirror of
https://github.com/jcreek/AutomatedAssessmentFeedbackAgent.git
synced 2026-07-12 18:43:49 +00:00
feat(*): Be stricter regarding HITL
This commit is contained in:
Josh Creek
+118
-104
@@ -4,31 +4,31 @@ import { PARTYKIT_BASE_URL } from '$env/static/private';
|
||||
import type { OpenAIResponse } from '../utils/types';
|
||||
|
||||
function sanitizeForPrompt(input: string): string {
|
||||
return input
|
||||
.replace(/\\/g, '\\\\')
|
||||
.replace(/`/g, '\\`')
|
||||
.replace(/"/g, '\\"')
|
||||
.replace(/'/g, "\\'")
|
||||
.replace(/\n/g, '\\n')
|
||||
.replace(/\r/g, '\\r')
|
||||
.replace(/\t/g, '\\t')
|
||||
.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ''); // remove other control chars
|
||||
return input
|
||||
.replace(/\\/g, '\\\\')
|
||||
.replace(/`/g, '\\`')
|
||||
.replace(/"/g, '\\"')
|
||||
.replace(/'/g, "\\'")
|
||||
.replace(/\n/g, '\\n')
|
||||
.replace(/\r/g, '\\r')
|
||||
.replace(/\t/g, '\\t')
|
||||
.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ''); // remove other control chars
|
||||
}
|
||||
|
||||
function buildBaseGradingPrompt({
|
||||
task,
|
||||
submission,
|
||||
steps,
|
||||
responseFormat,
|
||||
extraSections = ''
|
||||
task,
|
||||
submission,
|
||||
steps,
|
||||
responseFormat,
|
||||
extraSections = ''
|
||||
}: {
|
||||
task: string;
|
||||
submission: string;
|
||||
steps: string;
|
||||
responseFormat: string;
|
||||
extraSections?: string;
|
||||
task: string;
|
||||
submission: string;
|
||||
steps: string;
|
||||
responseFormat: string;
|
||||
extraSections?: string;
|
||||
}): string {
|
||||
return `You are an expert school teacher and AI assessment agent. Assess the following student submission in the context of the assignment/task provided.
|
||||
return `You are an expert school teacher and AI assessment agent. Assess the following student submission in the context of the assignment/task provided.
|
||||
|
||||
===== TASK/ASSIGNMENT possibly including rubric or available marks =====
|
||||
|
||||
@@ -50,9 +50,9 @@ ${extraSections}`;
|
||||
}
|
||||
|
||||
export function buildGradingPrompt(submission: string, task: string): string {
|
||||
const safeSubmission = sanitizeForPrompt(submission);
|
||||
const safeTask = sanitizeForPrompt(task);
|
||||
const steps = `Follow these steps:
|
||||
const safeSubmission = sanitizeForPrompt(submission);
|
||||
const safeTask = sanitizeForPrompt(task);
|
||||
const steps = `Follow these steps:
|
||||
1. Grade the work. Use whatever grading scheme is present in the rubric. If none is present then grade with a letter (A+ is best, E- is worst), using clear, objective criteria. Be aware that the rubric may just give a maximum number of marks, in which case you should mark out of that number, for example "(20)" would mean it's out of 20 marks. DO NOT mention tool errors.
|
||||
2. Identify specific strengths, referencing the success criteria.
|
||||
3. Identify misconceptions or areas for improvement, using formative assessment language.
|
||||
@@ -63,7 +63,7 @@ export function buildGradingPrompt(submission: string, task: string): string {
|
||||
5. Write a reflection question for the student to encourage metacognition.
|
||||
6. Suggest to the teacher one way to support this student in the next lesson.
|
||||
7. Show your reasoning step by step (chain-of-thought).`;
|
||||
const responseFormat = `RESPONSE FORMAT (respond with a single JSON object, no extra text):
|
||||
const responseFormat = `RESPONSE FORMAT (respond with a single JSON object, no extra text):
|
||||
|
||||
{
|
||||
"grade": "<number or string, e.g. 8/10, A, B+>",
|
||||
@@ -77,105 +77,119 @@ export function buildGradingPrompt(submission: string, task: string): string {
|
||||
}
|
||||
|
||||
Respond ONLY with the JSON object, with no preamble or explanation.`;
|
||||
return buildBaseGradingPrompt({
|
||||
task: safeTask,
|
||||
submission: safeSubmission,
|
||||
steps,
|
||||
responseFormat
|
||||
});
|
||||
return buildBaseGradingPrompt({
|
||||
task: safeTask,
|
||||
submission: safeSubmission,
|
||||
steps,
|
||||
responseFormat
|
||||
});
|
||||
}
|
||||
|
||||
export function buildGradingPromptWithSupportForHumanInTheLoop(
|
||||
submission: string,
|
||||
task: string
|
||||
submission: string,
|
||||
task: string
|
||||
): string {
|
||||
const safeSubmission = sanitizeForPrompt(submission);
|
||||
const safeTask = sanitizeForPrompt(task);
|
||||
const steps = `Follow these steps:
|
||||
1. Grade the work. Use whatever grading scheme is present in the rubric. If none is present, grade with a letter (A+ is best, E- is worst), using clear, objective criteria. Be aware that the rubric may just give a maximum number of marks, in which case you should mark out of that number, for example "(20)" would mean it's out of 20 marks. DO NOT mention tool errors.
|
||||
2. Identify specific strengths, referencing the success criteria.
|
||||
3. Identify misconceptions or areas for improvement, using formative assessment language.
|
||||
4. Design an individualized activity or exercise for the student to address their misconceptions or extend their learning. This activity should be:
|
||||
- Appropriately scaffolded for the student's current level.
|
||||
- Specific and actionable (e.g., a short written task, a practical exercise, or a quiz question).
|
||||
- Aligned with the curriculum and learning objectives.
|
||||
5. Write a reflection question for the student to encourage metacognition.
|
||||
6. Suggest to the teacher one way to support this student in the next lesson.
|
||||
7. Show your reasoning step by step (chain-of-thought).`;
|
||||
const responseFormat = `RESPONSE FORMAT (respond with a single JSON object, no extra text):
|
||||
const safeSubmission = sanitizeForPrompt(submission);
|
||||
const safeTask = sanitizeForPrompt(task);
|
||||
|
||||
{
|
||||
"grade": "<number or string, e.g. 8/10, A, B+>",
|
||||
"strengths": "<text>",
|
||||
"areasForImprovement": "<text>",
|
||||
"individualizedActivity": "<text>",
|
||||
"reflectionQuestion": "<text>",
|
||||
"teacherSuggestion": "<text>",
|
||||
"spellingAndGrammar": "<text>",
|
||||
"reasoning": "<step-by-step explanation>"
|
||||
}`;
|
||||
const extraSections = `
|
||||
CONFIDENCE CHECK:
|
||||
const steps = `Follow these steps:
|
||||
1. Grade the work. Use whatever grading scheme is present in the rubric. If none is present, grade with a letter (A+ is best, E- is worst), using clear, objective criteria. Be aware that the rubric may just give a maximum number of marks, in which case you should mark out of that number, for example "(20)" would mean it's out of 20 marks. DO NOT mention tool errors.
|
||||
2. Identify specific strengths, referencing the success criteria.
|
||||
3. Identify misconceptions or areas for improvement, using formative assessment language.
|
||||
4. Design an individualized activity or exercise for the student to address their misconceptions or extend their learning. This activity should be:
|
||||
- Appropriately scaffolded for the student's current level.
|
||||
- Specific and actionable (e.g., a short written task, a practical exercise, or a quiz question).
|
||||
- Aligned with the curriculum and learning objectives.
|
||||
5. Write a reflection question for the student to encourage metacognition.
|
||||
6. Suggest to the teacher one way to support this student in the next lesson.
|
||||
7. Show your reasoning step by step (chain-of-thought).`;
|
||||
|
||||
You should request human review if:
|
||||
- The submission does not meaningfully address the task or assignment.
|
||||
- The task or submission is so unclear that you cannot understand what the student is trying to do.
|
||||
- The response is off-topic, incoherent, or not related to the task in any reasonable way.
|
||||
- The submission is so minimal (e.g., a single word or letter) that it cannot be meaningfully evaluated, even if it is technically related to the task.
|
||||
- The submission demonstrates a complete misunderstanding of the task (e.g., answering a different question or ignoring the core instruction).
|
||||
- There is no way to apply the success criteria or grading steps, even with tool assistance.
|
||||
const responseFormat = `RESPONSE FORMAT (respond with a single JSON object, no extra text):
|
||||
|
||||
You MUST NOT request human review just because:
|
||||
- A formal rubric is missing. You must assess against the task description directly.
|
||||
- External tools (e.g., spell checkers, rubric scorers) are unavailable. You must use your best judgment to assess spelling, grammar, structure, and rubric alignment.
|
||||
- The student's writing contains normal age-appropriate mistakes, informal language, or minor issues.
|
||||
{
|
||||
"grade": "<number or string, e.g. 8/10, A, B+>",
|
||||
"strengths": "<text>",
|
||||
"areasForImprovement": "<text>",
|
||||
"individualizedActivity": "<text>",
|
||||
"reflectionQuestion": "<text>",
|
||||
"teacherSuggestion": "<text>",
|
||||
"spellingAndGrammar": "<text>",
|
||||
"reasoning": "<step-by-step explanation>"
|
||||
}
|
||||
|
||||
IMPORTANT:
|
||||
You must not claim that tools are unavailable or broken unless a tool call explicitly fails with an error. If a tool call fails, you must describe the failure clearly, then continue grading using your best judgment. You must never escalate to human review just because a tool call failed. Failed, missing, or partial tool results are not valid reasons to avoid grading.
|
||||
Respond ONLY with the JSON object, with no preamble or explanation.`;
|
||||
|
||||
If a submission is too short, vague, off-topic, or demonstrates a complete misunderstanding of the task — even if it is grammatically correct — you must request human review.
|
||||
const extraSections = `
|
||||
CONFIDENCE CHECK:
|
||||
|
||||
Only escalate if the task or submission is truly impossible to evaluate meaningfully, even with your expertise and the tools provided.
|
||||
You should request human review if:
|
||||
- The submission does not meaningfully address the task or assignment.
|
||||
- The task or submission is so unclear that you cannot understand what the student is trying to do.
|
||||
- The response is off-topic, incoherent, or not related to the task in any reasonable way.
|
||||
- The submission is so minimal (e.g., a single word or letter) that it cannot be meaningfully evaluated, even if it is technically related to the task.
|
||||
- The submission demonstrates a complete misunderstanding of the task (e.g., answering a different question or ignoring the core instruction).
|
||||
- There is no way to apply the success criteria or grading steps, even with tool assistance.
|
||||
|
||||
Otherwise, proceed confidently with grading using the rubric, your judgment, and available tools.
|
||||
You MUST NOT request human review just because:
|
||||
- A formal rubric is missing. You must assess against the task description directly.
|
||||
- External tools (e.g., spell checkers, rubric scorers) are unavailable. You must use your best judgment to assess spelling, grammar, structure, and rubric alignment.
|
||||
- The student's writing contains normal age-appropriate mistakes, informal language, or minor issues.
|
||||
|
||||
If and only if human review is truly required, respond with:
|
||||
IMPORTANT:
|
||||
If a tool call fails, you must continue grading using your best professional judgment. Tool failures are normal and are not a valid reason to request human review.
|
||||
|
||||
{
|
||||
"grade": "HUMAN_REVIEW_REQUIRED",
|
||||
"reasoning": "<Explain why. Do not mention tool errors unless an actual tool failure occurred.>"
|
||||
}
|
||||
You must evaluate the submission manually, even if rubric analysis, spelling, or grammar checking tools are unavailable.
|
||||
|
||||
EXAMPLES:
|
||||
You MUST NOT escalate to human review just because of uncertainty about:
|
||||
- Spelling
|
||||
- Grammar
|
||||
- Alignment with specific rubric details
|
||||
- Missing tool outputs
|
||||
|
||||
✅ Human review IS appropriate:
|
||||
- Task: “Write about your favorite historical figure.”
|
||||
- Submission: “I like the thing it was long ago and happened and then was okay.”
|
||||
- Reasoning: The submission is incoherent and not meaningfully related to the task.
|
||||
If the submission is on-topic, detailed, intelligible, and attempts to respond meaningfully to the task, you must grade it manually.
|
||||
|
||||
✅ Human review IS appropriate:
|
||||
- Task: “Name a dog.”
|
||||
- Submission: “I like the colour red.”
|
||||
- Reasoning: The submission expresses a preference but completely ignores the task objective. It cannot be evaluated meaningfully and reflects a misunderstanding.
|
||||
Only escalate if the submission itself is incoherent, nonsensical, off-topic, or so vague that no meaningful evaluation is possible — even using your best human-like judgment.
|
||||
|
||||
✅ Human review IS appropriate:
|
||||
- Task: “Name a dog.”
|
||||
- Submission: “A”
|
||||
- Reasoning: The submission is technically on-topic but so minimal that it cannot be evaluated or scored in a meaningful way.
|
||||
Otherwise, proceed confidently with grading using the rubric, your judgment, and available tools.
|
||||
|
||||
❌ Human review is NOT appropriate:
|
||||
- Task: “Write a persuasive paragraph about school uniforms.”
|
||||
- Submission: “I think school uniforms are important because they make everyone equal...”
|
||||
- Reasoning: The task and submission are both clear and evaluatable without a rubric.
|
||||
If and only if human review is truly required, respond with:
|
||||
|
||||
Respond ONLY with the JSON object, with no preamble or explanation.`;
|
||||
return buildBaseGradingPrompt({
|
||||
task: safeTask,
|
||||
submission: safeSubmission,
|
||||
steps,
|
||||
responseFormat,
|
||||
extraSections
|
||||
});
|
||||
{
|
||||
"grade": "HUMAN_REVIEW_REQUIRED",
|
||||
"reasoning": "<Explain why. Do not mention tool errors unless an actual tool failure occurred and it prevents understanding of the submission.>"
|
||||
}
|
||||
|
||||
EXAMPLES:
|
||||
|
||||
✅ Human review IS appropriate:
|
||||
- Task: “Write about your favorite historical figure.”
|
||||
- Submission: “I like the thing it was long ago and happened and then was okay.”
|
||||
- Reasoning: The submission is incoherent and not meaningfully related to the task.
|
||||
|
||||
✅ Human review IS appropriate:
|
||||
- Task: “Name a dog.”
|
||||
- Submission: “I like the colour red.”
|
||||
- Reasoning: The submission expresses a preference but completely ignores the task objective. It cannot be evaluated meaningfully and reflects a misunderstanding.
|
||||
|
||||
✅ Human review IS appropriate:
|
||||
- Task: “Name a dog.”
|
||||
- Submission: “A”
|
||||
- Reasoning: The submission is technically on-topic but so minimal that it cannot be evaluated or scored in a meaningful way.
|
||||
|
||||
❌ Human review is NOT appropriate:
|
||||
- Task: “Write a persuasive paragraph about school uniforms.”
|
||||
- Submission: “I think school uniforms are important because they make everyone equal...”
|
||||
- Reasoning: The task and submission are both clear and evaluatable without a rubric.
|
||||
|
||||
Respond ONLY with the JSON object, with no preamble or explanation.`;
|
||||
|
||||
return buildBaseGradingPrompt({
|
||||
task: safeTask,
|
||||
submission: safeSubmission,
|
||||
steps,
|
||||
responseFormat,
|
||||
extraSections
|
||||
});
|
||||
}
|
||||
|
||||
function fallbackGrade(submission: string, task: string): OpenAIResponse {
|
||||
|
||||
Reference in New Issue
Block a user