2. field_llm_judge / create_doc
{
"type": "field_llm_judge",
"tool": "create_doc",
"action_id": "create_reliability_case_study",
"path": "args",
"criterion": "The case study accurately identifies a stalled qualifying-human-admin source refresh affecting 17 pending CS auto-nudge decisions, says the alert fired 43 minutes before the freshness limit, and says Daniela restored the refresh with 18 minutes remaining. Missing or contradictory timing, count, source, or restoration information must fail.\n\nThe case study states that a fresh reevaluation occurred and accurately reports that 13 decisions proceeded while four were suppressed because current Customer Success adoption or renewal work was active. Missing, reversed, or contradictory dispositions must fail.\n\nThe case study clearly states that none of the affected decisions entered stale state, nobody manually released a decision, and neither a customer correction nor a rollback was required. Any claim that one of these events occurred must fail.\n\nThe document title clearly identifies this as an incident case study about the first live pre-breach source-age alert. A generic, irrelevant, or misleading title must fail.",
"check_id": "riley_161_01"
}Complete grading specification
{
"type": "tool_trace",
"config": {
"check_version": 2,
"today": "2028-01-01",
"semantic_judge_version": 2,
"assertions": [
{
"type": "field_equals",
"tool": "create_doc",
"action_id": "create_reliability_case_study",
"path": "result.ok",
"value": true,
"check_id": "riley_161_00"
},
{
"type": "field_llm_judge",
"tool": "create_doc",
"action_id": "create_reliability_case_study",
"path": "args",
"criterion": "The case study accurately identifies a stalled qualifying-human-admin source refresh affecting 17 pending CS auto-nudge decisions, says the alert fired 43 minutes before the freshness limit, and says Daniela restored the refresh with 18 minutes remaining. Missing or contradictory timing, count, source, or restoration information must fail.\n\nThe case study states that a fresh reevaluation occurred and accurately reports that 13 decisions proceeded while four were suppressed because current Customer Success adoption or renewal work was active. Missing, reversed, or contradictory dispositions must fail.\n\nThe case study clearly states that none of the affected decisions entered stale state, nobody manually released a decision, and neither a customer correction nor a rollback was required. Any claim that one of these events occurred must fail.\n\nThe document title clearly identifies this as an incident case study about the first live pre-breach source-age alert. A generic, irrelevant, or misleading title must fail.",
"check_id": "riley_161_01"
}
]
}
}