3. field_llm_judge / create_doc
{
"type": "field_llm_judge",
"tool": "create_doc",
"action_id": "create_q1_reconciliation",
"path": "args",
"criterion": "The title clearly identifies the document as a Q1 2024 lifecycle contractor reconciliation worksheet or equivalent.\n\nThe worksheet states that the governing Q1 Growth envelope was $24,000, the lifecycle QA and operations contractor engagement ran from January 15 through February 23, 2024, and Ines was the responsible reporting/engagement owner.\n\nThe worksheet identifies the permitted contractor work as active Customer.io campaign inventory, experiment-setup hygiene, suppression checks, and repeatable preflight work, or faithful equivalent descriptions of all four categories.\n\nThe worksheet separately lists invoice HS-204 for $2,600 of Helio Start paid-acquisition setup and clearly says it must not be charged to the Q1 2024 lifecycle contractor envelope. The exclusion must be framed as applying to this Q1 2024 envelope, not as a claim that paid acquisition was permanently or universally unfunded.\n\nThe worksheet lists LC-101 for $8,400 and LC-102 for $9,100 as supported paid charges within the permitted lifecycle scope and totals those supported charges as exactly $17,500.\n\nThe worksheet records that invoice and receipt evidence for February 17–23 remains outstanding or unresolved.\n\nThe worksheet calculates the $6,500 difference between the $24,000 envelope and $17,500 of supported in-scope charges, but labels that difference as not yet evidenced or otherwise unresolved rather than claiming the full envelope was spent or that $6,500 was definitively unused. It must keep the excluded $2,600 acquisition invoice out of the supported lifecycle total.",
"check_id": "riley_121_02"
}Complete grading specification
{
"type": "tool_trace",
"config": {
"check_version": 2,
"today": "2028-01-01",
"semantic_judge_version": 2,
"assertions": [
{
"type": "field_equals",
"tool": "create_doc",
"action_id": "create_q1_reconciliation",
"path": "result.ok",
"value": true,
"check_id": "riley_121_00"
},
{
"type": "field_equals",
"tool": "list_inbox",
"action_id": "review_reconciliation_inbox",
"path": "args.folder",
"value": "inbox",
"check_id": "riley_121_01"
},
{
"type": "field_llm_judge",
"tool": "create_doc",
"action_id": "create_q1_reconciliation",
"path": "args",
"criterion": "The title clearly identifies the document as a Q1 2024 lifecycle contractor reconciliation worksheet or equivalent.\n\nThe worksheet states that the governing Q1 Growth envelope was $24,000, the lifecycle QA and operations contractor engagement ran from January 15 through February 23, 2024, and Ines was the responsible reporting/engagement owner.\n\nThe worksheet identifies the permitted contractor work as active Customer.io campaign inventory, experiment-setup hygiene, suppression checks, and repeatable preflight work, or faithful equivalent descriptions of all four categories.\n\nThe worksheet separately lists invoice HS-204 for $2,600 of Helio Start paid-acquisition setup and clearly says it must not be charged to the Q1 2024 lifecycle contractor envelope. The exclusion must be framed as applying to this Q1 2024 envelope, not as a claim that paid acquisition was permanently or universally unfunded.\n\nThe worksheet lists LC-101 for $8,400 and LC-102 for $9,100 as supported paid charges within the permitted lifecycle scope and totals those supported charges as exactly $17,500.\n\nThe worksheet records that invoice and receipt evidence for February 17–23 remains outstanding or unresolved.\n\nThe worksheet calculates the $6,500 difference between the $24,000 envelope and $17,500 of supported in-scope charges, but labels that difference as not yet evidenced or otherwise unresolved rather than claiming the full envelope was spent or that $6,500 was definitively unused. It must keep the excluded $2,600 acquisition invoice out of the supported lifecycle total.",
"check_id": "riley_121_02"
}
]
}
}