3. field_llm_judge / create_doc
{
"type": "field_llm_judge",
"tool": "create_doc",
"action_id": "create_midpoint_note",
"path": "args",
"criterion": "The document body must accurately describe the Mar 17 first-week midpoint for EXP-2023-02-pricing-page: treatment was about 0.6 percentage points below control on conversion, the confidence interval crossed zero so the estimate was treated as noise or inconclusive rather than a verdict, and pricing_page_2023q2_rev was deliberately left unchanged until the powered two-week read. Missing or contradictory treatment of any of these midpoint facts must fail.\n\nThe document body must clearly distinguish the later powered two-week result from the earlier midpoint and accurately report the archived result for EXP-2023-02-pricing-page: signup-to-paid conversion uplift was -1.8 percentage points with p=0.03. It must not present these powered values as the first-week estimate.",
"check_id": "riley_078_02"
}Complete grading specification
{
"type": "tool_trace",
"config": {
"check_version": 2,
"today": "2028-01-01",
"semantic_judge_version": 2,
"assertions": [
{
"type": "field_equals",
"tool": "create_doc",
"action_id": "create_midpoint_note",
"path": "result.ok",
"value": true,
"check_id": "riley_078_00"
},
{
"check_id": "riley_078_01",
"type": "field_equals",
"tool": "query_experiment_results",
"action_id": "query_powered_result",
"path": "args.experiment_id",
"value": "EXP-2023-02-pricing-page"
},
{
"type": "field_llm_judge",
"tool": "create_doc",
"action_id": "create_midpoint_note",
"path": "args",
"criterion": "The document body must accurately describe the Mar 17 first-week midpoint for EXP-2023-02-pricing-page: treatment was about 0.6 percentage points below control on conversion, the confidence interval crossed zero so the estimate was treated as noise or inconclusive rather than a verdict, and pricing_page_2023q2_rev was deliberately left unchanged until the powered two-week read. Missing or contradictory treatment of any of these midpoint facts must fail.\n\nThe document body must clearly distinguish the later powered two-week result from the earlier midpoint and accurately report the archived result for EXP-2023-02-pricing-page: signup-to-paid conversion uplift was -1.8 percentage points with p=0.03. It must not present these powered values as the first-week estimate.",
"check_id": "riley_078_02"
}
]
}
}