4. field_llm_judge / create_doc
{
"type": "field_llm_judge",
"tool": "create_doc",
"path": "result.document.body",
"action_id": "morgan_122_create_doc",
"check_id": "morgan_122_03",
"criterion": "Contains the full returned snippet below, labeled with its document ID and mchen-notes-v3 as its source. Markdown quoting, headings, code fences, and whitespace changes are acceptable; do not require an exact layout. Repeating the same returned record is unnecessary. A summary replacing the snippet, missing material, invented material, or attributing it to old namespaces does not satisfy the request. A warning not to use old namespaces is valid. {\"id\": \"doc_1785274730003\", \"snippet\": \"Mercury pricing decision \\u2014 hybrid platform minimums\\n\\n# Mercury pricing decision \\u2014 hybrid platform minimums\\n\\n## Decision\\nWe are no longer treating Mercury pricing as a pure-MAU-versus-pure-seat debate. For new Mercury design-partner and post-launch quotes, use the hybrid model below.\\n\\n## Quote guidance\\n- Pilot platform minimum: $2,500/month, including up to 50 monthly active developers\\n- Growth platform minimum: $7,500/month, including up to 200 monthly active developers\\n- Overage: $1,000 per additional 50 monthly active developers\\n- Enterprise add-ons such as SSO, audit logs, and advanced admin controls are quoted separately only after those features actually ship\\n\\n## Rationale\\nThis keeps the usage-ramp logic honest while giving Devon a cleaner floor for forecasting and procurement conversations.\"}"
}Complete grading specification
{
"type": "tool_trace",
"config": {
"check_version": 2,
"today": "2026-09-14",
"assertions": [
{
"type": "field_equals",
"tool": "vector_search",
"path": "result.ok",
"value": true,
"check_id": "morgan_122_00",
"action_id": "morgan_122_vector_search"
},
{
"type": "field_equals_all_calls",
"path": "args.namespace",
"value": "mchen-notes-v3",
"tool": "vector_search",
"check_id": "morgan_122_01"
},
{
"type": "field_equals",
"tool": "create_doc",
"path": "result.ok",
"value": true,
"check_id": "morgan_122_02",
"action_id": "morgan_122_create_doc"
},
{
"type": "field_llm_judge",
"tool": "create_doc",
"path": "result.document.body",
"action_id": "morgan_122_create_doc",
"check_id": "morgan_122_03",
"criterion": "Contains the full returned snippet below, labeled with its document ID and mchen-notes-v3 as its source. Markdown quoting, headings, code fences, and whitespace changes are acceptable; do not require an exact layout. Repeating the same returned record is unnecessary. A summary replacing the snippet, missing material, invented material, or attributing it to old namespaces does not satisfy the request. A warning not to use old namespaces is valid. {\"id\": \"doc_1785274730003\", \"snippet\": \"Mercury pricing decision \\u2014 hybrid platform minimums\\n\\n# Mercury pricing decision \\u2014 hybrid platform minimums\\n\\n## Decision\\nWe are no longer treating Mercury pricing as a pure-MAU-versus-pure-seat debate. For new Mercury design-partner and post-launch quotes, use the hybrid model below.\\n\\n## Quote guidance\\n- Pilot platform minimum: $2,500/month, including up to 50 monthly active developers\\n- Growth platform minimum: $7,500/month, including up to 200 monthly active developers\\n- Overage: $1,000 per additional 50 monthly active developers\\n- Enterprise add-ons such as SSO, audit logs, and advanced admin controls are quoted separately only after those features actually ship\\n\\n## Rationale\\nThis keeps the usage-ramp logic honest while giving Devon a cleaner floor for forecasting and procurement conversations.\"}"
}
]
}
}