{ "id": "ranking-evaluate", "name": "Supplied ranking evaluation", "description": "Score ranked document IDs against caller-supplied graded judgments for up to 50 queries: precision/recall, reciprocal rank, truncated average precision and NDCG at k. Explicit gain, unjudged and zero-denominator conventions; evidence lists ranked judgments and missed relevant IDs. No search or relevance inference.", "category": "data-assurance", "priceUsd": "0.01", "pricingStatus": "proposed-unverified", "pricing": { "unit": "one successful operation call", "proposedNominalUsd": "0.01", "sixDecimalTokenBaseUnits": "10000", "subscription": false, "includesPayerWalletOrNetworkFees": false, "liveQuoteVerified": false, "condition": "Actual SDK challenge is authoritative only within the caller's explicit authorization; configured six-decimal token peg is an operator assertion, not a conversion guarantee." }, "intents": [ "evaluate retrieval ranking ndcg map mrr precision recall at k", "score search results against supplied relevance judgments", "compare RAG retrieval quality using judged document ids" ], "whenToUse": [ "evaluate retrieval ranking ndcg map mrr precision recall at k", "score search results against supplied relevance judgments", "compare RAG retrieval quality using judged document ids" ], "whenNotToUse": [ "Search the web or determine semantic document relevance", "Claim benchmark completeness from partial judgments" ], "capabilities": [ "graded ranking metrics", "explicit unjudged handling", "linear or exponential gain", "query-level metric evidence" ], "related": [ "classification-evaluate", "span-evaluate" ], "limits": { "inputBytes": 100000, "outputBytes": 400000, "jsonNodes": 12000, "depth": 20, "evidence": 200, "keyWorkBytes": 3000000, "matchingSteps": 2000000, "queries": 50, "retrievedPerQuery": 100, "judgmentsPerQuery": 100, "maximumK": 100 }, "errorCodes": [ "INPUT_LIMIT", "OUTPUT_LIMIT", "COMPLEXITY_LIMIT", "UNSAFE_KEY", "INVALID_NUMBER", "INVALID_POINTER", "DUPLICATE_ID", "DUPLICATE_FIELD", "INVALID_RULE", "KEY_WORK_LIMIT", "LABEL_LIMIT", "UNJUDGED_DOCUMENT", "INVALID_SPAN", "DUPLICATE_SPAN", "INVALID_UNICODE", "MATCHING_LIMIT", "INVALID_CADENCE", "INVALID_INTERVAL", "NUMERIC_OVERFLOW" ], "requirementsProfile": { "format": "declared-requirements-v1", "facts": { "execution.suppliedCode": false, "execution.remoteMutation": false, "execution.llmInference": false, "execution.paidMcp": false, "verification.semanticTruth": false, "verification.sourceAuthenticity": false, "verification.liveVulnerabilities": false, "numbers.arbitraryPrecisionJson": false, "numbers.model": "ieee754-binary64", "privacy.requestBodyPersisted": false, "execution.deterministic": true, "execution.networkAccess": false, "privacy.resultBodyPersisted": false, "payment.x402": true, "payment.mpp": true, "operation.id": "ranking-evaluate", "operation.category": "data-assurance", "limit.httpRequestBytes": 131072, "limit.requestBytes": 100000, "limit.responseBytes": 524288, "limit.resultBytes": 400000, "limit.jsonDepth": 20, "limit.jsonNodes": 12000, "input.queries.maxItems": 50, "input.queries.minItems": 1, "input.k.maximum": 100, "input.k.minimum": 1, "input.relevantAtLeast.maximum": 10 }, "unknownPolicy": "Undeclared requirements are unknown, never compatible. Matching declared facts does not establish semantic fit or input validity.", "preflight": "/preflight" }, "documentation": "/reference/tools/ranking-evaluate.md", "serviceContract": "/reference/services/ranking-evaluate.json", "errors": [ { "status": 400, "meaning": "Malformed JSON, missing/invalid idempotency key, or payment identifier mismatch", "retry": "Correct the request before payment" }, { "status": 402, "meaning": "Payment challenge or rejected payment", "retry": "Use official protocol SDK; inspect payment outcome before another payment" }, { "status": 409, "meaning": "Idempotency conflict, duplicate proof, or PAYMENT_UNCERTAIN", "retry": "Keep original key, body, and proof; reconcile uncertainty with operator; never blindly repay" }, { "status": 413, "meaning": "Input or generated output too large", "retry": "Reduce input; no payment attempted for validation failure" }, { "status": 415, "meaning": "Unsupported media type or compression", "retry": "Send uncompressed application/json" }, { "status": 422, "meaning": "Schema or service-specific semantic validation failure", "retry": "Correct input using returned error code; no payment attempted" }, { "status": 429, "meaning": "Request/payment-attempt rate exceeded", "retry": "Wait for rate limit window; preserve existing payment identity" }, { "status": 503, "meaning": "Payment configuration/provider/state unavailable, or live DNS preparation failed before settlement", "retry": "Check readiness; DNS preparation failures may retry the identical key/body/credential only; uncertainty requires reconciliation" } ], "numericPrecision": "JavaScript IEEE-754 numbers; use strings for large integer IDs/exact decimals where the schema accepts strings. No lossless numeric parsing.", "paymentWorkflow": { "discoveryOnly": false, "supportedProtocols": [ "x402", "mpp" ], "x402": { "credentialHeader": "PAYMENT-SIGNATURE", "challengeHeader": "PAYMENT-REQUIRED", "receiptHeader": "PAYMENT-RESPONSE", "version": 2, "scheme": "exact", "paymentIdentifier": "payment-identifier extension MUST equal the HTTP Idempotency-Key", "sdk": "@x402/core with @x402/evm" }, "mpp": { "supported": true, "credentialHeader": "Authorization", "challengeHeader": "WWW-Authenticate", "receiptHeader": "Payment-Receipt", "method": "tempo", "intent": "charge", "sdk": "mppx", "tokenDecimals": 6 }, "steps": [ "Check configured readiness and the exact service schema", "Generate a fresh random Idempotency-Key for this operation; never use a discovery probe fixture for purchases", "Send valid input without a credential to obtain the official protocol challenge", "Use the official SDK and authorized wallet to fulfill the challenge", "Retry only with identical key, body, protocol and credential", "On PAYMENT_UNCERTAIN stop and request operator reconciliation; never blindly pay again" ], "versionedRetries": "Request fingerprint includes service release version. Retries across a version upgrade can conflict; coordinate upgrades outside the 24-hour replay window and reconcile pending attempts.", "docs": "/llms.txt" }, "method": "POST", "paths": { "x402": "/v1/x402/ranking-evaluate", "mpp": "/v1/mpp/ranking-evaluate" }, "inputSchema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { "queries": { "minItems": 1, "maxItems": 50, "type": "array", "items": { "type": "object", "properties": { "id": { "type": "string", "minLength": 1, "maxLength": 80 }, "retrieved": { "maxItems": 100, "type": "array", "items": { "type": "string", "minLength": 1, "maxLength": 80 } }, "judgments": { "maxItems": 100, "type": "array", "items": { "type": "object", "properties": { "id": { "type": "string", "minLength": 1, "maxLength": 80 }, "grade": { "type": "number", "minimum": 0, "maximum": 10 } }, "required": [ "id", "grade" ], "additionalProperties": false } } }, "required": [ "id", "retrieved", "judgments" ], "additionalProperties": false } }, "k": { "default": 10, "type": "integer", "minimum": 1, "maximum": 100 }, "relevantAtLeast": { "default": 1, "type": "number", "exclusiveMinimum": 0, "maximum": 10 }, "gain": { "default": "linear", "type": "string", "enum": [ "linear", "exponential" ] }, "unjudged": { "default": "error", "type": "string", "enum": [ "error", "nonrelevant" ] } }, "required": [ "queries" ], "additionalProperties": false }, "outputSchema": { "type": "object", "required": [ "operation", "version", "result", "provenance" ], "properties": { "operation": { "const": "ranking-evaluate", "type": "string" }, "version": { "const": "0.29.0", "type": "string" }, "result": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { "profile": { "type": "string", "const": "supplied-ranking-evaluation-v1" }, "k": { "type": "integer", "minimum": 0, "maximum": 9007199254740991 }, "queryCount": { "type": "integer", "minimum": 0, "maximum": 9007199254740991 }, "queries": { "type": "array", "items": { "type": "object", "properties": { "precisionAtK": { "type": "number" }, "recallAtK": { "type": [ "number", "null" ] }, "reciprocalRankAtK": { "type": [ "number", "null" ] }, "averagePrecisionAtK": { "type": [ "number", "null" ] }, "ndcgAtK": { "type": [ "number", "null" ] }, "id": { "type": "string" }, "relevantJudgmentCount": { "type": "integer", "minimum": 0, "maximum": 9007199254740991 }, "retrievedAtKCount": { "type": "integer", "minimum": 0, "maximum": 9007199254740991 }, "hitsAtK": { "type": "integer", "minimum": 0, "maximum": 9007199254740991 }, "unjudgedAtKCount": { "type": "integer", "minimum": 0, "maximum": 9007199254740991 }, "dcgAtK": { "type": "number" }, "idealDcgAtK": { "type": "number" }, "missedRelevantIds": { "type": "array", "items": { "type": "string" } }, "ranking": { "type": "array", "items": { "type": "object", "properties": { "rank": { "type": "integer", "minimum": 0, "maximum": 9007199254740991 }, "id": { "type": "string" }, "grade": { "type": [ "number", "null" ] }, "relevant": { "type": "boolean" }, "judged": { "type": "boolean" } }, "required": [ "rank", "id", "grade", "relevant", "judged" ], "additionalProperties": false } } }, "required": [ "precisionAtK", "recallAtK", "reciprocalRankAtK", "averagePrecisionAtK", "ndcgAtK", "id", "relevantJudgmentCount", "retrievedAtKCount", "hitsAtK", "unjudgedAtKCount", "dcgAtK", "idealDcgAtK", "missedRelevantIds", "ranking" ], "additionalProperties": false } }, "macro": { "type": "object", "properties": { "precisionAtK": { "type": [ "number", "null" ] }, "recallAtK": { "type": [ "number", "null" ] }, "reciprocalRankAtK": { "type": [ "number", "null" ] }, "averagePrecisionAtK": { "type": [ "number", "null" ] }, "ndcgAtK": { "type": [ "number", "null" ] } }, "required": [ "precisionAtK", "recallAtK", "reciprocalRankAtK", "averagePrecisionAtK", "ndcgAtK" ], "additionalProperties": false }, "definedQueryCounts": { "type": "object", "properties": { "precisionAtK": { "type": "integer", "minimum": 0, "maximum": 9007199254740991 }, "recallAtK": { "type": "integer", "minimum": 0, "maximum": 9007199254740991 }, "reciprocalRankAtK": { "type": "integer", "minimum": 0, "maximum": 9007199254740991 }, "averagePrecisionAtK": { "type": "integer", "minimum": 0, "maximum": 9007199254740991 }, "ndcgAtK": { "type": "integer", "minimum": 0, "maximum": 9007199254740991 } }, "required": [ "precisionAtK", "recallAtK", "reciprocalRankAtK", "averagePrecisionAtK", "ndcgAtK" ], "additionalProperties": false }, "conventions": { "type": "array", "items": { "type": "string" } } }, "required": [ "profile", "k", "queryCount", "queries", "macro", "definedQueryCounts", "conventions" ], "additionalProperties": false }, "provenance": { "type": "object", "required": [ "inputSha256", "outputSha256", "deterministic", "externalRequests" ], "properties": { "inputSha256": { "type": "string", "pattern": "^[a-f0-9]{64}$" }, "outputSha256": { "type": "string", "pattern": "^[a-f0-9]{64}$" }, "deterministic": { "const": true }, "externalRequests": { "const": 0 } } } }, "additionalProperties": false }, "exampleInput": { "queries": [ { "id": "q1", "retrieved": [ "b", "a", "c" ], "judgments": [ { "id": "a", "grade": 3 }, { "id": "b", "grade": 1 }, { "id": "c", "grade": 0 }, { "id": "d", "grade": 2 } ] } ], "k": 3 }, "exampleResponse": { "operation": "ranking-evaluate", "version": "0.29.0", "result": { "profile": "supplied-ranking-evaluation-v1", "k": 3, "queryCount": 1, "queries": [ { "precisionAtK": 0.6666666666666666, "recallAtK": 0.6666666666666666, "reciprocalRankAtK": 1, "averagePrecisionAtK": 0.6666666666666666, "ndcgAtK": 0.6074915180456525, "id": "q1", "relevantJudgmentCount": 3, "retrievedAtKCount": 3, "hitsAtK": 2, "unjudgedAtKCount": 0, "dcgAtK": 2.8927892607143724, "idealDcgAtK": 4.7618595071429155, "missedRelevantIds": [ "d" ], "ranking": [ { "rank": 1, "id": "b", "grade": 1, "relevant": true, "judged": true }, { "rank": 2, "id": "a", "grade": 3, "relevant": true, "judged": true }, { "rank": 3, "id": "c", "grade": 0, "relevant": false, "judged": true } ] } ], "macro": { "precisionAtK": 0.6666666666666666, "recallAtK": 0.6666666666666666, "reciprocalRankAtK": 1, "averagePrecisionAtK": 0.6666666666666666, "ndcgAtK": 0.6074915180456525 }, "definedQueryCounts": { "precisionAtK": 1, "recallAtK": 1, "reciprocalRankAtK": 1, "averagePrecisionAtK": 1, "ndcgAtK": 1 }, "conventions": [ "Precision denominator is k even when fewer results are supplied. Recall and truncated AP denominators use all supplied relevant judgments; reciprocal rank is truncated at k.", "DCG uses linear gain and log2(rank + 1) discount. Ideal DCG uses all supplied judgments sorted by grade, truncated at k.", "No-relevant-judgment recall, reciprocal rank and AP are null; zero-ideal-gain NDCG is null. Macro averages exclude nulls and report denominators.", "Evaluation uses supplied judgments only. It does not retrieve documents, judge semantic relevance, or establish complete relevance coverage." ] }, "provenance": { "inputSha256": "a830c056a36bdf25d24ce65b0f72f06975f4ac98f7a9cb4aa70c79f3c3fd8b4f", "outputSha256": "948595d1d37750d1e18558475116ee9dd440ddde396ae7edc49b7d9f909ac4d0", "deterministic": true, "externalRequests": 0 } }, "requiredHeaders": { "Content-Type": "application/json", "Idempotency-Key": "random 16–128 character operation identifier" }, "fixedExample": "/reference/examples/ranking-evaluate.json", "execution": { "deterministic": true, "externalRequests": 0, "maxExternalRequests": 0, "resultSnapshotPersisted": false, "fixedExampleIsIllustrativeSnapshot": false, "requiresPayment": true, "supportsMcpExecution": false }, "releaseSnapshot": { "format": "static-release-reference-v1", "sourceVersion": "0.29.0", "sourceRegistrySha256": "a2b5ec09bf6c38b9d2879d746a4fded374f5928b445377b0270ef6aa8e6cac65", "generatedAt": "2026-10-06T15:38:00Z", "releaseAcceptance": "not-verified-by-generator", "runtimeReadiness": "not-evaluated", "livePaymentsVerified": false, "indexingVerified": false, "apiOrigin": null, "notice": "Build-time release reference. Runtime readiness, deployment, payment settlement and external indexing are not evaluated here. Prices are proposed; this file cannot authorize execution or payment." } }