{
  "id": "span-evaluate",
  "name": "Annotation span evaluation",
  "description": "Evaluate supplied labeled text spans with exact boundaries or thresholded intersection-over-union. Bounded deterministic maximum-cardinality one-to-one matching returns precision/recall/F1, matches and unmatched IDs. UTF-16 half-open offsets are validated; labels and text meaning are not judged.",
  "category": "data-assurance",
  "priceUsd": "0.01",
  "pricingStatus": "proposed-unverified",
  "pricing": {
    "unit": "one successful operation call",
    "proposedNominalUsd": "0.01",
    "sixDecimalTokenBaseUnits": "10000",
    "subscription": false,
    "includesPayerWalletOrNetworkFees": false,
    "liveQuoteVerified": false,
    "condition": "Actual SDK challenge is authoritative only within the caller's explicit authorization; configured six-decimal token peg is an operator assertion, not a conversion guarantee."
  },
  "intents": [
    "evaluate named entity extraction annotation spans",
    "match predicted and reference text spans using exact offsets or iou",
    "calculate one-to-one span precision recall and unmatched evidence"
  ],
  "whenToUse": [
    "evaluate named entity extraction annotation spans",
    "match predicted and reference text spans using exact offsets or iou",
    "calculate one-to-one span precision recall and unmatched evidence"
  ],
  "whenNotToUse": [
    "Semantic factuality or entailment checking",
    "Tokenization byte offsets or globally maximum total IoU matching"
  ],
  "capabilities": [
    "exact or overlap span evaluation",
    "maximum-cardinality matching",
    "surrogate-safe UTF-16 boundaries",
    "per-label metrics and unmatched evidence"
  ],
  "related": [
    "classification-evaluate",
    "evidence-quote-check"
  ],
  "limits": {
    "inputBytes": 100000,
    "outputBytes": 400000,
    "jsonNodes": 12000,
    "depth": 20,
    "evidence": 200,
    "keyWorkBytes": 3000000,
    "matchingSteps": 2000000,
    "textUtf16Units": 50000,
    "references": 100,
    "predictions": 100
  },
  "errorCodes": [
    "INPUT_LIMIT",
    "OUTPUT_LIMIT",
    "COMPLEXITY_LIMIT",
    "UNSAFE_KEY",
    "INVALID_NUMBER",
    "INVALID_POINTER",
    "DUPLICATE_ID",
    "DUPLICATE_FIELD",
    "INVALID_RULE",
    "KEY_WORK_LIMIT",
    "LABEL_LIMIT",
    "UNJUDGED_DOCUMENT",
    "INVALID_SPAN",
    "DUPLICATE_SPAN",
    "INVALID_UNICODE",
    "MATCHING_LIMIT",
    "INVALID_CADENCE",
    "INVALID_INTERVAL",
    "NUMERIC_OVERFLOW"
  ],
  "requirementsProfile": {
    "format": "declared-requirements-v1",
    "facts": {
      "execution.suppliedCode": false,
      "execution.remoteMutation": false,
      "execution.llmInference": false,
      "execution.paidMcp": false,
      "verification.semanticTruth": false,
      "verification.sourceAuthenticity": false,
      "verification.liveVulnerabilities": false,
      "numbers.arbitraryPrecisionJson": false,
      "numbers.model": "ieee754-binary64",
      "privacy.requestBodyPersisted": false,
      "execution.deterministic": true,
      "execution.networkAccess": false,
      "privacy.resultBodyPersisted": false,
      "payment.x402": true,
      "payment.mpp": true,
      "operation.id": "span-evaluate",
      "operation.category": "data-assurance",
      "limit.httpRequestBytes": 131072,
      "limit.requestBytes": 100000,
      "limit.responseBytes": 524288,
      "limit.resultBytes": 400000,
      "limit.jsonDepth": 20,
      "limit.jsonNodes": 12000,
      "text.offsetUnit": "utf16-code-units",
      "input.text.maxLength": 50000,
      "input.references.maxItems": 100,
      "input.predictions.maxItems": 100,
      "input.minimumIou.maximum": 1
    },
    "unknownPolicy": "Undeclared requirements are unknown, never compatible. Matching declared facts does not establish semantic fit or input validity.",
    "preflight": "/preflight"
  },
  "documentation": "/reference/tools/span-evaluate.md",
  "serviceContract": "/reference/services/span-evaluate.json",
  "errors": [
    {
      "status": 400,
      "meaning": "Malformed JSON, missing/invalid idempotency key, or payment identifier mismatch",
      "retry": "Correct the request before payment"
    },
    {
      "status": 402,
      "meaning": "Payment challenge or rejected payment",
      "retry": "Use official protocol SDK; inspect payment outcome before another payment"
    },
    {
      "status": 409,
      "meaning": "Idempotency conflict, duplicate proof, or PAYMENT_UNCERTAIN",
      "retry": "Keep original key, body, and proof; reconcile uncertainty with operator; never blindly repay"
    },
    {
      "status": 413,
      "meaning": "Input or generated output too large",
      "retry": "Reduce input; no payment attempted for validation failure"
    },
    {
      "status": 415,
      "meaning": "Unsupported media type or compression",
      "retry": "Send uncompressed application/json"
    },
    {
      "status": 422,
      "meaning": "Schema or service-specific semantic validation failure",
      "retry": "Correct input using returned error code; no payment attempted"
    },
    {
      "status": 429,
      "meaning": "Request/payment-attempt rate exceeded",
      "retry": "Wait for rate limit window; preserve existing payment identity"
    },
    {
      "status": 503,
      "meaning": "Payment configuration/provider/state unavailable, or live DNS preparation failed before settlement",
      "retry": "Check readiness; DNS preparation failures may retry the identical key/body/credential only; uncertainty requires reconciliation"
    }
  ],
  "numericPrecision": "JavaScript IEEE-754 numbers; use strings for large integer IDs/exact decimals where the schema accepts strings. No lossless numeric parsing.",
  "paymentWorkflow": {
    "discoveryOnly": false,
    "supportedProtocols": [
      "x402",
      "mpp"
    ],
    "x402": {
      "credentialHeader": "PAYMENT-SIGNATURE",
      "challengeHeader": "PAYMENT-REQUIRED",
      "receiptHeader": "PAYMENT-RESPONSE",
      "version": 2,
      "scheme": "exact",
      "paymentIdentifier": "payment-identifier extension MUST equal the HTTP Idempotency-Key",
      "sdk": "@x402/core with @x402/evm"
    },
    "mpp": {
      "supported": true,
      "credentialHeader": "Authorization",
      "challengeHeader": "WWW-Authenticate",
      "receiptHeader": "Payment-Receipt",
      "method": "tempo",
      "intent": "charge",
      "sdk": "mppx",
      "tokenDecimals": 6
    },
    "steps": [
      "Check configured readiness and the exact service schema",
      "Generate a fresh random Idempotency-Key for this operation; never use a discovery probe fixture for purchases",
      "Send valid input without a credential to obtain the official protocol challenge",
      "Use the official SDK and authorized wallet to fulfill the challenge",
      "Retry only with identical key, body, protocol and credential",
      "On PAYMENT_UNCERTAIN stop and request operator reconciliation; never blindly pay again"
    ],
    "versionedRetries": "Request fingerprint includes service release version. Retries across a version upgrade can conflict; coordinate upgrades outside the 24-hour replay window and reconcile pending attempts.",
    "docs": "/llms.txt"
  },
  "method": "POST",
  "paths": {
    "x402": "/v1/x402/span-evaluate",
    "mpp": "/v1/mpp/span-evaluate"
  },
  "inputSchema": {
    "$schema": "https://json-schema.org/draft/2020-12/schema",
    "type": "object",
    "properties": {
      "text": {
        "type": "string",
        "maxLength": 50000
      },
      "references": {
        "maxItems": 100,
        "type": "array",
        "items": {
          "type": "object",
          "properties": {
            "id": {
              "type": "string",
              "minLength": 1,
              "maxLength": 80
            },
            "label": {
              "type": "string",
              "minLength": 1,
              "maxLength": 80
            },
            "start": {
              "type": "integer",
              "minimum": 0,
              "maximum": 9007199254740991
            },
            "end": {
              "type": "integer",
              "minimum": 0,
              "maximum": 9007199254740991
            }
          },
          "required": [
            "id",
            "label",
            "start",
            "end"
          ],
          "additionalProperties": false
        }
      },
      "predictions": {
        "maxItems": 100,
        "type": "array",
        "items": {
          "type": "object",
          "properties": {
            "id": {
              "type": "string",
              "minLength": 1,
              "maxLength": 80
            },
            "label": {
              "type": "string",
              "minLength": 1,
              "maxLength": 80
            },
            "start": {
              "type": "integer",
              "minimum": 0,
              "maximum": 9007199254740991
            },
            "end": {
              "type": "integer",
              "minimum": 0,
              "maximum": 9007199254740991
            }
          },
          "required": [
            "id",
            "label",
            "start",
            "end"
          ],
          "additionalProperties": false
        }
      },
      "match": {
        "default": "exact",
        "type": "string",
        "enum": [
          "exact",
          "overlap"
        ]
      },
      "minimumIou": {
        "default": 0.5,
        "type": "number",
        "exclusiveMinimum": 0,
        "maximum": 1
      }
    },
    "required": [
      "text",
      "references",
      "predictions"
    ],
    "additionalProperties": false
  },
  "outputSchema": {
    "type": "object",
    "required": [
      "operation",
      "version",
      "result",
      "provenance"
    ],
    "properties": {
      "operation": {
        "const": "span-evaluate",
        "type": "string"
      },
      "version": {
        "const": "0.29.0",
        "type": "string"
      },
      "result": {
        "$schema": "https://json-schema.org/draft/2020-12/schema",
        "type": "object",
        "properties": {
          "profile": {
            "type": "string",
            "const": "annotation-span-evaluation-v1"
          },
          "offsetUnit": {
            "type": "string",
            "const": "utf16-code-unit"
          },
          "referenceCount": {
            "type": "integer",
            "minimum": 0,
            "maximum": 9007199254740991
          },
          "predictionCount": {
            "type": "integer",
            "minimum": 0,
            "maximum": 9007199254740991
          },
          "matchedCount": {
            "type": "integer",
            "minimum": 0,
            "maximum": 9007199254740991
          },
          "precision": {
            "type": "number"
          },
          "recall": {
            "type": "number"
          },
          "f1": {
            "type": "number"
          },
          "precisionDefined": {
            "type": "boolean"
          },
          "recallDefined": {
            "type": "boolean"
          },
          "meanMatchedIou": {
            "type": [
              "number",
              "null"
            ]
          },
          "matches": {
            "type": "array",
            "items": {
              "type": "object",
              "properties": {
                "predictionId": {
                  "type": "string"
                },
                "referenceId": {
                  "type": "string"
                },
                "label": {
                  "type": "string"
                },
                "iou": {
                  "type": "number"
                },
                "exactBoundaries": {
                  "type": "boolean"
                }
              },
              "required": [
                "predictionId",
                "referenceId",
                "label",
                "iou",
                "exactBoundaries"
              ],
              "additionalProperties": false
            }
          },
          "unmatchedPredictionIds": {
            "type": "array",
            "items": {
              "type": "string"
            }
          },
          "unmatchedReferenceIds": {
            "type": "array",
            "items": {
              "type": "string"
            }
          },
          "perLabel": {
            "type": "array",
            "items": {
              "type": "object",
              "properties": {
                "label": {
                  "type": "string"
                },
                "referenceCount": {
                  "type": "integer",
                  "minimum": 0,
                  "maximum": 9007199254740991
                },
                "predictionCount": {
                  "type": "integer",
                  "minimum": 0,
                  "maximum": 9007199254740991
                },
                "matchedCount": {
                  "type": "integer",
                  "minimum": 0,
                  "maximum": 9007199254740991
                },
                "precision": {
                  "type": "number"
                },
                "recall": {
                  "type": "number"
                },
                "f1": {
                  "type": "number"
                }
              },
              "required": [
                "label",
                "referenceCount",
                "predictionCount",
                "matchedCount",
                "precision",
                "recall",
                "f1"
              ],
              "additionalProperties": false
            }
          },
          "conventions": {
            "type": "array",
            "items": {
              "type": "string"
            }
          }
        },
        "required": [
          "profile",
          "offsetUnit",
          "referenceCount",
          "predictionCount",
          "matchedCount",
          "precision",
          "recall",
          "f1",
          "precisionDefined",
          "recallDefined",
          "meanMatchedIou",
          "matches",
          "unmatchedPredictionIds",
          "unmatchedReferenceIds",
          "perLabel",
          "conventions"
        ],
        "additionalProperties": false
      },
      "provenance": {
        "type": "object",
        "required": [
          "inputSha256",
          "outputSha256",
          "deterministic",
          "externalRequests"
        ],
        "properties": {
          "inputSha256": {
            "type": "string",
            "pattern": "^[a-f0-9]{64}$"
          },
          "outputSha256": {
            "type": "string",
            "pattern": "^[a-f0-9]{64}$"
          },
          "deterministic": {
            "const": true
          },
          "externalRequests": {
            "const": 0
          }
        }
      }
    },
    "additionalProperties": false
  },
  "exampleInput": {
    "text": "Ada builds agents in London.",
    "references": [
      {
        "id": "r1",
        "label": "person",
        "start": 0,
        "end": 3
      },
      {
        "id": "r2",
        "label": "place",
        "start": 21,
        "end": 27
      }
    ],
    "predictions": [
      {
        "id": "p1",
        "label": "person",
        "start": 0,
        "end": 3
      },
      {
        "id": "p2",
        "label": "place",
        "start": 21,
        "end": 26
      }
    ],
    "match": "overlap",
    "minimumIou": 0.8
  },
  "exampleResponse": {
    "operation": "span-evaluate",
    "version": "0.29.0",
    "result": {
      "profile": "annotation-span-evaluation-v1",
      "offsetUnit": "utf16-code-unit",
      "referenceCount": 2,
      "predictionCount": 2,
      "matchedCount": 2,
      "precision": 1,
      "recall": 1,
      "f1": 1,
      "precisionDefined": true,
      "recallDefined": true,
      "meanMatchedIou": 0.9166666666666667,
      "matches": [
        {
          "predictionId": "p1",
          "referenceId": "r1",
          "label": "person",
          "iou": 1,
          "exactBoundaries": true
        },
        {
          "predictionId": "p2",
          "referenceId": "r2",
          "label": "place",
          "iou": 0.8333333333333334,
          "exactBoundaries": false
        }
      ],
      "unmatchedPredictionIds": [],
      "unmatchedReferenceIds": [],
      "perLabel": [
        {
          "label": "person",
          "referenceCount": 1,
          "predictionCount": 1,
          "matchedCount": 1,
          "precision": 1,
          "recall": 1,
          "f1": 1
        },
        {
          "label": "place",
          "referenceCount": 1,
          "predictionCount": 1,
          "matchedCount": 1,
          "precision": 1,
          "recall": 1,
          "f1": 1
        }
      ],
      "conventions": [
        "Spans are nonempty half-open UTF-16 code-unit ranges; surrogate pairs cannot be split. Labels match exactly. Repeated label/boundary spans on the same side are rejected.",
        "Matching maximizes one-to-one pair count. Deterministic ID order and local IoU preference break ties; total IoU is not globally maximized.",
        "Undefined precision/recall/F1 uses 0. This evaluates supplied annotations and boundaries, not the truth or meaning of their labels."
      ]
    },
    "provenance": {
      "inputSha256": "17fd96b5568ac2fb16fcbdb52e9741bfa919234807e289f37c7ea47b30710f28",
      "outputSha256": "69f28072bd7f33501f9d2566743476fd9024f8437868687e435fa8156ff5c4f1",
      "deterministic": true,
      "externalRequests": 0
    }
  },
  "requiredHeaders": {
    "Content-Type": "application/json",
    "Idempotency-Key": "random 16–128 character operation identifier"
  },
  "fixedExample": "/reference/examples/span-evaluate.json",
  "execution": {
    "deterministic": true,
    "externalRequests": 0,
    "maxExternalRequests": 0,
    "resultSnapshotPersisted": false,
    "fixedExampleIsIllustrativeSnapshot": false,
    "requiresPayment": true,
    "supportsMcpExecution": false
  },
  "releaseSnapshot": {
    "format": "static-release-reference-v1",
    "sourceVersion": "0.29.0",
    "sourceRegistrySha256": "a2b5ec09bf6c38b9d2879d746a4fded374f5928b445377b0270ef6aa8e6cac65",
    "generatedAt": "2026-10-06T15:38:00Z",
    "releaseAcceptance": "not-verified-by-generator",
    "runtimeReadiness": "not-evaluated",
    "livePaymentsVerified": false,
    "indexingVerified": false,
    "apiOrigin": null,
    "notice": "Build-time release reference. Runtime readiness, deployment, payment settlement and external indexing are not evaluated here. Prices are proposed; this file cannot authorize execution or payment."
  }
}