{
  "schema": "shvydko-public-served-comparison-v1",
  "report_id": "shvydko-v2-served-vs-jev-1.14-2026-10-16",
  "title": "Served Shvydko v2 vs Jev 1.14: limited development diagnostic",
  "date": "2026-10-16",
  "publication_date": "2026-10-17",
  "status": "estimable",
  "source_sha256": "f7e428d8ac1bb55060fc7acbb379ac52917acacb297841350d53f8e489671523",
  "source_hashes": {
    "candidate": "e1f54eeec4d259d00c0636c67b41319761c265af1e2abff9de570d168b014dae",
    "panel": "2c3a8c5ebe9cdeb7e699c66c03e64084f78755cb1e87f2ad247e098c954d709e",
    "protocol": "39bef07b764af59055068066a49a1c4137e9edb887071a6d0a478ce7aa3cb24d",
    "reference": "37862ce5d654b1bf2bd8f75a6a66bc862f7d83523718353da0aa6b335599a9b2",
    "served_summary": "886a1daf4dc0240a7bae1de1430cb1e7bc7540bb4e68499641b6b339edd6b988",
    "policy_manifest": "8e82f9ee8ede412dfd8674127fdddb993c997360e0e4dafe2f30519a184b8d45"
  },
  "versions": {
    "candidate": {
      "name": "Shvydko v2",
      "execution": "served endpoint",
      "measured_on": "2026-10-16",
      "snapshot_label": "Shvydko v2 served / 2026-10-16",
      "revision_sha256": "18e046808454bae251bbb020f9771adf2bf84aab72669d36462e5b8cbba1d4ad",
      "scope": "Measured service snapshot; the mutable public alias is not pinned to this version."
    },
    "reference": {
      "name": "Jev 1.14",
      "model_id": "typesafe/jev-1.14-20261014",
      "execution": "saved responses to the same frozen inputs",
      "immutable_revision": null,
      "scope": "The model identifier suffix is not asserted as the request date. No immutable vendor revision is available."
    }
  },
  "metric": {
    "name": "family_macro_accuracy",
    "definition": "Accuracy within each task family, then the equal-weight mean of family accuracies for each language; not pooled question accuracy.",
    "decision": "Argmax over answer probabilities; exact ties choose the first option.",
    "difference_unit": "absolute percentage points",
    "required_families": {
      "ru": [
        "entailment",
        "passage_supported_inference",
        "policy_rule_application"
      ],
      "uk": [
        "entailment",
        "passage_supported_inference",
        "policy_rule_application"
      ],
      "surzhyk": [
        "policy_rule_application"
      ]
    }
  },
  "primary": {
    "ru": {
      "adjusted_interval": [
        -0.017228691661086187,
        0.018358414424569376
      ],
      "candidate_accuracy": 0.9057350843259404,
      "candidate_accuracy_partial": null,
      "delta": 0.0007655967052418289,
      "lower_bound_positive": false,
      "reference_accuracy": 0.9049694876206986,
      "reference_accuracy_partial": null
    },
    "surzhyk": {
      "adjusted_interval": [
        0.025,
        0.07875
      ],
      "candidate_accuracy": 1.0,
      "candidate_accuracy_partial": null,
      "delta": 0.04875,
      "lower_bound_positive": true,
      "reference_accuracy": 0.95125,
      "reference_accuracy_partial": null
    },
    "uk": {
      "adjusted_interval": [
        0.007659508635369129,
        0.05098510178754072
      ],
      "candidate_accuracy": 0.9050080158369141,
      "candidate_accuracy_partial": null,
      "delta": 0.02931430283851047,
      "lower_bound_positive": true,
      "reference_accuracy": 0.8756937129984037,
      "reference_accuracy_partial": null
    }
  },
  "point_details": {
    "candidate": {
      "ru": {
        "complete": true,
        "families": [
          {
            "accounted": 294,
            "accuracy": 0.8163265306122449,
            "correct": 240,
            "failures": 0,
            "family": "entailment",
            "n": 294,
            "observed_accuracy_partial": 0.8163265306122449,
            "pending": 0,
            "unaccounted": 0
          },
          {
            "accounted": 260,
            "accuracy": 0.9153846153846154,
            "correct": 238,
            "failures": 0,
            "family": "passage_supported_inference",
            "n": 260,
            "observed_accuracy_partial": 0.9153846153846154,
            "pending": 0,
            "unaccounted": 0
          },
          {
            "accounted": 1103,
            "accuracy": 0.985494106980961,
            "correct": 1087,
            "failures": 0,
            "family": "policy_rule_application",
            "n": 1103,
            "observed_accuracy_partial": 0.985494106980961,
            "pending": 0,
            "unaccounted": 0
          }
        ],
        "family_macro_accuracy": 0.9057350843259404,
        "family_macro_accuracy_partial": null
      },
      "surzhyk": {
        "complete": true,
        "families": [
          {
            "accounted": 800,
            "accuracy": 1.0,
            "correct": 800,
            "failures": 0,
            "family": "policy_rule_application",
            "n": 800,
            "observed_accuracy_partial": 1.0,
            "pending": 0,
            "unaccounted": 0
          }
        ],
        "family_macro_accuracy": 1.0,
        "family_macro_accuracy_partial": null
      },
      "uk": {
        "complete": true,
        "families": [
          {
            "accounted": 297,
            "accuracy": 0.7878787878787878,
            "correct": 234,
            "failures": 0,
            "family": "entailment",
            "n": 297,
            "observed_accuracy_partial": 0.7878787878787878,
            "pending": 0,
            "unaccounted": 0
          },
          {
            "accounted": 262,
            "accuracy": 0.9389312977099237,
            "correct": 246,
            "failures": 0,
            "family": "passage_supported_inference",
            "n": 262,
            "observed_accuracy_partial": 0.9389312977099237,
            "pending": 0,
            "unaccounted": 0
          },
          {
            "accounted": 1103,
            "accuracy": 0.9882139619220308,
            "correct": 1090,
            "failures": 0,
            "family": "policy_rule_application",
            "n": 1103,
            "observed_accuracy_partial": 0.9882139619220308,
            "pending": 0,
            "unaccounted": 0
          }
        ],
        "family_macro_accuracy": 0.9050080158369141,
        "family_macro_accuracy_partial": null
      }
    },
    "reference": {
      "ru": {
        "complete": true,
        "families": [
          {
            "accounted": 294,
            "accuracy": 0.826530612244898,
            "correct": 243,
            "failures": 0,
            "family": "entailment",
            "n": 294,
            "observed_accuracy_partial": 0.826530612244898,
            "pending": 0,
            "unaccounted": 0
          },
          {
            "accounted": 260,
            "accuracy": 0.9346153846153846,
            "correct": 243,
            "failures": 0,
            "family": "passage_supported_inference",
            "n": 260,
            "observed_accuracy_partial": 0.9346153846153846,
            "pending": 0,
            "unaccounted": 0
          },
          {
            "accounted": 1103,
            "accuracy": 0.9537624660018132,
            "correct": 1052,
            "failures": 0,
            "family": "policy_rule_application",
            "n": 1103,
            "observed_accuracy_partial": 0.9537624660018132,
            "pending": 0,
            "unaccounted": 0
          }
        ],
        "family_macro_accuracy": 0.9049694876206986,
        "family_macro_accuracy_partial": null
      },
      "surzhyk": {
        "complete": true,
        "families": [
          {
            "accounted": 800,
            "accuracy": 0.95125,
            "correct": 761,
            "failures": 0,
            "family": "policy_rule_application",
            "n": 800,
            "observed_accuracy_partial": 0.95125,
            "pending": 0,
            "unaccounted": 0
          }
        ],
        "family_macro_accuracy": 0.95125,
        "family_macro_accuracy_partial": null
      },
      "uk": {
        "complete": true,
        "families": [
          {
            "accounted": 297,
            "accuracy": 0.734006734006734,
            "correct": 218,
            "failures": 0,
            "family": "entailment",
            "n": 297,
            "observed_accuracy_partial": 0.734006734006734,
            "pending": 0,
            "unaccounted": 0
          },
          {
            "accounted": 262,
            "accuracy": 0.9465648854961832,
            "correct": 248,
            "failures": 0,
            "family": "passage_supported_inference",
            "n": 262,
            "observed_accuracy_partial": 0.9465648854961832,
            "pending": 0,
            "unaccounted": 0
          },
          {
            "accounted": 1103,
            "accuracy": 0.9465095194922938,
            "correct": 1044,
            "failures": 0,
            "family": "policy_rule_application",
            "n": 1103,
            "observed_accuracy_partial": 0.9465095194922938,
            "pending": 0,
            "unaccounted": 0
          }
        ],
        "family_macro_accuracy": 0.8756937129984037,
        "family_macro_accuracy_partial": null
      }
    }
  },
  "bootstrap": {
    "draw_procedure": "Eligible questions sorted by (stratum, group, key); units are distinct (stratum, group) pairs in that order. rng = numpy.random.default_rng(seed); for each draw and stratum in ascending order with k units, rng.integers(0, k, size=k) samples unit positions with replacement. One draw is shared by every language, family and both arms. draws_sha256 hashes global unit positions as int64 little-endian, concatenated in draw order.",
    "draw_signs": {
      "ru": {
        "negative": 9128,
        "positive": 10872,
        "zero": 0
      },
      "surzhyk": {
        "negative": 0,
        "positive": 20000,
        "zero": 0
      },
      "uk": {
        "negative": 12,
        "positive": 19988,
        "zero": 0
      }
    },
    "draws": 20000,
    "draws_sha256": "55aaaefe8b8b4fb7a57e520a481287b97b9c36207c5bc178f59893db5dde062d",
    "empty_cell_policy": "An empty required language/family cell in the original panel or any prespecified draw makes the primary comparison not_estimable. Preserve original point results and counts of empty draws; no omission, reweighting, or replacement draws.",
    "empty_cells": {},
    "empty_draws": 0,
    "interval_procedure": "Per draw and family: num = sum of multiplicity * (a - b) and den = sum of multiplicity, integers. Language delta = equal-weight mean of num/den over the language's families in float64; a draw within 1e-9 of zero is recomputed exactly (Python integers), so every draw carries its exact sign and an exact zero is 0.0. Endpoints: Hyndman-Fan type 7 (numpy 'linear') quantiles at the registered levels of the exact rational draw values; the two order statistics around each level are recomputed exactly from every draw within 1e-9 of them. Each endpoint is rounded once to float64; lower_bound_positive is the exact sign of the lower endpoint.",
    "min_groups_per_stratum": 80,
    "near_zero_draws_recomputed_exactly": {
      "ru": 0,
      "surzhyk": 0,
      "uk": 0
    },
    "pass": "All three adjusted lower bounds for candidate minus Jev must be strictly above zero.",
    "quantile_method": "linear",
    "quantiles": [
      0.008333333333333333,
      0.9916666666666667
    ],
    "questions_in_singleton_strata": 0,
    "questions_in_small_strata": 0,
    "resamples": 20000,
    "resampling_strata": 3,
    "resampling_units": 420,
    "resampling_units_sha256": "9ae2b4fad43e253dc03a2241d76106724dfee97718a2ad91d3a81023960538db",
    "seed": 20261003,
    "singleton_strata": 0,
    "small_strata": 0,
    "strata": "Explicit disjoint resampling blocks supplied by the panel. A group crossing supplied blocks is invalid; the comparator never silently splits or regroups it.",
    "unit": "Explicit global original-scenario group, with all included questions and language variants sampled together in both arms."
  },
  "interval": {
    "confidence_level_percent": 98.33333333333333,
    "display_level_percent": 98.33,
    "adjustment": "Bonferroni across three language comparisons",
    "familywise_confidence_percent": 95
  },
  "panel_counts": {
    "cell_groups": {
      "ru/entailment": 98,
      "ru/passage_supported_inference": 140,
      "ru/policy_rule_application": 80,
      "surzhyk/policy_rule_application": 80,
      "uk/entailment": 100,
      "uk/passage_supported_inference": 141,
      "uk/policy_rule_application": 80
    },
    "cell_questions": {
      "ru/entailment": 294,
      "ru/passage_supported_inference": 260,
      "ru/policy_rule_application": 1103,
      "surzhyk/policy_rule_application": 800,
      "uk/entailment": 297,
      "uk/passage_supported_inference": 262,
      "uk/policy_rule_application": 1103
    },
    "cross_family_groups": 0,
    "cross_language_groups": 219,
    "excluded_questions": 9100,
    "exclusion_reasons": {
      "language_status:needs_review": 12,
      "no_primary_language:codeswitch": 800,
      "no_primary_language:cross": 800,
      "no_primary_language:translit": 800,
      "no_primary_language:typos": 800,
      "secondary_task_not_in_draft_allowlist": 5888
    },
    "groups": 420,
    "included_questions": 4119,
    "questions": 13219,
    "strata": {
      "entailment": {
        "groups": 198,
        "questions": 591
      },
      "passage_supported_inference": {
        "groups": 142,
        "questions": 522
      },
      "policy_rule_application": {
        "groups": 80,
        "questions": 3006
      }
    }
  },
  "coverage": {
    "requests": 1119,
    "questions_by_language": {
      "ru": 1657,
      "surzhyk": 800,
      "uk": 1662
    },
    "served_rejected_requests": 0,
    "served_successful_questions": 4119,
    "reference_successful_questions": 4119,
    "partition": "development",
    "panel_frozen_on": "2026-10-08"
  },
  "statistical_primary_pass": false,
  "training_related_scope": {
    "policy_questions": 3006,
    "shared_training_scenario_type_questions": 1802,
    "heldout_scenario_type_questions": 1204,
    "shared_generator": true,
    "unit": "scenario types, not entirely unseen broad subject domains",
    "exact_record_overlap_established": false
  },
  "registered_primary_gate_pass": false,
  "final_claim_pass": false,
  "final_claim_reason": "Primary diagnostic only; English guard, speed, economics and final-panel selection are not assessed here.",
  "not_assessed": [
    "english_guard",
    "speed",
    "economics",
    "final_panel_selection",
    "human_review"
  ],
  "warnings": [
    "reference: 9 successful answers are exact argmax ties, 3 credited by the first-option rule"
  ],
  "scope_notes": [
    "Exploratory comparison on a limited development panel; not an untouched final test or general superiority claim.",
    "Only 4,119 of 13,219 panel questions are eligible; exclusions are explicit in panel_counts.",
    "Ukrainian and Russian cover three task families; Surzhyk covers only synthetic policy-rule variants, not general or real-user Surzhyk ability.",
    "The policy generator is shared with training; 1,802 of 3,006 policy questions use scenario types also used for training. The other 1,204 use held-out scenario types, not necessarily new broad subject domains. This is scenario-type/generator overlap, not evidence that exact evaluation records were in training.",
    "The Russian interval crosses zero; superiority is not established. No equivalence test was performed.",
    "The Ukrainian aggregate does not imply improvement in every task family; all family counts and accuracies are included.",
    "All three adjusted lower bounds must be positive for the statistical primary pass. This condition is not met.",
    "No matched speed or cost comparison, English guard or final-panel claim is established by this diagnostic.",
    "Aggregate statistics only; no private examples, implementation, provider details or credentials."
  ],
  "previous_report": "/research/decision-evaluation-2026-10-14.json",
  "publisher": {
    "name": "Codebridge Technology, Inc.",
    "url": "https://www.codebridge.tech/"
  }
}