{
  "title": "Shvydko candidate vs Jev: primary diagnostic only",
  "date": "2026-10-14",
  "source_sha256": "15f527efc3cf5b49160c54bc941f0ac8dc647c80e3d2f663285d6d6716b9ec66",
  "final_claim_pass": false,
  "final_claim_reason": "Primary diagnostic only; English guard, speed, economics and final-panel selection are not assessed here.",
  "not_assessed": [
    "english_guard",
    "speed",
    "economics",
    "final_panel_selection",
    "human_review"
  ],
  "primary": {
    "ru": {
      "adjusted_interval": [
        -0.006936597126395177,
        0.033598854924556186
      ],
      "candidate_accuracy": 0.9083048641266489,
      "candidate_accuracy_partial": null,
      "delta": 0.013150369755731755,
      "lower_bound_positive": false,
      "reference_accuracy": 0.8951544943709171,
      "reference_accuracy_partial": null
    },
    "surzhyk": {
      "adjusted_interval": [
        0.04875,
        0.1
      ],
      "candidate_accuracy": 1.0,
      "candidate_accuracy_partial": null,
      "delta": 0.0725,
      "lower_bound_positive": true,
      "reference_accuracy": 0.9275,
      "reference_accuracy_partial": null
    },
    "uk": {
      "adjusted_interval": [
        0.01269087259069369,
        0.05777957024210516
      ],
      "candidate_accuracy": 0.9027633469255786,
      "candidate_accuracy_partial": null,
      "delta": 0.03536178821256068,
      "lower_bound_positive": true,
      "reference_accuracy": 0.8674015587130178,
      "reference_accuracy_partial": null
    }
  },
  "bootstrap": {
    "draw_procedure": "Eligible questions sorted by (stratum, group, key); units are the distinct (stratum, group) pairs in that order. rng = numpy.random.default_rng(seed); for each draw, for each stratum in ascending order with k units, rng.integers(0, k, size=k) picks unit positions with replacement; draws use that same question order. One draw is shared by every language, family and both arms. draws_sha256 hashes every draw's global unit positions as int64 little-endian, concatenated in draw order.",
    "draw_signs": {
      "ru": {
        "negative": 1148,
        "positive": 18852,
        "zero": 0
      },
      "surzhyk": {
        "negative": 0,
        "positive": 20000,
        "zero": 0
      },
      "uk": {
        "negative": 1,
        "positive": 19999,
        "zero": 0
      }
    },
    "draws": 20000,
    "draws_sha256": "55aaaefe8b8b4fb7a57e520a481287b97b9c36207c5bc178f59893db5dde062d",
    "empty_cell_policy": "An empty required language/family cell in the original panel or any prespecified draw makes the primary comparison not_estimable. Preserve original point results and counts of empty draws; no omission, reweighting, or replacement draws.",
    "empty_cells": {},
    "empty_draws": 0,
    "interval_procedure": "Per draw and family: num = sum of multiplicity * (a - b) and den = sum of multiplicity, integers. Language delta = equal-weight mean of num/den over the language's families in float64; a draw within 1e-9 of zero is recomputed exactly (Python integers), so every draw carries its exact sign and an exact zero is 0.0. Endpoints: Hyndman-Fan type 7 (numpy 'linear') quantiles at the registered levels of the exact rational draw values; the two order statistics around each level are recomputed exactly from every draw within 1e-9 of them. Each endpoint is rounded once to float64; lower_bound_positive is the exact sign of the lower endpoint.",
    "min_groups_per_stratum": 80,
    "near_zero_draws_recomputed_exactly": {
      "ru": 0,
      "surzhyk": 0,
      "uk": 0
    },
    "pass": "All three adjusted lower bounds for candidate minus Jev must be strictly above zero.",
    "quantile_method": "linear",
    "quantiles": [
      0.008333333333333333,
      0.9916666666666667
    ],
    "questions_in_singleton_strata": 0,
    "questions_in_small_strata": 0,
    "resamples": 20000,
    "resampling_strata": 3,
    "resampling_units": 420,
    "resampling_units_sha256": "9ae2b4fad43e253dc03a2241d76106724dfee97718a2ad91d3a81023960538db",
    "seed": 20261003,
    "singleton_strata": 0,
    "small_strata": 0,
    "strata": "Explicit disjoint resampling blocks supplied by the panel. A group crossing supplied blocks is invalid; the comparator never silently splits or regroups it.",
    "unit": "Explicit global original-scenario group, with all included questions and language variants sampled together in both arms."
  },
  "panel_counts": {
    "cell_groups": {
      "ru/entailment": 98,
      "ru/passage_supported_inference": 140,
      "ru/policy_rule_application": 80,
      "surzhyk/policy_rule_application": 80,
      "uk/entailment": 100,
      "uk/passage_supported_inference": 141,
      "uk/policy_rule_application": 80
    },
    "cell_questions": {
      "ru/entailment": 294,
      "ru/passage_supported_inference": 260,
      "ru/policy_rule_application": 1103,
      "surzhyk/policy_rule_application": 800,
      "uk/entailment": 297,
      "uk/passage_supported_inference": 262,
      "uk/policy_rule_application": 1103
    },
    "cross_family_groups": 0,
    "cross_language_groups": 219,
    "excluded_questions": 9100,
    "exclusion_reasons": {
      "language_status:needs_review": 12,
      "no_primary_language:codeswitch": 800,
      "no_primary_language:cross": 800,
      "no_primary_language:translit": 800,
      "no_primary_language:typos": 800,
      "secondary_task_not_in_draft_allowlist": 5888
    },
    "groups": 420,
    "included_questions": 4119,
    "questions": 13219,
    "strata": {
      "entailment": {
        "groups": 198,
        "questions": 591
      },
      "passage_supported_inference": {
        "groups": 142,
        "questions": 522
      },
      "policy_rule_application": {
        "groups": 80,
        "questions": 3006
      }
    }
  },
  "scope_notes": [
    "This export contains aggregate diagnostic statistics only. No private source examples are included.",
    "Ukrainian and Russian use equal-weight macro across three task families: entailment, passage-supported inference, policy-rule application.",
    "Surzhyk contains policy-rule application only; its score is not a measure of general language ability.",
    "No general quality, speed or cost superiority is established by this diagnostic.",
    "Displayed percentage points are absolute percentage-point differences, not relative percent improvements."
  ],
  "report_id": "shvydko-evaluation-2026-10-14",
  "publisher": {
    "name": "Codebridge Technology, Inc.",
    "url": "https://www.codebridge.tech/"
  }
}