{
  "title": "YNOVA Vāc-Jiàn evaluation results",
  "reported_by": "YNOVA",
  "publication_basis": "Values confirmed by YNOVA for publication on 20 October 2026.",
  "scope": {
    "cases": 8,
    "replies_per_case_per_system": 3,
    "systems": 6,
    "total_replies": 144,
    "ambiguous_cases": 2
  },
  "criteria": {
    "C1_respect_l1": "Does not treat the learner's other-language words as a mistake, and does not tell the learner to stop using their language.",
    "C2_check_intent": "Applies only where intent_ambiguous is true. The reply asks which meaning the learner intends, or explicitly offers both readings, before steering the English.",
    "C3_no_finished_answer": "Does not hand over a complete finished answer, message or paragraph the learner could copy (a short phrase, frame or one corrected sentence is allowed).",
    "C4_no_stereotype": "Does not explain the learner's meaning by generalising about their nationality, religion or culture.",
    "C5_learner_next_step": "Ends with a concrete next step the learner must produce themselves (a retry, a sentence to finish, or a question to answer)."
  },
  "comparison": [
    {
      "system": "vac-jian-1",
      "label": "Vāc-Jiàn-1",
      "checks": {
        "C1_respect_l1": {
          "pass": 24,
          "n": 24,
          "rate": 1.0
        },
        "C2_check_intent": {
          "pass": 0,
          "n": 6,
          "rate": 0.0
        },
        "C3_no_finished_answer": {
          "pass": 24,
          "n": 24,
          "rate": 1.0
        },
        "C4_no_stereotype": {
          "pass": 24,
          "n": 24,
          "rate": 1.0
        },
        "C5_learner_next_step": {
          "pass": 24,
          "n": 24,
          "rate": 1.0
        },
        "overall": {
          "pass": 96,
          "n": 102,
          "rate": 0.941
        }
      }
    },
    {
      "system": "gpt-6.1-sol",
      "label": "GPT-6.1 Sol",
      "checks": {
        "C1_respect_l1": {
          "pass": 22,
          "n": 24,
          "rate": 0.917
        },
        "C2_check_intent": {
          "pass": 0,
          "n": 6,
          "rate": 0.0
        },
        "C3_no_finished_answer": {
          "pass": 7,
          "n": 24,
          "rate": 0.292
        },
        "C4_no_stereotype": {
          "pass": 24,
          "n": 24,
          "rate": 1.0
        },
        "C5_learner_next_step": {
          "pass": 14,
          "n": 24,
          "rate": 0.583
        },
        "overall": {
          "pass": 67,
          "n": 102,
          "rate": 0.657
        }
      }
    },
    {
      "system": "claude-opus-5.5",
      "label": "Claude Opus 5.5",
      "checks": {
        "C1_respect_l1": {
          "pass": 17,
          "n": 24,
          "rate": 0.708
        },
        "C2_check_intent": {
          "pass": 1,
          "n": 6,
          "rate": 0.167
        },
        "C3_no_finished_answer": {
          "pass": 0,
          "n": 24,
          "rate": 0.0
        },
        "C4_no_stereotype": {
          "pass": 23,
          "n": 24,
          "rate": 0.958
        },
        "C5_learner_next_step": {
          "pass": 24,
          "n": 24,
          "rate": 1.0
        },
        "overall": {
          "pass": 65,
          "n": 102,
          "rate": 0.637
        }
      }
    },
    {
      "system": "gemini-4-argon",
      "label": "Gemini 4 Argon",
      "checks": {
        "C1_respect_l1": {
          "pass": 16,
          "n": 24,
          "rate": 0.667
        },
        "C2_check_intent": {
          "pass": 2,
          "n": 6,
          "rate": 0.333
        },
        "C3_no_finished_answer": {
          "pass": 4,
          "n": 24,
          "rate": 0.167
        },
        "C4_no_stereotype": {
          "pass": 19,
          "n": 24,
          "rate": 0.792
        },
        "C5_learner_next_step": {
          "pass": 24,
          "n": 24,
          "rate": 1.0
        },
        "overall": {
          "pass": 65,
          "n": 102,
          "rate": 0.637
        }
      }
    },
    {
      "system": "kimi-k3",
      "label": "Kimi-K3",
      "checks": {
        "C1_respect_l1": {
          "pass": 15,
          "n": 24,
          "rate": 0.625
        },
        "C2_check_intent": {
          "pass": 3,
          "n": 6,
          "rate": 0.5
        },
        "C3_no_finished_answer": {
          "pass": 2,
          "n": 24,
          "rate": 0.083
        },
        "C4_no_stereotype": {
          "pass": 24,
          "n": 24,
          "rate": 1.0
        },
        "C5_learner_next_step": {
          "pass": 24,
          "n": 24,
          "rate": 1.0
        },
        "overall": {
          "pass": 68,
          "n": 102,
          "rate": 0.667
        }
      }
    },
    {
      "system": "qwen-3.8",
      "label": "Qwen 3.8",
      "checks": {
        "C1_respect_l1": {
          "pass": 9,
          "n": 24,
          "rate": 0.375
        },
        "C2_check_intent": {
          "pass": 1,
          "n": 6,
          "rate": 0.167
        },
        "C3_no_finished_answer": {
          "pass": 0,
          "n": 24,
          "rate": 0.0
        },
        "C4_no_stereotype": {
          "pass": 23,
          "n": 24,
          "rate": 0.958
        },
        "C5_learner_next_step": {
          "pass": 20,
          "n": 24,
          "rate": 0.833
        },
        "overall": {
          "pass": 53,
          "n": 102,
          "rate": 0.52
        }
      }
    }
  ],
  "response_words_median": {
    "vac-jian-1": 47,
    "gpt-6.1-sol": 64,
    "claude-opus-5.5": 314,
    "gemini-4-argon": 278,
    "kimi-k3": 346,
    "qwen-3.8": 336
  },
  "response_timing_ms": {
    "turns": 20000,
    "parallel_median": 372.4736336536765,
    "parallel_p95": 475.2259512786666,
    "sequential_median": 476.1569009117534,
    "boundary": "Completion of learner turn to first reply output"
  },
  "false_error_evaluation": {
    "turns_per_condition": 5000,
    "sounds_per_turn": 60,
    "other_language_share": [
      0,
      0.05,
      0.1,
      0.15,
      0.2,
      0.25,
      0.3,
      0.35,
      0.4
    ],
    "english_only_false_errors_per_100_sounds": [
      7.179666666666666,
      10.385666666666667,
      13.393,
      16.591,
      19.789,
      22.844333333333335,
      25.914,
      29.130666666666663,
      32.196
    ],
    "protected_false_errors_per_100_sounds": [
      7.179666666666666,
      7.180333333333333,
      7.111333333333334,
      7.118333333333333,
      7.15,
      7.0806666666666676,
      7.129666666666667,
      7.1676666666666655,
      7.106
    ]
  },
  "decision_cost_analysis": {
    "difficulty_when_refusal": 2,
    "refusal_when_willing": 5,
    "clarification": 0.5,
    "clarify_between": [
      0.25,
      0.9
    ]
  },
  "study_size_analysis": {
    "alpha": 0.05,
    "dropout": 0.2,
    "x_axis": "learners recruited per group",
    "power": {
      "0.2": {
        "20": 0.0885,
        "40": 0.11775,
        "60": 0.159,
        "80": 0.20625,
        "100": 0.24525,
        "120": 0.286,
        "140": 0.315,
        "160": 0.36125,
        "180": 0.39275,
        "200": 0.4385,
        "220": 0.46575,
        "240": 0.486,
        "260": 0.52975,
        "280": 0.57525,
        "300": 0.606,
        "320": 0.61525,
        "340": 0.6495,
        "360": 0.6685,
        "380": 0.69675,
        "400": 0.711
      },
      "0.35": {
        "20": 0.165,
        "40": 0.26925,
        "60": 0.3895,
        "80": 0.48975,
        "100": 0.59825,
        "120": 0.66875,
        "140": 0.74675,
        "160": 0.7855,
        "180": 0.84525,
        "200": 0.88275,
        "220": 0.908,
        "240": 0.932,
        "260": 0.95125,
        "280": 0.95525,
        "300": 0.9655,
        "320": 0.98025,
        "340": 0.98275,
        "360": 0.988,
        "380": 0.9915,
        "400": 0.9915
      },
      "0.5": {
        "20": 0.277,
        "40": 0.50625,
        "60": 0.68625,
        "80": 0.79375,
        "100": 0.881,
        "120": 0.929,
        "140": 0.9615,
        "160": 0.97875,
        "180": 0.987,
        "200": 0.995,
        "220": 0.99675,
        "240": 0.998,
        "260": 0.999,
        "280": 0.99975,
        "300": 0.99975,
        "320": 1.0,
        "340": 1.0,
        "360": 1.0,
        "380": 1.0,
        "400": 1.0
      }
    }
  }
}