{
  "benchmarks": [
    {
      "id": "fsr",
      "title": "FrontierScience-Research",
      "description": "FrontierScience-Research tests whether a model can work through unfamiliar, multi-step research problems in physics, chemistry, and biology.",
      "metric": "Success rate (%)",
      "sampleCount": null,
      "axis": {
        "min": 20,
        "max": 40,
        "ticks": [
          20,
          25,
          30,
          35,
          40
        ]
      },
      "models": [
        {
          "name": "GLM-5.3",
          "role": "base",
          "score": 24.4
        },
        {
          "name": "Kimi-K3",
          "role": "comparison",
          "score": 29.7
        },
        {
          "name": "GPT-6-Astra",
          "role": "frontier",
          "score": 33.3
        },
        {
          "name": "Hydrogen-1.0",
          "role": "nous",
          "score": 38.3
        }
      ]
    },
    {
      "id": "gpqa",
      "title": "GPQA-Diamonds",
      "description": "GPQA-Diamonds tests precise scientific knowledge through expert-written multiple-choice questions in biology, chemistry, and physics. Its questions are selected for expert agreement while remaining difficult for non-experts, even with web access.",
      "metric": "Accuracy (%)",
      "sampleCount": null,
      "axis": {
        "min": 85,
        "max": 100,
        "ticks": [
          85,
          90,
          95,
          100
        ]
      },
      "models": [
        {
          "name": "GLM-5.3",
          "role": "base",
          "score": 90
        },
        {
          "name": "Kimi-K3",
          "role": "comparison",
          "score": 93.5
        },
        {
          "name": "GPT-6-Astra",
          "role": "frontier",
          "score": 96.1
        },
        {
          "name": "Hydrogen-1.0",
          "role": "nous",
          "score": 93
        }
      ]
    }
  ],
  "domainTasks": [
    {
      "id": "scientific-computing",
      "name": "Scientific computing",
      "description": "Scientific computing uses numerical methods and simulation software to study molecules and materials.",
      "sampleCount": 39,
      "excludedCount": 0,
      "metric": "Mean response score (%)",
      "models": [
        {
          "name": "Hydrogen-1.0",
          "role": "nous",
          "score": 76.92307692307693
        },
        {
          "name": "GPT-6-Astra",
          "role": "frontier",
          "score": 71.7948717948718
        }
      ]
    },
    {
      "id": "forward-reactions",
      "name": "Forward reactions",
      "description": "Forward reaction prediction identifies likely chemical products from starting materials and reaction conditions.",
      "sampleCount": 42,
      "excludedCount": 0,
      "metric": "Mean response score (%)",
      "models": [
        {
          "name": "Hydrogen-1.0",
          "role": "nous",
          "score": 90.47619047619048
        },
        {
          "name": "GPT-6-Astra",
          "role": "frontier",
          "score": 83.33333333333334
        }
      ]
    },
    {
      "id": "nmr-structure",
      "name": "NMR structure elucidation",
      "description": "NMR structure elucidation determines a molecule’s structure from patterns in its nuclear magnetic resonance spectra.",
      "sampleCount": 30,
      "excludedCount": 1,
      "metric": "Mean response score (%)",
      "models": [
        {
          "name": "Hydrogen-1.0",
          "role": "nous",
          "score": 26.666666666666668
        },
        {
          "name": "GPT-6-Astra",
          "role": "frontier",
          "score": 30
        }
      ]
    }
  ],
  "experimentPlanning": {
    "title": "Experiment design and planning",
    "description": "Mean rubric score (/10) on 117 shared questions.",
    "metric": "Mean score (/10)",
    "maxScore": 10,
    "sampleCount": 117,
    "gradedAnswerCount": {
      "kimi": 117,
      "hydrogen": 219
    },
    "axis": {
      "min": 4,
      "max": 6,
      "ticks": [
        4,
        4.5,
        5,
        5.5,
        6
      ]
    },
    "outputTokenEstimate": {
      "encoding": "cl100k_base",
      "includesReasoning": true,
      "kind": "shared-tokenizer estimate"
    },
    "apiCostEstimate": {
      "currency": "USD",
      "basis": "output-only",
      "asOfDate": "2026-10-27"
    },
    "models": [
      {
        "name": "Kimi-K3",
        "role": "comparison",
        "score": 4.568376068376068,
        "meanTotalOutputTokens": 19578.74358974359,
        "outputUsdPerMillion": 15,
        "meanOutputApiCostUsd": 0.2936811538461539
      },
      {
        "name": "Hydrogen-1.0",
        "role": "nous",
        "score": 5.587606837606837,
        "meanTotalOutputTokens": 2698.74358974359,
        "outputUsdPerMillion": 4.4,
        "meanOutputApiCostUsd": 0.011874471794871798
      }
    ]
  }
}