{
  "device": {
    "jax": "0.11.2",
    "backend": "gpu",
    "devices": [
      "cuda:0"
    ],
    "python": "3.14.8",
    "device_kind": "NVIDIA GeForce RTX 3090",
    "gpu": "NVIDIA GeForce RTX 3090",
    "driver": "615.78.08",
    "gpu_memory_gb": 24.0,
    "torch": "2.14.1+cu130",
    "torch_cuda": "13.0"
  },
  "config": {
    "output": "docs/benchmarks/gpu_attention_tiles.json",
    "append": false,
    "dtypes": [
      "bfloat16",
      "float32"
    ],
    "head_dims": [
      64,
      128
    ],
    "seq_lens": [
      2048,
      8192
    ],
    "modes": [
      "fwd",
      "fwd_bwd"
    ],
    "batch": 4,
    "heads": 8,
    "repeats": 5,
    "auto_only": false,
    "tag": null,
    "fwd_block_q": [
      32,
      64,
      128
    ],
    "fwd_block_k": [
      32,
      64,
      128
    ],
    "fwd_stages": [
      1,
      2,
      3
    ],
    "fwd_warps": [
      4,
      8
    ],
    "bwd_outer": [
      32,
      64,
      128
    ],
    "bwd_inner": [
      16,
      32,
      64,
      128
    ],
    "bwd_stages": [
      1,
      2
    ],
    "bwd_warps": [
      4,
      8
    ],
    "bwd_fwd_config": null,
    "smem_limit_bytes": 101376
  },
  "results": [
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "auto",
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.1825575469993055,
      "inner": 68,
      "median_ms": 0.5675104852450793,
      "q1_ms": 0.5674552500268499,
      "q3_ms": 0.5675157941261525
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "auto",
      "rel_err": 0.005375336389988661,
      "compile_s": 0.43514215100731235,
      "inner": 20,
      "median_ms": 2.4425959498330485,
      "q1_ms": 2.4422428497928195,
      "q3_ms": 2.4442445996101014
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 32,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.1298139069986064,
      "inner": 61,
      "median_ms": 0.7716165082605525,
      "q1_ms": 0.7713888688337394,
      "q3_ms": 0.7716457378100146
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 32,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.12362104500061832,
      "inner": 38,
      "median_ms": 1.2936993156663927,
      "q1_ms": 1.2926470525072593,
      "q3_ms": 1.2942448156536557
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 32,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.13603280899405945,
      "inner": 60,
      "median_ms": 0.7769363667951742,
      "q1_ms": 0.7767562000177956,
      "q3_ms": 0.7793379166590361
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 32,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.12652578900451772,
      "inner": 38,
      "median_ms": 1.2824569999601838,
      "q1_ms": 1.282406105402563,
      "q3_ms": 1.2824941578927744
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 32,
      "num_stages": 3,
      "num_warps": 4,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.14129546099866275,
      "inner": 55,
      "median_ms": 0.8472395999970931,
      "q1_ms": 0.8467931088737466,
      "q3_ms": 0.8473241090541705
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 32,
      "num_stages": 3,
      "num_warps": 8,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.1303093140013516,
      "inner": 37,
      "median_ms": 1.3083327298895835,
      "q1_ms": 1.3083189189421467,
      "q3_ms": 1.3085060270748579
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 64,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.14740943400829565,
      "inner": 67,
      "median_ms": 0.668320358068849,
      "q1_ms": 0.6680464178302895,
      "q3_ms": 0.6684554031000598
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 64,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.12838411300617736,
      "inner": 48,
      "median_ms": 0.9673830830555138,
      "q1_ms": 0.9672090000094613,
      "q3_ms": 0.9677729791898552
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 64,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.15345239300222602,
      "inner": 67,
      "median_ms": 0.6930228955209701,
      "q1_ms": 0.6923152986521235,
      "q3_ms": 0.6933607015091536
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 64,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.13834417899488471,
      "inner": 48,
      "median_ms": 0.9582435001599757,
      "q1_ms": 0.9581597914802842,
      "q3_ms": 0.958291937422473
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 64,
      "num_stages": 3,
      "num_warps": 4,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.1601546089950716,
      "inner": 58,
      "median_ms": 0.772691672519331,
      "q1_ms": 0.7726419310623395,
      "q3_ms": 0.7731553103377369
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 64,
      "num_stages": 3,
      "num_warps": 8,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.14237357099773362,
      "inner": 47,
      "median_ms": 1.0005401275888521,
      "q1_ms": 1.0001502340991406,
      "q3_ms": 1.0025652340350752
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 128,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.18897143799404148,
      "inner": 66,
      "median_ms": 0.6768136363737567,
      "q1_ms": 0.6761947425371599,
      "q3_ms": 0.6769429696071187
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 128,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.15537853600108065,
      "inner": 53,
      "median_ms": 0.8596210188083,
      "q1_ms": 0.8595837735931195,
      "q3_ms": 0.8597098678605244
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 128,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.19720733400026802,
      "inner": 42,
      "median_ms": 1.0302362619161916,
      "q1_ms": 1.030048309379102,
      "q3_ms": 1.0303593569135825
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 128,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.15970162200392224,
      "inner": 42,
      "median_ms": 1.0891109524804743,
      "q1_ms": 1.0890956667329494,
      "q3_ms": 1.0891954048669763
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 128,
      "num_stages": 3,
      "num_warps": 4,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.20948448199487757,
      "inner": 53,
      "median_ms": 0.8559979433589175,
      "q1_ms": 0.8559554152493805,
      "q3_ms": 0.8560002076918399
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 128,
      "num_stages": 3,
      "num_warps": 8,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.1664606139966054,
      "inner": 44,
      "median_ms": 0.9964444999985228,
      "q1_ms": 0.9961015910613985,
      "q3_ms": 0.9966917952459137
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 32,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.14078367300680839,
      "inner": 71,
      "median_ms": 0.5821604365696081,
      "q1_ms": 0.5816119295892529,
      "q3_ms": 0.5822522958419221
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 32,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.13546122799743898,
      "inner": 43,
      "median_ms": 1.101142488593279,
      "q1_ms": 1.1009882558895232,
      "q3_ms": 1.1011974880645095
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 32,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.14899763799621724,
      "inner": 75,
      "median_ms": 0.5782374400102223,
      "q1_ms": 0.5755327333463356,
      "q3_ms": 0.5791645200224593
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 32,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.14099025400355458,
      "inner": 43,
      "median_ms": 1.1114487674921145,
      "q1_ms": 1.1113012557426944,
      "q3_ms": 1.1116724185845923
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 32,
      "num_stages": 3,
      "num_warps": 4,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.15145855200535152,
      "inner": 75,
      "median_ms": 0.5765314132440835,
      "q1_ms": 0.5764615465886891,
      "q3_ms": 0.5765768400548648
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 32,
      "num_stages": 3,
      "num_warps": 8,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.14204381599847693,
      "inner": 43,
      "median_ms": 1.0828103952727084,
      "q1_ms": 1.080126488421567,
      "q3_ms": 1.090313186110972
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 64,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.17664598699775524,
      "inner": 81,
      "median_ms": 0.5674153332092604,
      "q1_ms": 0.5674152097874609,
      "q3_ms": 0.5674179259653168
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 64,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.16437468999356497,
      "inner": 36,
      "median_ms": 1.234245055254885,
      "q1_ms": 1.2336971109713584,
      "q3_ms": 1.234839805369524
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 64,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.18029864500567783,
      "inner": 78,
      "median_ms": 0.5577913205996036,
      "q1_ms": 0.557668653924245,
      "q3_ms": 0.5578280513127072
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 64,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.16775409098772798,
      "inner": 38,
      "median_ms": 1.2028424734078151,
      "q1_ms": 1.2021163684518175,
      "q3_ms": 1.2029379209515785
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 64,
      "num_stages": 3,
      "num_warps": 4,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.18324889500217978,
      "inner": 78,
      "median_ms": 0.5659387819692852,
      "q1_ms": 0.5656335896884019,
      "q3_ms": 0.5683647692561722
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 64,
      "num_stages": 3,
      "num_warps": 8,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.17189395999594126,
      "inner": 40,
      "median_ms": 1.129356350065791,
      "q1_ms": 1.1274207252427004,
      "q3_ms": 1.1344790498696966
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 128,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.25072416599141434,
      "inner": 71,
      "median_ms": 0.5853643942571861,
      "q1_ms": 0.5853594507051396,
      "q3_ms": 0.585407281620756
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 128,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.22765412500302773,
      "inner": 37,
      "median_ms": 1.2072117568931315,
      "q1_ms": 1.2071424323555737,
      "q3_ms": 1.208053621372233
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 128,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.24970115099858958,
      "inner": 60,
      "median_ms": 0.6827701164487129,
      "q1_ms": 0.6819149833366586,
      "q3_ms": 0.6830272667381602
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 128,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.23233967600390315,
      "inner": 40,
      "median_ms": 1.1328858003253117,
      "q1_ms": 1.1324011247779708,
      "q3_ms": 1.1335139748553047
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 128,
      "num_stages": 3,
      "num_warps": 4,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.25686238500929903,
      "inner": 69,
      "median_ms": 0.6116435652602112,
      "q1_ms": 0.6105381450480849,
      "q3_ms": 0.6116745072717279
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 128,
      "num_stages": 3,
      "num_warps": 8,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.2420328880107263,
      "inner": 42,
      "median_ms": 1.0997472619194362,
      "q1_ms": 1.0994962857304407,
      "q3_ms": 1.0997835239062883
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 32,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.1967195909965085,
      "inner": 65,
      "median_ms": 0.6270875075223068,
      "q1_ms": 0.626334538593745,
      "q3_ms": 0.6277499845824563
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 32,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.13794169899483677,
      "inner": 68,
      "median_ms": 0.6124413382220903,
      "q1_ms": 0.6091480735288796,
      "q3_ms": 0.6126905001806068
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 32,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.20018445399182383,
      "inner": 67,
      "median_ms": 0.6011812984613256,
      "q1_ms": 0.6011560298207063,
      "q3_ms": 0.6012538208629685
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 32,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.14168932600296102,
      "inner": 76,
      "median_ms": 0.6029979079980158,
      "q1_ms": 0.6029845922296916,
      "q3_ms": 0.6030932105970447
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 32,
      "num_stages": 3,
      "num_warps": 4,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.20815557999594603,
      "inner": 69,
      "median_ms": 0.5796551014772494,
      "q1_ms": 0.5791084056355706,
      "q3_ms": 0.5796851594071916
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 32,
      "num_stages": 3,
      "num_warps": 8,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.14666899500298314,
      "inner": 70,
      "median_ms": 0.5932349428933646,
      "q1_ms": 0.5723291143242802,
      "q3_ms": 0.5932379428746312
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 64,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.2561848040058976,
      "inner": 69,
      "median_ms": 0.5858260435912678,
      "q1_ms": 0.5857887246774213,
      "q3_ms": 0.5859044490996208
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 64,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.1645872109947959,
      "inner": 59,
      "median_ms": 0.6870865423263887,
      "q1_ms": 0.6828351186537105,
      "q3_ms": 0.6871111693464504
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 64,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.2623126719990978,
      "inner": 71,
      "median_ms": 0.5612158873216839,
      "q1_ms": 0.5599374224556427,
      "q3_ms": 0.5612534226547979
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 64,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.17381399200530723,
      "inner": 65,
      "median_ms": 0.6334462923964915,
      "q1_ms": 0.6333982001076668,
      "q3_ms": 0.6334515384738691
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 64,
      "num_stages": 3,
      "num_warps": 4,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.2758140270016156,
      "inner": 73,
      "median_ms": 0.5921174932354167,
      "q1_ms": 0.5921030822544907,
      "q3_ms": 0.5921654110975334
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 64,
      "num_stages": 3,
      "num_warps": 8,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.18180355199729092,
      "inner": 71,
      "median_ms": 0.5980294647062538,
      "q1_ms": 0.597999267637546,
      "q3_ms": 0.5980843520710166
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 128,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.41617762499663513,
      "inner": 68,
      "median_ms": 0.6528165440078523,
      "q1_ms": 0.6527245883155367,
      "q3_ms": 0.6529508971331833
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 128,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.22718358600104693,
      "inner": 65,
      "median_ms": 0.6482724921974855,
      "q1_ms": 0.647653015375209,
      "q3_ms": 0.6483715999190911
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 128,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.4535509959969204,
      "inner": 56,
      "median_ms": 0.7172343037901945,
      "q1_ms": 0.7172114105092727,
      "q3_ms": 0.7176756784506974
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 128,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.23371014799340628,
      "inner": 64,
      "median_ms": 0.6141092187590402,
      "q1_ms": 0.6139287343103206,
      "q3_ms": 0.6141490000572958
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 128,
      "num_stages": 3,
      "num_warps": 4,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.589492674000212,
      "inner": 52,
      "median_ms": 0.8091383463649365,
      "q1_ms": 0.8090227308387582,
      "q3_ms": 0.8093570191144513
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 128,
      "num_stages": 3,
      "num_warps": 8,
      "rel_err": 0.0031628250144422054,
      "compile_s": 0.24713429700932465,
      "inner": 69,
      "median_ms": 0.5929769563869964,
      "q1_ms": 0.5927941448939289,
      "q3_ms": 0.5931846088429481
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 16,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.005375336389988661,
      "compile_s": 0.24057269698823802,
      "inner": 15,
      "median_ms": 3.3908368000993505,
      "q1_ms": 3.388736799630957,
      "q3_ms": 3.3936421328689903
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 16,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.005375336389988661,
      "compile_s": 0.23554103200149257,
      "inner": 9,
      "median_ms": 5.845242443481563,
      "q1_ms": 5.845152333171831,
      "q3_ms": 5.863590555640662
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 16,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.005375336389988661,
      "compile_s": 0.2565535300091142,
      "inner": 17,
      "median_ms": 3.1432722351061835,
      "q1_ms": 3.1313091175808734,
      "q3_ms": 3.147706588874023
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 16,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.005375336389988661,
      "compile_s": 0.2627209640049841,
      "inner": 8,
      "median_ms": 6.082383126340574,
      "q1_ms": 6.061665124434512,
      "q3_ms": 6.088783999075531
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 32,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.005375336389988661,
      "compile_s": 0.24841850499797147,
      "inner": 19,
      "median_ms": 2.7644360525300726,
      "q1_ms": 2.7637131049567345,
      "q3_ms": 2.7653857894045744
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 32,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.005375336389988661,
      "compile_s": 0.22583062699413858,
      "inner": 10,
      "median_ms": 4.860483198717702,
      "q1_ms": 4.860210799961351,
      "q3_ms": 4.862365800363477
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 32,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.005375336389988661,
      "compile_s": 0.26005888999497984,
      "inner": 19,
      "median_ms": 2.650599684688802,
      "q1_ms": 2.6497122629447594,
      "q3_ms": 2.6518953159929404
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 32,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.005375336389988661,
      "compile_s": 0.24711019100504927,
      "inner": 11,
      "median_ms": 4.676140909644097,
      "q1_ms": 4.659606818511913,
      "q3_ms": 4.678082817752676
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 64,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.005375336389988661,
      "compile_s": 0.2993795539950952,
      "inner": 19,
      "median_ms": 2.7502466316371668,
      "q1_ms": 2.746880209949603,
      "q3_ms": 2.75843889424945
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 64,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.005375336389988661,
      "compile_s": 0.2578813319996698,
      "inner": 12,
      "median_ms": 4.206243250033974,
      "q1_ms": 4.2058566662793355,
      "q3_ms": 4.218183416620984
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 64,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.005375336389988661,
      "compile_s": 0.30919383600121364,
      "inner": 20,
      "median_ms": 2.5972485498641618,
      "q1_ms": 2.59095104993321,
      "q3_ms": 2.5985669999499805
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 64,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.005375336389988661,
      "compile_s": 0.27035298800910823,
      "inner": 13,
      "median_ms": 4.029940769004707,
      "q1_ms": 4.029730385249982,
      "q3_ms": 4.030632076767058
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 128,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.005375336389988661,
      "compile_s": 0.42583892699622083,
      "inner": 17,
      "median_ms": 2.913940411150072,
      "q1_ms": 2.9103617065736804,
      "q3_ms": 2.922095235108453
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 128,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.005375336389988661,
      "compile_s": 0.34661471599247307,
      "inner": 12,
      "median_ms": 4.06763866582575,
      "q1_ms": 4.058174999954645,
      "q3_ms": 4.067986832524184
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 128,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.005375336389988661,
      "compile_s": 0.42983776998880785,
      "inner": 17,
      "median_ms": 2.7302146473255777,
      "q1_ms": 2.720870470511727,
      "q3_ms": 2.745834176654599
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 128,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.005375336389988661,
      "compile_s": 0.35232995499973185,
      "inner": 11,
      "median_ms": 4.377009635473686,
      "q1_ms": 4.371030272026969,
      "q3_ms": 4.377490545845252
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 16,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.005375336389988661,
      "compile_s": 0.3167878670064965,
      "inner": 10,
      "median_ms": 4.416078800568357,
      "q1_ms": 4.412145300011616,
      "q3_ms": 4.436762801196892
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 16,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.005375336389988661,
      "compile_s": 0.31082564100506715,
      "inner": 5,
      "median_ms": 9.91093380143866,
      "q1_ms": 9.860801001195796,
      "q3_ms": 9.936676599318162
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 16,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.005375336389988661,
      "compile_s": 0.3287464830063982,
      "inner": 12,
      "median_ms": 4.147614833830933,
      "q1_ms": 4.145928500899269,
      "q3_ms": 4.151563083723886
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 16,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.005375336389988661,
      "compile_s": 0.32417178199102636,
      "inner": 5,
      "median_ms": 9.09721280040685,
      "q1_ms": 9.050953201949596,
      "q3_ms": 9.115040602046065
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 32,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.005375336389988661,
      "compile_s": 0.3511996379966149,
      "inner": 14,
      "median_ms": 3.276892356682635,
      "q1_ms": 3.2711493570656915,
      "q3_ms": 3.2969946426289556
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 32,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.005375336389988661,
      "compile_s": 0.33787321399722714,
      "inner": 7,
      "median_ms": 7.08144685735793,
      "q1_ms": 7.079790999081784,
      "q3_ms": 7.085302713676356
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 32,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.005375336389988661,
      "compile_s": 0.3649501829931978,
      "inner": 17,
      "median_ms": 3.0182063526487632,
      "q1_ms": 3.0176877061261194,
      "q3_ms": 3.023574176682707
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 32,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.005375336389988661,
      "compile_s": 0.3559557819971815,
      "inner": 8,
      "median_ms": 6.178902000101516,
      "q1_ms": 6.173586874865578,
      "q3_ms": 6.186775624883012
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 64,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.005375336389988661,
      "compile_s": 0.4101126750028925,
      "inner": 18,
      "median_ms": 2.60382716644219,
      "q1_ms": 2.6033691109559083,
      "q3_ms": 2.6038411112191775
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 64,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.005375336389988661,
      "compile_s": 0.3933942970033968,
      "inner": 9,
      "median_ms": 5.496593222940444,
      "q1_ms": 5.488707000040449,
      "q3_ms": 5.504208555470945
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 64,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.005375336389988661,
      "compile_s": 0.4325425150018418,
      "inner": 20,
      "median_ms": 2.4642910502734594,
      "q1_ms": 2.4632780499814544,
      "q3_ms": 2.4658880494826008
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 64,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.005375336389988661,
      "compile_s": 0.3902449609886389,
      "inner": 10,
      "median_ms": 4.943369099055417,
      "q1_ms": 4.929203100618906,
      "q3_ms": 4.952251900976989
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 128,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.005375336389988661,
      "compile_s": 0.6327145350078354,
      "inner": 14,
      "median_ms": 3.4163374293711968,
      "q1_ms": 3.4144739277378124,
      "q3_ms": 3.4198777856155567
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 128,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.005375336389988661,
      "compile_s": 0.5145694769889815,
      "inner": 9,
      "median_ms": 5.3063411113624985,
      "q1_ms": 5.303098333469179,
      "q3_ms": 5.306962334240477
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 128,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.005375336389988661,
      "compile_s": 0.6337048879940994,
      "inner": 18,
      "median_ms": 2.7500409446980076,
      "q1_ms": 2.7500153333272057,
      "q3_ms": 2.7519200553393195
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 128,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.005375336389988661,
      "compile_s": 0.5480516590032494,
      "inner": 10,
      "median_ms": 4.542291499092244,
      "q1_ms": 4.540236599859782,
      "q3_ms": 4.54549349960871
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 16,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.005375336389988661,
      "compile_s": 0.4862018770072609,
      "inner": 4,
      "median_ms": 10.491261247807415,
      "q1_ms": 10.467235752003035,
      "q3_ms": 10.493465499166632
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 16,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.005375336389988661,
      "compile_s": 0.3979700109921396,
      "inner": 5,
      "median_ms": 8.800451798015274,
      "q1_ms": 8.75066959997639,
      "q3_ms": 8.807096199598163
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 16,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.005375336389988661,
      "compile_s": 0.4882381380011793,
      "inner": 6,
      "median_ms": 7.546857833707084,
      "q1_ms": 7.542414333632526,
      "q3_ms": 7.557955500184714
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 16,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.005375336389988661,
      "compile_s": 0.4033311800012598,
      "inner": 5,
      "median_ms": 8.401154601597227,
      "q1_ms": 8.365833602147177,
      "q3_ms": 8.40574320172891
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 32,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.005375336389988661,
      "compile_s": 0.5265422200027388,
      "inner": 12,
      "median_ms": 3.719711500404325,
      "q1_ms": 3.71943591623373,
      "q3_ms": 3.723066999858323
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 32,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.005375336389988661,
      "compile_s": 0.4209365849965252,
      "inner": 8,
      "median_ms": 5.642910749884322,
      "q1_ms": 5.641703624860384,
      "q3_ms": 5.669256874170969
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 32,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.005375336389988661,
      "compile_s": 0.5517988659994444,
      "inner": 9,
      "median_ms": 5.079319222002393,
      "q1_ms": 5.044950110863687,
      "q3_ms": 5.083852555900294
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 32,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.005375336389988661,
      "compile_s": 0.43113329898915254,
      "inner": 9,
      "median_ms": 5.474642666134362,
      "q1_ms": 5.472260444851902,
      "q3_ms": 5.479552999733844
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 64,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.005375336389988661,
      "compile_s": 0.6495157899917103,
      "inner": 14,
      "median_ms": 3.432971071950825,
      "q1_ms": 3.4250611435189575,
      "q3_ms": 3.443877427863689
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 64,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.005375336389988661,
      "compile_s": 0.4666821100108791,
      "inner": 11,
      "median_ms": 4.178457907983102,
      "q1_ms": 4.175575091556476,
      "q3_ms": 4.17864463551351
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 64,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.005375336389988661,
      "compile_s": 0.6629981180012692,
      "inner": 14,
      "median_ms": 3.376900784912453,
      "q1_ms": 3.3760785705193745,
      "q3_ms": 3.378307713969012
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 64,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.005375336389988661,
      "compile_s": 0.49234191900177393,
      "inner": 12,
      "median_ms": 3.942142917367164,
      "q1_ms": 3.9384426672768313,
      "q3_ms": 3.9617309169746777
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 128,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.005375336389988661,
      "compile_s": 1.1744652780034812,
      "inner": 7,
      "median_ms": 6.489238285991762,
      "q1_ms": 6.464099571790679,
      "q3_ms": 6.508715286534945
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 128,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.005375336389988661,
      "compile_s": 0.6087720149953384,
      "inner": 13,
      "median_ms": 3.7980359999122673,
      "q1_ms": 3.793613000128132,
      "q3_ms": 3.8016898459038484
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 128,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.005375336389988661,
      "compile_s": 1.3220261889946414,
      "inner": 6,
      "median_ms": 7.151895835704636,
      "q1_ms": 7.147682833116657,
      "q3_ms": 7.155322333953033
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 128,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.005375336389988661,
      "compile_s": 0.6390573709941236,
      "inner": 14,
      "median_ms": 3.3060260008954043,
      "q1_ms": 3.2935545719478148,
      "q3_ms": 3.306364500271489
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "auto",
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.19175155500124674,
      "inner": 5,
      "median_ms": 8.455120399594307,
      "q1_ms": 8.395491799456067,
      "q3_ms": 8.46168680000119
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "auto",
      "rel_err": 0.0038456202019006014,
      "compile_s": 0.46634554500633385,
      "inner": 1,
      "median_ms": 38.88530000403989,
      "q1_ms": 38.876152990269475,
      "q3_ms": 39.207751004141755
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 32,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.1428719419927802,
      "inner": 4,
      "median_ms": 11.943177247303538,
      "q1_ms": 11.929268501262413,
      "q3_ms": 12.000580751191592
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 32,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.13700652099214494,
      "inner": 2,
      "median_ms": 20.482617495872546,
      "q1_ms": 20.386131502164062,
      "q3_ms": 20.600229501724243
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 32,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.14771805801137816,
      "inner": 4,
      "median_ms": 11.99681599973701,
      "q1_ms": 11.991561248578364,
      "q3_ms": 12.004478248854866
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 32,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.1407451109989779,
      "inner": 2,
      "median_ms": 20.19641499646241,
      "q1_ms": 20.161195505352225,
      "q3_ms": 20.304771001974586
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 32,
      "num_stages": 3,
      "num_warps": 4,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.1530790969991358,
      "inner": 4,
      "median_ms": 13.041397000051802,
      "q1_ms": 13.01642425096361,
      "q3_ms": 13.05731424872647
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 32,
      "num_stages": 3,
      "num_warps": 8,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.14802263399178628,
      "inner": 2,
      "median_ms": 20.781792503839824,
      "q1_ms": 20.752311997057404,
      "q3_ms": 20.79248749942053
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 64,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.1565089630021248,
      "inner": 5,
      "median_ms": 10.358999200980179,
      "q1_ms": 10.34145979792811,
      "q3_ms": 10.408354399260134
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 64,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.14570103300502524,
      "inner": 3,
      "median_ms": 15.27159466544011,
      "q1_ms": 15.26340733107645,
      "q3_ms": 15.317249000266505
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 64,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.1618761940044351,
      "inner": 5,
      "median_ms": 10.472680200473405,
      "q1_ms": 10.471780199441127,
      "q3_ms": 10.48348500044085
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 64,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.14976101199863479,
      "inner": 3,
      "median_ms": 14.93542299916347,
      "q1_ms": 14.933679000629732,
      "q3_ms": 14.941708330297843
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 64,
      "num_stages": 3,
      "num_warps": 4,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.17457658299827017,
      "inner": 4,
      "median_ms": 11.898101998667698,
      "q1_ms": 11.881686001288472,
      "q3_ms": 12.000810998870293
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 64,
      "num_stages": 3,
      "num_warps": 8,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.1558712379919598,
      "inner": 3,
      "median_ms": 15.616447330103256,
      "q1_ms": 15.601893663794423,
      "q3_ms": 15.650371996647056
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 128,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.1961390540091088,
      "inner": 5,
      "median_ms": 10.409572799107991,
      "q1_ms": 10.380431800149381,
      "q3_ms": 10.412891002488323
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 128,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.16008720100217033,
      "inner": 4,
      "median_ms": 13.355002251046244,
      "q1_ms": 13.352665249840356,
      "q3_ms": 13.543896751798457
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 128,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.21251704799942672,
      "inner": 3,
      "median_ms": 16.013768666501466,
      "q1_ms": 15.974932670360431,
      "q3_ms": 16.24644233379513
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 128,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.17688217300747056,
      "inner": 3,
      "median_ms": 16.8584560015006,
      "q1_ms": 16.796847000174846,
      "q3_ms": 16.971473000012338
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 128,
      "num_stages": 3,
      "num_warps": 4,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.22123558601015247,
      "inner": 3,
      "median_ms": 13.064899001619779,
      "q1_ms": 12.956773998060575,
      "q3_ms": 13.112328335409984
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 128,
      "num_stages": 3,
      "num_warps": 8,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.18198146799113601,
      "inner": 3,
      "median_ms": 15.238970333787924,
      "q1_ms": 15.216787665849552,
      "q3_ms": 15.302239999679538
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 32,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.15493836300447583,
      "inner": 6,
      "median_ms": 8.611776667142598,
      "q1_ms": 8.591152834317958,
      "q3_ms": 8.618961997854058
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 32,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.14865086200006772,
      "inner": 3,
      "median_ms": 16.894377336332884,
      "q1_ms": 16.808903334701125,
      "q3_ms": 16.93172500138947
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 32,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.15362144200480543,
      "inner": 6,
      "median_ms": 8.671409501403105,
      "q1_ms": 8.657009166199714,
      "q3_ms": 8.677243832304763
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 32,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.1570780189940706,
      "inner": 3,
      "median_ms": 16.866728003757697,
      "q1_ms": 16.859301331957493,
      "q3_ms": 16.918029000711005
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 32,
      "num_stages": 3,
      "num_warps": 4,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.1605792409973219,
      "inner": 6,
      "median_ms": 8.627428166316045,
      "q1_ms": 8.59783166621734,
      "q3_ms": 8.640700000493476
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 32,
      "num_stages": 3,
      "num_warps": 8,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.1582175039948197,
      "inner": 3,
      "median_ms": 16.530159002286382,
      "q1_ms": 16.50272699771449,
      "q3_ms": 16.55873666459229
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 64,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.17702072599786334,
      "inner": 6,
      "median_ms": 8.450653665931895,
      "q1_ms": 8.440866998474425,
      "q3_ms": 8.454048499212755
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 64,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.18027861799055245,
      "inner": 2,
      "median_ms": 18.9963475058903,
      "q1_ms": 18.940690002636984,
      "q3_ms": 19.024299501325004
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 64,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.18228669499512762,
      "inner": 6,
      "median_ms": 8.397793666517828,
      "q1_ms": 8.378727834497113,
      "q3_ms": 8.39947183461239
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 64,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.185954592001508,
      "inner": 2,
      "median_ms": 18.44426699972246,
      "q1_ms": 18.426433998683933,
      "q3_ms": 18.44488799542887
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 64,
      "num_stages": 3,
      "num_warps": 4,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.19324210500053596,
      "inner": 6,
      "median_ms": 8.50215949806928,
      "q1_ms": 8.490819668319697,
      "q3_ms": 8.568979001817448
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 64,
      "num_stages": 3,
      "num_warps": 8,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.1899226959940279,
      "inner": 3,
      "median_ms": 17.45593666661686,
      "q1_ms": 17.440824667573906,
      "q3_ms": 17.477897670081195
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 128,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.2597575600084383,
      "inner": 5,
      "median_ms": 8.803308999631554,
      "q1_ms": 8.794107800349593,
      "q3_ms": 8.809312400990166
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 128,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.2397068800055422,
      "inner": 2,
      "median_ms": 18.55960000102641,
      "q1_ms": 18.559144999017008,
      "q3_ms": 18.56124850019114
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 128,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.26384549199428875,
      "inner": 4,
      "median_ms": 10.336185499909334,
      "q1_ms": 10.309497247362742,
      "q3_ms": 10.363051751482999
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 128,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.24748033100331668,
      "inner": 2,
      "median_ms": 17.286397000134457,
      "q1_ms": 17.283001994655933,
      "q3_ms": 17.30881899857195
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 128,
      "num_stages": 3,
      "num_warps": 4,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.2635955890000332,
      "inner": 5,
      "median_ms": 9.173817801638506,
      "q1_ms": 9.147111000493169,
      "q3_ms": 9.176941798068583
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 128,
      "num_stages": 3,
      "num_warps": 8,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.25595886599330697,
      "inner": 3,
      "median_ms": 16.686201333262336,
      "q1_ms": 16.600451664999127,
      "q3_ms": 16.722031665267423
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 32,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.20759465798619203,
      "inner": 5,
      "median_ms": 8.60838060034439,
      "q1_ms": 8.583103999262676,
      "q3_ms": 8.692855798290111
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 32,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.14880698699562345,
      "inner": 6,
      "median_ms": 8.552197332998427,
      "q1_ms": 8.540405166665247,
      "q3_ms": 8.555009497892266
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 32,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.21326096401026007,
      "inner": 6,
      "median_ms": 8.497135000652634,
      "q1_ms": 8.496088000053229,
      "q3_ms": 8.507287500833627
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 32,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.1534750149876345,
      "inner": 6,
      "median_ms": 8.390842333028559,
      "q1_ms": 8.374396165891085,
      "q3_ms": 8.4190323347381
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 32,
      "num_stages": 3,
      "num_warps": 4,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.21618321099958848,
      "inner": 6,
      "median_ms": 8.204853831557557,
      "q1_ms": 8.196272666585477,
      "q3_ms": 8.217015165428165
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 32,
      "num_stages": 3,
      "num_warps": 8,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.15709063400572632,
      "inner": 6,
      "median_ms": 8.372621002005568,
      "q1_ms": 8.359606833740449,
      "q3_ms": 8.424731500175161
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 64,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.26540566098992713,
      "inner": 6,
      "median_ms": 8.220041000943942,
      "q1_ms": 8.20756733204083,
      "q3_ms": 8.246908168075606
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 64,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.17889059300068766,
      "inner": 5,
      "median_ms": 9.979469201061875,
      "q1_ms": 9.974598401458934,
      "q3_ms": 9.983454801840708
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 64,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.2708079279982485,
      "inner": 6,
      "median_ms": 7.975956000639902,
      "q1_ms": 7.96666499809362,
      "q3_ms": 8.008315499561528
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 64,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.18243724999774713,
      "inner": 5,
      "median_ms": 9.121741200215183,
      "q1_ms": 9.094349600491114,
      "q3_ms": 9.12406359857414
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 64,
      "num_stages": 3,
      "num_warps": 4,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.27715884998906404,
      "inner": 6,
      "median_ms": 8.501150999412252,
      "q1_ms": 8.493584832952669,
      "q3_ms": 8.503765835484955
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 64,
      "num_stages": 3,
      "num_warps": 8,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.1862242120114388,
      "inner": 6,
      "median_ms": 8.579260499876304,
      "q1_ms": 8.55337483274828,
      "q3_ms": 8.617155166575685
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 128,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.4260259809962008,
      "inner": 5,
      "median_ms": 9.146147198043764,
      "q1_ms": 9.142073601833545,
      "q3_ms": 9.148256000480615
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 128,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.23557139800686855,
      "inner": 5,
      "median_ms": 9.319184999912977,
      "q1_ms": 9.31077879795339,
      "q3_ms": 9.341190397390164
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 128,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.46903219498926774,
      "inner": 4,
      "median_ms": 10.262964748108061,
      "q1_ms": 10.25041574757779,
      "q3_ms": 10.278874749928946
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 128,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.2464739770075539,
      "inner": 5,
      "median_ms": 8.777203998761252,
      "q1_ms": 8.755794999888167,
      "q3_ms": 8.786667199456133
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 128,
      "num_stages": 3,
      "num_warps": 4,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.6062964029988507,
      "inner": 4,
      "median_ms": 12.29656549912761,
      "q1_ms": 12.28890100173885,
      "q3_ms": 12.297925750317518
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 128,
      "num_stages": 3,
      "num_warps": 8,
      "rel_err": 0.0027605073992162943,
      "compile_s": 0.249844892998226,
      "inner": 6,
      "median_ms": 8.426530000482066,
      "q1_ms": 8.414377000008244,
      "q3_ms": 8.459724165732041
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 16,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0038456202019006014,
      "compile_s": 0.2890827600058401,
      "inner": 1,
      "median_ms": 59.495848006918095,
      "q1_ms": 59.49571898963768,
      "q3_ms": 59.592213001451455
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 16,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0038456202019006014,
      "compile_s": 0.3179171929950826,
      "inner": 1,
      "median_ms": 96.75063799659256,
      "q1_ms": 96.31458300282247,
      "q3_ms": 97.62511799635831
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 16,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0038456202019006014,
      "compile_s": 0.30244323999795597,
      "inner": 1,
      "median_ms": 53.49405999004375,
      "q1_ms": 53.19325000164099,
      "q3_ms": 53.83053599507548
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 16,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0038456202019006014,
      "compile_s": 0.34400019100576174,
      "inner": 1,
      "median_ms": 100.06156899908092,
      "q1_ms": 100.00807799224276,
      "q3_ms": 100.21358700760175
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 32,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0038456202019006014,
      "compile_s": 0.2962622310005827,
      "inner": 1,
      "median_ms": 50.197252989164554,
      "q1_ms": 50.09857800905593,
      "q3_ms": 50.238702999195084
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 32,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0038456202019006014,
      "compile_s": 0.304980136002996,
      "inner": 1,
      "median_ms": 82.67927600536495,
      "q1_ms": 82.15750700037461,
      "q3_ms": 83.1647909944877
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 32,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0038456202019006014,
      "compile_s": 0.3096101529954467,
      "inner": 1,
      "median_ms": 49.211060992092825,
      "q1_ms": 49.10367701086216,
      "q3_ms": 49.31436599872541
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 32,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0038456202019006014,
      "compile_s": 0.3240274600102566,
      "inner": 1,
      "median_ms": 78.16298399120569,
      "q1_ms": 78.0783939990215,
      "q3_ms": 78.58348300214857
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 64,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0038456202019006014,
      "compile_s": 0.3522962810093304,
      "inner": 1,
      "median_ms": 49.45418999704998,
      "q1_ms": 49.396640999475494,
      "q3_ms": 49.47719001211226
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 64,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0038456202019006014,
      "compile_s": 0.316107179998653,
      "inner": 1,
      "median_ms": 71.14067299698945,
      "q1_ms": 71.08774299558718,
      "q3_ms": 71.17400599236134
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 64,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0038456202019006014,
      "compile_s": 0.3592669050121913,
      "inner": 1,
      "median_ms": 48.33550400508102,
      "q1_ms": 48.16667399427388,
      "q3_ms": 48.446553002577275
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 64,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0038456202019006014,
      "compile_s": 0.3302703559893416,
      "inner": 1,
      "median_ms": 68.39985899569001,
      "q1_ms": 68.29971900151577,
      "q3_ms": 69.01899099466391
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 128,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0038456202019006014,
      "compile_s": 0.4621368549996987,
      "inner": 1,
      "median_ms": 50.048463002895005,
      "q1_ms": 50.01408600946888,
      "q3_ms": 50.11775600723922
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 128,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0038456202019006014,
      "compile_s": 0.39941869099857286,
      "inner": 1,
      "median_ms": 64.92831200012006,
      "q1_ms": 64.80019199079834,
      "q3_ms": 65.14314000378363
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 128,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0038456202019006014,
      "compile_s": 0.46931884800142143,
      "inner": 1,
      "median_ms": 48.71340899262577,
      "q1_ms": 48.666828995919786,
      "q3_ms": 48.72952899313532
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 128,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0038456202019006014,
      "compile_s": 0.41332683399377856,
      "inner": 1,
      "median_ms": 68.27312000677921,
      "q1_ms": 68.01688599807676,
      "q3_ms": 68.31117799447384
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 16,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0038456202019006014,
      "compile_s": 0.37487249900004826,
      "inner": 1,
      "median_ms": 67.41059701016638,
      "q1_ms": 67.28329700126778,
      "q3_ms": 67.86547999945469
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 16,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0038456202019006014,
      "compile_s": 0.4485811990016373,
      "inner": 1,
      "median_ms": 155.41058601229452,
      "q1_ms": 155.3156570007559,
      "q3_ms": 155.60602600453421
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 16,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0038456202019006014,
      "compile_s": 0.39078917101141997,
      "inner": 1,
      "median_ms": 64.69509001180995,
      "q1_ms": 64.46788400353398,
      "q3_ms": 64.70865198934916
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 16,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0038456202019006014,
      "compile_s": 0.4511238669947488,
      "inner": 1,
      "median_ms": 141.0665900039021,
      "q1_ms": 140.86002799740527,
      "q3_ms": 141.13809500122443
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 32,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0038456202019006014,
      "compile_s": 0.3931363589945249,
      "inner": 1,
      "median_ms": 49.51307000010274,
      "q1_ms": 49.08651299774647,
      "q3_ms": 49.523120003868826
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 32,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0038456202019006014,
      "compile_s": 0.43819874299515504,
      "inner": 1,
      "median_ms": 109.58465900330339,
      "q1_ms": 109.54521200619638,
      "q3_ms": 109.7042730107205
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 32,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0038456202019006014,
      "compile_s": 0.40404849698825274,
      "inner": 1,
      "median_ms": 47.12059399753343,
      "q1_ms": 46.853029998601414,
      "q3_ms": 47.27349599124864
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 32,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0038456202019006014,
      "compile_s": 0.43841386899293866,
      "inner": 1,
      "median_ms": 95.46242500073276,
      "q1_ms": 95.4007139953319,
      "q3_ms": 95.6855089898454
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 64,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0038456202019006014,
      "compile_s": 0.45506851900427137,
      "inner": 1,
      "median_ms": 40.79256900877226,
      "q1_ms": 40.789892998873256,
      "q3_ms": 40.94696100219153
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 64,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0038456202019006014,
      "compile_s": 0.4692745040083537,
      "inner": 1,
      "median_ms": 84.41761399444658,
      "q1_ms": 84.38712400675286,
      "q3_ms": 84.54674699169118
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 64,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0038456202019006014,
      "compile_s": 0.4691683029959677,
      "inner": 1,
      "median_ms": 39.572385998326354,
      "q1_ms": 39.55251300067175,
      "q3_ms": 39.646574994549155
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 64,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0038456202019006014,
      "compile_s": 0.456935758003965,
      "inner": 1,
      "median_ms": 75.17468299192842,
      "q1_ms": 75.02951700007543,
      "q3_ms": 75.24687898694538
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 128,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0038456202019006014,
      "compile_s": 0.6861790389957605,
      "inner": 1,
      "median_ms": 54.52397699991707,
      "q1_ms": 54.44737100333441,
      "q3_ms": 54.60984099772759
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 128,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0038456202019006014,
      "compile_s": 0.5895896860020002,
      "inner": 1,
      "median_ms": 83.25018899631687,
      "q1_ms": 82.99023400468286,
      "q3_ms": 83.82440100831445
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 128,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0038456202019006014,
      "compile_s": 0.6728326180018485,
      "inner": 1,
      "median_ms": 41.046761005418375,
      "q1_ms": 40.906573995016515,
      "q3_ms": 41.10539700195659
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 128,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0038456202019006014,
      "compile_s": 0.6121396029921016,
      "inner": 1,
      "median_ms": 68.80936400557403,
      "q1_ms": 68.60913700074889,
      "q3_ms": 69.31776800774969
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 16,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0038456202019006014,
      "compile_s": 0.6045516949961893,
      "inner": 1,
      "median_ms": 133.78636000561528,
      "q1_ms": 133.5893780051265,
      "q3_ms": 133.94621199404355
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 16,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0038456202019006014,
      "compile_s": 0.5211549230007222,
      "inner": 1,
      "median_ms": 128.34583199582994,
      "q1_ms": 128.29330000386108,
      "q3_ms": 128.43958000303246
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 16,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0038456202019006014,
      "compile_s": 0.5858910620008828,
      "inner": 1,
      "median_ms": 101.9579660060117,
      "q1_ms": 101.81691899197176,
      "q3_ms": 102.12730499915779
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 16,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0038456202019006014,
      "compile_s": 0.5232053780055139,
      "inner": 1,
      "median_ms": 123.3806119998917,
      "q1_ms": 123.30923399713356,
      "q3_ms": 123.54327899811324
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 32,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0038456202019006014,
      "compile_s": 0.5816506120027043,
      "inner": 1,
      "median_ms": 52.372196994838305,
      "q1_ms": 52.321665003546514,
      "q3_ms": 52.861363990814425
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 32,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0038456202019006014,
      "compile_s": 0.49303859600331634,
      "inner": 1,
      "median_ms": 82.57113098807167,
      "q1_ms": 82.14751799823716,
      "q3_ms": 82.74816299672239
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 32,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0038456202019006014,
      "compile_s": 0.6128220230020816,
      "inner": 1,
      "median_ms": 66.86892500147223,
      "q1_ms": 66.80363300256431,
      "q3_ms": 67.22611999430228
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 32,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0038456202019006014,
      "compile_s": 0.5117316599935293,
      "inner": 1,
      "median_ms": 80.66782300011255,
      "q1_ms": 80.36252099554986,
      "q3_ms": 80.78878901142161
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 64,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0038456202019006014,
      "compile_s": 0.6947778909961926,
      "inner": 1,
      "median_ms": 50.32104899873957,
      "q1_ms": 50.280134004424326,
      "q3_ms": 50.38759201124776
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 64,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0038456202019006014,
      "compile_s": 0.5236444790061796,
      "inner": 1,
      "median_ms": 61.33797600341495,
      "q1_ms": 60.91050599934533,
      "q3_ms": 61.4112409966765
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 64,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0038456202019006014,
      "compile_s": 0.7181648510013474,
      "inner": 1,
      "median_ms": 48.86546399211511,
      "q1_ms": 48.84005700296257,
      "q3_ms": 49.06335200939793
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 64,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0038456202019006014,
      "compile_s": 0.5492674309934955,
      "inner": 1,
      "median_ms": 57.8208429942606,
      "q1_ms": 57.778892005444504,
      "q3_ms": 57.91619299270678
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 128,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0038456202019006014,
      "compile_s": 1.28091402699647,
      "inner": 1,
      "median_ms": 115.58760100160725,
      "q1_ms": 115.5618929915363,
      "q3_ms": 115.7142009906238
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 128,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0038456202019006014,
      "compile_s": 0.6510332799953176,
      "inner": 1,
      "median_ms": 56.2247439956991,
      "q1_ms": 56.023833007202484,
      "q3_ms": 56.269779001013376
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 128,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0038456202019006014,
      "compile_s": 1.400863749004202,
      "inner": 1,
      "median_ms": 108.72342000948265,
      "q1_ms": 108.4486210020259,
      "q3_ms": 108.82404199219309
    },
    {
      "dtype": "bfloat16",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 128,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0038456202019006014,
      "compile_s": 0.6796256749948952,
      "inner": 1,
      "median_ms": 48.508248990401626,
      "q1_ms": 48.28270399593748,
      "q3_ms": 48.53771600755863
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "auto",
      "rel_err": 0.0028994381427764893,
      "compile_s": 0.24389969899493735,
      "inner": 29,
      "median_ms": 1.2843068277639949,
      "q1_ms": 1.283617275726885,
      "q3_ms": 1.2844688623364822
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "auto",
      "rel_err": 0.0036413150373846292,
      "compile_s": 0.48698486799548846,
      "inner": 9,
      "median_ms": 5.3386576667738455,
      "q1_ms": 5.33674633415002,
      "q3_ms": 5.3433087774060875
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 32,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0028994381427764893,
      "compile_s": 0.15759573900140822,
      "inner": 37,
      "median_ms": 1.3444377565630585,
      "q1_ms": 1.339891054305701,
      "q3_ms": 1.3447061351758454
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 32,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0028994381427764893,
      "compile_s": 0.13904484499653336,
      "inner": 25,
      "median_ms": 2.028049520449713,
      "q1_ms": 2.027912840130739,
      "q3_ms": 2.0323111600009724
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 32,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0028994381427764893,
      "compile_s": 0.16487070800212678,
      "inner": 25,
      "median_ms": 1.5238816000055522,
      "q1_ms": 1.5237461595097557,
      "q3_ms": 1.5292645996669307
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 32,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0028994381427764893,
      "compile_s": 0.14170064798963722,
      "inner": 23,
      "median_ms": 2.157020695765665,
      "q1_ms": 2.156482304638738,
      "q3_ms": 2.157464999902953
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 32,
      "num_stages": 3,
      "num_warps": 4,
      "rel_err": 0.0028994381427764893,
      "compile_s": 0.17265378800220788,
      "inner": 26,
      "median_ms": 1.8741875767143656,
      "q1_ms": 1.8721907304447645,
      "q3_ms": 1.879875653965148
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 32,
      "num_stages": 3,
      "num_warps": 8,
      "rel_err": 0.0028994381427764893,
      "compile_s": 0.1494902690028539,
      "inner": 18,
      "median_ms": 2.6299031661538823,
      "q1_ms": 2.624720611493103,
      "q3_ms": 2.6370105002165976
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 64,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0028994381427764893,
      "compile_s": 0.19514838999020867,
      "inner": 38,
      "median_ms": 1.2541147102877546,
      "q1_ms": 1.2538855792770751,
      "q3_ms": 1.258340079067765
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 64,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0028994381427764893,
      "compile_s": 0.15129303900175728,
      "inner": 34,
      "median_ms": 1.4764596763758593,
      "q1_ms": 1.4758691468159668,
      "q3_ms": 1.4773602058138589
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 64,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0028994381427764893,
      "compile_s": 0.20196706500428263,
      "inner": 24,
      "median_ms": 1.9493854585258912,
      "q1_ms": 1.949150874982782,
      "q3_ms": 1.9578389583330136
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 64,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0028994381427764893,
      "compile_s": 0.1608557639992796,
      "inner": 21,
      "median_ms": 2.185645904752337,
      "q1_ms": 2.1855739524610165,
      "q3_ms": 2.1863305238574475
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 64,
      "num_stages": 3,
      "num_warps": 4,
      "rel_err": 0.0028994381427764893,
      "compile_s": 0.20684537099441513,
      "inner": 31,
      "median_ms": 1.5894566773391898,
      "q1_ms": 1.5887799034375818,
      "q3_ms": 1.5908260322776773
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 64,
      "num_stages": 3,
      "num_warps": 8,
      "rel_err": 0.0028994381427764893,
      "compile_s": 0.16630276599607896,
      "inner": 24,
      "median_ms": 1.90466049995545,
      "q1_ms": 1.9045798750691272,
      "q3_ms": 1.9071230417466722
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 128,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0028994381427764893,
      "compile_s": 0.25008388501009904,
      "inner": 34,
      "median_ms": 1.4126790586434415,
      "q1_ms": 1.4120602350243752,
      "q3_ms": 1.412709676729077
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 128,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0028994381427764893,
      "compile_s": 0.18665629900351632,
      "inner": 32,
      "median_ms": 1.5138865310291294,
      "q1_ms": 1.5138718122216233,
      "q3_ms": 1.5145305624173488
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 128,
      "num_stages": 2,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 114688, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 128,
      "num_stages": 2,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 114688, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 128,
      "num_stages": 3,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 180224, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 128,
      "num_stages": 3,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 180224, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 32,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0028994381427764893,
      "compile_s": 0.23430486299912445,
      "inner": 37,
      "median_ms": 1.280361702712882,
      "q1_ms": 1.2801973510225226,
      "q3_ms": 1.2803999187953368
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 32,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0028994381427764893,
      "compile_s": 0.19333626299339812,
      "inner": 24,
      "median_ms": 2.033163499921405,
      "q1_ms": 2.03079108359816,
      "q3_ms": 2.048605208377315
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 32,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0028994381427764893,
      "compile_s": 0.24082917200576048,
      "inner": 36,
      "median_ms": 1.3202168330382038,
      "q1_ms": 1.3200122496831075,
      "q3_ms": 1.321109888926407
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 32,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0028994381427764893,
      "compile_s": 0.20106491800106596,
      "inner": 24,
      "median_ms": 2.0524199165568766,
      "q1_ms": 2.039946333146266,
      "q3_ms": 2.056603208378268
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 32,
      "num_stages": 3,
      "num_warps": 4,
      "rel_err": 0.0028994381427764893,
      "compile_s": 0.259040663993801,
      "inner": 30,
      "median_ms": 1.5642613667296246,
      "q1_ms": 1.5641391000826843,
      "q3_ms": 1.5644514331749328
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 32,
      "num_stages": 3,
      "num_warps": 8,
      "rel_err": 0.0028994381427764893,
      "compile_s": 0.20185442200454418,
      "inner": 21,
      "median_ms": 2.3215023809046085,
      "q1_ms": 2.318426619500055,
      "q3_ms": 2.321975189954087
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 64,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0028994381427764893,
      "compile_s": 0.28484113900049124,
      "inner": 37,
      "median_ms": 1.2251486485773646,
      "q1_ms": 1.2246425675756827,
      "q3_ms": 1.225928486501043
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 64,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0028994381427764893,
      "compile_s": 0.22512901898880955,
      "inner": 34,
      "median_ms": 1.4000669706386841,
      "q1_ms": 1.39962670586743,
      "q3_ms": 1.40121352930745
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 64,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0028994381427764893,
      "compile_s": 0.29231587500544265,
      "inner": 26,
      "median_ms": 1.531633961489066,
      "q1_ms": 1.5313661541977825,
      "q3_ms": 1.5336462305500531
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 64,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0028994381427764893,
      "compile_s": 0.2271468060062034,
      "inner": 25,
      "median_ms": 1.8554102000780404,
      "q1_ms": 1.8551500799367204,
      "q3_ms": 1.8566889996873215
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 64,
      "num_stages": 3,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 106496, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 64,
      "num_stages": 3,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 106496, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 128,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0028994381427764893,
      "compile_s": 0.3979726449906593,
      "inner": 27,
      "median_ms": 1.5113505182347984,
      "q1_ms": 1.5096914818665634,
      "q3_ms": 1.5116685185434848
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 128,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0028994381427764893,
      "compile_s": 0.2651212139899144,
      "inner": 28,
      "median_ms": 1.6307133212519278,
      "q1_ms": 1.6306789646997848,
      "q3_ms": 1.6311302138741925
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 128,
      "num_stages": 2,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 131072, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 128,
      "num_stages": 2,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 131072, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 128,
      "num_stages": 3,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 196608, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 128,
      "num_stages": 3,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 196608, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 32,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0028994381427764893,
      "compile_s": 0.32071581498894375,
      "inner": 36,
      "median_ms": 1.1358050275399971,
      "q1_ms": 1.1356647499067345,
      "q3_ms": 1.135966166556399
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 32,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0028994381427764893,
      "compile_s": 0.18592462500964757,
      "inner": 33,
      "median_ms": 1.3610407269990186,
      "q1_ms": 1.3600731212081332,
      "q3_ms": 1.3610810909250919
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 32,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0028994381427764893,
      "compile_s": 0.3124373589962488,
      "inner": 35,
      "median_ms": 1.2848714000678487,
      "q1_ms": 1.2804862860191082,
      "q3_ms": 1.285262142690564
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 32,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0028994381427764893,
      "compile_s": 0.19585181000002194,
      "inner": 33,
      "median_ms": 1.2164943032213629,
      "q1_ms": 1.2159493331141262,
      "q3_ms": 1.2167332425826425
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 32,
      "num_stages": 3,
      "num_warps": 4,
      "rel_err": 0.0028994381427764893,
      "compile_s": 0.336650571989594,
      "inner": 38,
      "median_ms": 1.1455538946105854,
      "q1_ms": 1.1453102365761112,
      "q3_ms": 1.1459198155391372
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 32,
      "num_stages": 3,
      "num_warps": 8,
      "rel_err": 0.0028994381427764893,
      "compile_s": 0.20068651299516205,
      "inner": 39,
      "median_ms": 1.215988692143956,
      "q1_ms": 1.2158920771281163,
      "q3_ms": 1.2162427435843992
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 64,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0028994381427764893,
      "compile_s": 0.4269605279987445,
      "inner": 21,
      "median_ms": 2.249150809283102,
      "q1_ms": 2.2486255239484656,
      "q3_ms": 2.2533992852827183
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 64,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0028994381427764893,
      "compile_s": 0.2463098070002161,
      "inner": 35,
      "median_ms": 1.3008510569177036,
      "q1_ms": 1.300299457008285,
      "q3_ms": 1.300958399744039
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 64,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0028994381427764893,
      "compile_s": 0.4292056539998157,
      "inner": 37,
      "median_ms": 1.1932642971530456,
      "q1_ms": 1.1922629460428111,
      "q3_ms": 1.1933176484072228
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 64,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0028994381427764893,
      "compile_s": 0.23998018600104842,
      "inner": 40,
      "median_ms": 1.1769535998610081,
      "q1_ms": 1.1759612498281058,
      "q3_ms": 1.177007499791216
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 64,
      "num_stages": 3,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 114688, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 64,
      "num_stages": 3,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 114688, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 128,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0028994381427764893,
      "compile_s": 0.7165157720010029,
      "inner": 13,
      "median_ms": 3.6842939231204443,
      "q1_ms": 3.683088614847153,
      "q3_ms": 3.686037307488732
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 128,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0028994381427764893,
      "compile_s": 0.35550647201307584,
      "inner": 27,
      "median_ms": 1.6139627405209467,
      "q1_ms": 1.6124662220978212,
      "q3_ms": 1.614136000474294
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 128,
      "num_stages": 2,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 131072, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 128,
      "num_stages": 2,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 131072, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 128,
      "num_stages": 3,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 196608, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 128,
      "num_stages": 3,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 196608, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 16,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0036413150373846292,
      "compile_s": 0.5093660959973931,
      "inner": 6,
      "median_ms": 7.988346168228114,
      "q1_ms": 7.984471999710271,
      "q3_ms": 7.989561832800973
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 16,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0036413150373846292,
      "compile_s": 0.3076326520094881,
      "inner": 5,
      "median_ms": 10.498114401707426,
      "q1_ms": 10.49597020028159,
      "q3_ms": 10.501314399880357
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 16,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0036413150373846292,
      "compile_s": 0.520152404991677,
      "inner": 7,
      "median_ms": 6.318990571474258,
      "q1_ms": 6.3180659989094625,
      "q3_ms": 6.327811715891585
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 16,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0036413150373846292,
      "compile_s": 0.31864824499643873,
      "inner": 5,
      "median_ms": 9.671849600272253,
      "q1_ms": 9.64978159754537,
      "q3_ms": 9.67186999914702
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 32,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0036413150373846292,
      "compile_s": 0.5072687490028329,
      "inner": 7,
      "median_ms": 6.9585962857153,
      "q1_ms": 6.917431000536973,
      "q3_ms": 6.9792619989519675
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 32,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0036413150373846292,
      "compile_s": 0.3070330580085283,
      "inner": 6,
      "median_ms": 7.824414499433866,
      "q1_ms": 7.81665816612076,
      "q3_ms": 7.855753499219039
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 32,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0036413150373846292,
      "compile_s": 0.510082730994327,
      "inner": 9,
      "median_ms": 5.4738244458955405,
      "q1_ms": 5.459945888206776,
      "q3_ms": 5.474731665647899
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 32,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0036413150373846292,
      "compile_s": 0.3157558249949943,
      "inner": 7,
      "median_ms": 7.5323155719421,
      "q1_ms": 7.505084857777027,
      "q3_ms": 7.544673000146369
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 64,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0036413150373846292,
      "compile_s": 0.5198563549929531,
      "inner": 6,
      "median_ms": 7.871054166268246,
      "q1_ms": 7.8508324998741355,
      "q3_ms": 7.911295833764598
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 64,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0036413150373846292,
      "compile_s": 0.3448933010076871,
      "inner": 7,
      "median_ms": 7.4210232872116775,
      "q1_ms": 7.419657999499967,
      "q3_ms": 7.462556286814755
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 64,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0036413150373846292,
      "compile_s": 0.5168546490021981,
      "inner": 8,
      "median_ms": 5.859575499925995,
      "q1_ms": 5.855279998286278,
      "q3_ms": 5.872874375199899
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 64,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0036413150373846292,
      "compile_s": 0.3439180059940554,
      "inner": 7,
      "median_ms": 7.160646714120438,
      "q1_ms": 7.15432600035066,
      "q3_ms": 7.1882388583617285
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 128,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0036413150373846292,
      "compile_s": 0.6244765349983936,
      "inner": 6,
      "median_ms": 7.628380000824109,
      "q1_ms": 7.604288664879277,
      "q3_ms": 7.6440161671295455
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 128,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0036413150373846292,
      "compile_s": 0.4621385780046694,
      "inner": 7,
      "median_ms": 7.382247428592693,
      "q1_ms": 7.3757361429410855,
      "q3_ms": 7.436351000381235
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 128,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0036413150373846292,
      "compile_s": 0.6464394319918938,
      "inner": 8,
      "median_ms": 6.320432124994113,
      "q1_ms": 6.311569000899908,
      "q3_ms": 6.321939876215765
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 128,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0036413150373846292,
      "compile_s": 0.47733894499833696,
      "inner": 7,
      "median_ms": 6.951565428088153,
      "q1_ms": 6.931308430336815,
      "q3_ms": 6.9519292862553685
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 16,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0036413150373846292,
      "compile_s": 0.5158971979981288,
      "inner": 6,
      "median_ms": 7.459355498819302,
      "q1_ms": 7.440500000181298,
      "q3_ms": 7.460636666413241
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 16,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0036413150373846292,
      "compile_s": 0.31612475200381596,
      "inner": 5,
      "median_ms": 10.046716601937078,
      "q1_ms": 10.041687201010063,
      "q3_ms": 10.056439001346007
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 16,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0036413150373846292,
      "compile_s": 0.5153592299902812,
      "inner": 8,
      "median_ms": 5.953848874924006,
      "q1_ms": 5.943103624304058,
      "q3_ms": 5.968144500002381
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 16,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0036413150373846292,
      "compile_s": 0.3325227049936075,
      "inner": 5,
      "median_ms": 8.909822400892153,
      "q1_ms": 8.90535400249064,
      "q3_ms": 8.912976400461048
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 32,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0036413150373846292,
      "compile_s": 0.5129756719979923,
      "inner": 7,
      "median_ms": 7.019518284193639,
      "q1_ms": 7.008342999532553,
      "q3_ms": 7.0296118584727605
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 32,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0036413150373846292,
      "compile_s": 0.3433225690096151,
      "inner": 7,
      "median_ms": 6.957581142030124,
      "q1_ms": 6.950729715754278,
      "q3_ms": 6.967468713160737
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 32,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0036413150373846292,
      "compile_s": 0.5160885990044335,
      "inner": 9,
      "median_ms": 5.405667554845826,
      "q1_ms": 5.3940446669002995,
      "q3_ms": 5.407239889286251
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 32,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0036413150373846292,
      "compile_s": 0.3635132449999219,
      "inner": 7,
      "median_ms": 6.555925713785525,
      "q1_ms": 6.5415830010481715,
      "q3_ms": 6.568569999737
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 64,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0036413150373846292,
      "compile_s": 0.6120571970095625,
      "inner": 7,
      "median_ms": 6.852838857282352,
      "q1_ms": 6.843761715572327,
      "q3_ms": 6.865936570518118
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 64,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0036413150373846292,
      "compile_s": 0.40596234699478373,
      "inner": 9,
      "median_ms": 5.65491811074834,
      "q1_ms": 5.631972445472557,
      "q3_ms": 5.656613554391597
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 64,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0036413150373846292,
      "compile_s": 0.6352723629970569,
      "inner": 9,
      "median_ms": 5.450199776937047,
      "q1_ms": 5.433343333101624,
      "q3_ms": 5.455621110741049
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 64,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0036413150373846292,
      "compile_s": 0.42003204599313904,
      "inner": 9,
      "median_ms": 5.490350110145907,
      "q1_ms": 5.482979555381462,
      "q3_ms": 5.507096221360068
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 128,
      "num_stages": 1,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 114688, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 128,
      "num_stages": 1,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 114688, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 128,
      "num_stages": 2,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 115712, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 128,
      "num_stages": 2,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 115712, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 16,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0036413150373846292,
      "compile_s": 0.8753498229925754,
      "inner": 2,
      "median_ms": 20.823917497182265,
      "q1_ms": 20.654425999964587,
      "q3_ms": 20.98687599936966
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 16,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0036413150373846292,
      "compile_s": 0.6207859959977213,
      "inner": 3,
      "median_ms": 15.112901669150839,
      "q1_ms": 15.067522666261842,
      "q3_ms": 15.136801332118921
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 16,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0036413150373846292,
      "compile_s": 0.7935162770008901,
      "inner": 2,
      "median_ms": 22.3196264996659,
      "q1_ms": 22.318278497550637,
      "q3_ms": 22.335360998113174
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 16,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0036413150373846292,
      "compile_s": 0.5694087299925741,
      "inner": 2,
      "median_ms": 17.601917999854777,
      "q1_ms": 17.573646997334436,
      "q3_ms": 17.625168002268765
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 32,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0036413150373846292,
      "compile_s": 0.9175277010072023,
      "inner": 3,
      "median_ms": 15.696883002722945,
      "q1_ms": 15.69177033282661,
      "q3_ms": 15.743544669627834
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 32,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0036413150373846292,
      "compile_s": 0.6100839459977578,
      "inner": 5,
      "median_ms": 9.409010800300166,
      "q1_ms": 9.395949798636138,
      "q3_ms": 9.415701200487092
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 32,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0036413150373846292,
      "compile_s": 0.9359684259979986,
      "inner": 3,
      "median_ms": 16.410402332743008,
      "q1_ms": 16.404301330718834,
      "q3_ms": 16.466769001757104
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 32,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0036413150373846292,
      "compile_s": 0.6289027060120134,
      "inner": 4,
      "median_ms": 12.030149249767419,
      "q1_ms": 12.008528250589734,
      "q3_ms": 12.04688849975355
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 64,
      "num_stages": 1,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 114688, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 64,
      "num_stages": 1,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 114688, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 64,
      "num_stages": 2,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 115200, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 64,
      "num_stages": 2,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 115200, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 128,
      "num_stages": 1,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 163840, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 128,
      "num_stages": 1,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 163840, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 128,
      "num_stages": 2,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 164864, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 128,
      "num_stages": 2,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 164864, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "auto",
      "rel_err": 0.001886138692498207,
      "compile_s": 0.26254172599874437,
      "inner": 3,
      "median_ms": 16.687402996467426,
      "q1_ms": 16.661190000983577,
      "q3_ms": 16.75844400112207
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "auto",
      "rel_err": 0.0027109296061098576,
      "compile_s": 0.5623520240042126,
      "inner": 1,
      "median_ms": 79.57090499985497,
      "q1_ms": 79.44481799495406,
      "q3_ms": 79.6929869975429
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 32,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.001886138692498207,
      "compile_s": 0.17676847901020665,
      "inner": 2,
      "median_ms": 20.76219099399168,
      "q1_ms": 20.72259750275407,
      "q3_ms": 20.87174850021256
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 32,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.001886138692498207,
      "compile_s": 0.16280099500727374,
      "inner": 1,
      "median_ms": 32.03483100514859,
      "q1_ms": 32.02906300430186,
      "q3_ms": 32.210310993832536
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 32,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.001886138692498207,
      "compile_s": 0.18497545100399293,
      "inner": 2,
      "median_ms": 23.53678349754773,
      "q1_ms": 23.489749000873417,
      "q3_ms": 23.56280250387499
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 32,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.001886138692498207,
      "compile_s": 0.17291165501228534,
      "inner": 1,
      "median_ms": 34.33137899264693,
      "q1_ms": 34.25090700329747,
      "q3_ms": 34.34625499357935
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 32,
      "num_stages": 3,
      "num_warps": 4,
      "rel_err": 0.001886138692498207,
      "compile_s": 0.2045023699902231,
      "inner": 1,
      "median_ms": 29.249298007925972,
      "q1_ms": 29.244206991279498,
      "q3_ms": 29.258032998768613
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 32,
      "num_stages": 3,
      "num_warps": 8,
      "rel_err": 0.001886138692498207,
      "compile_s": 0.18527128000278026,
      "inner": 1,
      "median_ms": 41.273677998105995,
      "q1_ms": 41.27250600140542,
      "q3_ms": 41.39914599363692
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 64,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.001886138692498207,
      "compile_s": 0.20289749500807375,
      "inner": 2,
      "median_ms": 19.4472634975682,
      "q1_ms": 19.44672950048698,
      "q3_ms": 19.447990496701095
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 64,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.001886138692498207,
      "compile_s": 0.17288671700225677,
      "inner": 2,
      "median_ms": 23.041271495458204,
      "q1_ms": 22.985906500252895,
      "q3_ms": 23.054194993164856
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 64,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.001886138692498207,
      "compile_s": 0.22566730801190715,
      "inner": 1,
      "median_ms": 30.563923006411642,
      "q1_ms": 30.56323400232941,
      "q3_ms": 30.67060599278193
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 64,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.001886138692498207,
      "compile_s": 0.1976674130128231,
      "inner": 1,
      "median_ms": 34.3417060066713,
      "q1_ms": 34.17596701183356,
      "q3_ms": 34.559028994408436
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 64,
      "num_stages": 3,
      "num_warps": 4,
      "rel_err": 0.001886138692498207,
      "compile_s": 0.2259044459933648,
      "inner": 2,
      "median_ms": 24.58093850145815,
      "q1_ms": 24.538096506148577,
      "q3_ms": 24.58681899588555
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 64,
      "num_stages": 3,
      "num_warps": 8,
      "rel_err": 0.001886138692498207,
      "compile_s": 0.19213511999987531,
      "inner": 1,
      "median_ms": 29.630277000251226,
      "q1_ms": 29.63026899669785,
      "q3_ms": 29.631701996549964
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 128,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.001886138692498207,
      "compile_s": 0.27053404100297485,
      "inner": 2,
      "median_ms": 22.00733049539849,
      "q1_ms": 22.0065345056355,
      "q3_ms": 22.062960502807982
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 128,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.001886138692498207,
      "compile_s": 0.19812132199876942,
      "inner": 2,
      "median_ms": 23.544939002022147,
      "q1_ms": 23.522670999227557,
      "q3_ms": 23.632484502741136
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 128,
      "num_stages": 2,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 114688, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 128,
      "num_stages": 2,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 114688, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 128,
      "num_stages": 3,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 180224, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 128,
      "num_stages": 3,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 180224, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 32,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.001886138692498207,
      "compile_s": 0.25075059600931127,
      "inner": 2,
      "median_ms": 19.489594997139648,
      "q1_ms": 19.463099000859074,
      "q3_ms": 19.496121996780857
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 32,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.001886138692498207,
      "compile_s": 0.2213709820061922,
      "inner": 1,
      "median_ms": 31.305006006732583,
      "q1_ms": 31.261823009117506,
      "q3_ms": 31.323563001933508
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 32,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.001886138692498207,
      "compile_s": 0.26419849900412373,
      "inner": 2,
      "median_ms": 19.922111998312175,
      "q1_ms": 19.907399502699263,
      "q3_ms": 19.98504049697658
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 32,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.001886138692498207,
      "compile_s": 0.2197542249923572,
      "inner": 1,
      "median_ms": 30.310905000078492,
      "q1_ms": 30.301077000331134,
      "q3_ms": 30.41211598610971
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 32,
      "num_stages": 3,
      "num_warps": 4,
      "rel_err": 0.001886138692498207,
      "compile_s": 0.2708031590009341,
      "inner": 2,
      "median_ms": 23.934780503623188,
      "q1_ms": 23.933708500408102,
      "q3_ms": 23.936364501423668
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 32,
      "num_stages": 3,
      "num_warps": 8,
      "rel_err": 0.001886138692498207,
      "compile_s": 0.2385594490042422,
      "inner": 1,
      "median_ms": 35.804152008495294,
      "q1_ms": 35.70770099759102,
      "q3_ms": 35.83474300103262
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 64,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.001886138692498207,
      "compile_s": 0.30437629400694277,
      "inner": 2,
      "median_ms": 18.55040899681626,
      "q1_ms": 18.432475000736304,
      "q3_ms": 18.57980999920983
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 64,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.001886138692498207,
      "compile_s": 0.23813940500258468,
      "inner": 2,
      "median_ms": 21.296667502610944,
      "q1_ms": 21.131269000761677,
      "q3_ms": 21.29707299900474
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 64,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.001886138692498207,
      "compile_s": 0.3184343210014049,
      "inner": 2,
      "median_ms": 23.70279799652053,
      "q1_ms": 23.6748089955654,
      "q3_ms": 23.736851493595168
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 64,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.001886138692498207,
      "compile_s": 0.25323309999657795,
      "inner": 1,
      "median_ms": 28.541097999550402,
      "q1_ms": 28.539575010654517,
      "q3_ms": 28.55370099132415
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 64,
      "num_stages": 3,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 106496, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 64,
      "num_stages": 3,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 106496, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 128,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.001886138692498207,
      "compile_s": 0.4075556399911875,
      "inner": 2,
      "median_ms": 23.295216502447147,
      "q1_ms": 23.256773500179406,
      "q3_ms": 23.40125800401438
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 128,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.001886138692498207,
      "compile_s": 0.28478583399555646,
      "inner": 1,
      "median_ms": 25.0465190038085,
      "q1_ms": 25.031630997546017,
      "q3_ms": 25.152280999463983
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 128,
      "num_stages": 2,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 131072, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 128,
      "num_stages": 2,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 131072, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 128,
      "num_stages": 3,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 196608, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 128,
      "num_stages": 3,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 196608, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 32,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.001886138692498207,
      "compile_s": 0.3446114580001449,
      "inner": 3,
      "median_ms": 15.90853433299344,
      "q1_ms": 15.89554232972053,
      "q3_ms": 15.920815664382342
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 32,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.001886138692498207,
      "compile_s": 0.2028481820016168,
      "inner": 2,
      "median_ms": 19.356146498466842,
      "q1_ms": 19.33084450138267,
      "q3_ms": 19.4398954990902
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 32,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.001886138692498207,
      "compile_s": 0.32875431899446994,
      "inner": 2,
      "median_ms": 18.25264550279826,
      "q1_ms": 18.24924349784851,
      "q3_ms": 18.25636750436388
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 32,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.001886138692498207,
      "compile_s": 0.20994139699905645,
      "inner": 2,
      "median_ms": 17.57244349573739,
      "q1_ms": 17.496584994660225,
      "q3_ms": 17.585934001544956
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 32,
      "num_stages": 3,
      "num_warps": 4,
      "rel_err": 0.001886138692498207,
      "compile_s": 0.352099758994882,
      "inner": 3,
      "median_ms": 16.37366266610722,
      "q1_ms": 16.360821997902047,
      "q3_ms": 16.376060996359836
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 32,
      "num_stages": 3,
      "num_warps": 8,
      "rel_err": 0.001886138692498207,
      "compile_s": 0.2147395000065444,
      "inner": 2,
      "median_ms": 17.5331990030827,
      "q1_ms": 17.430344996682834,
      "q3_ms": 17.562900502525736
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 64,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.001886138692498207,
      "compile_s": 0.46085302700521424,
      "inner": 1,
      "median_ms": 37.037757007055916,
      "q1_ms": 36.96576500078663,
      "q3_ms": 37.322955002309754
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 64,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.001886138692498207,
      "compile_s": 0.2508766640094109,
      "inner": 2,
      "median_ms": 18.37660450109979,
      "q1_ms": 18.317207002837677,
      "q3_ms": 18.381168498308398
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 64,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.001886138692498207,
      "compile_s": 0.44206938199931756,
      "inner": 3,
      "median_ms": 16.84377066461214,
      "q1_ms": 16.83835967075235,
      "q3_ms": 16.845265330630355
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 64,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.001886138692498207,
      "compile_s": 0.25988053300534375,
      "inner": 3,
      "median_ms": 16.799177664021652,
      "q1_ms": 16.757759333510574,
      "q3_ms": 16.845084663752157
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 64,
      "num_stages": 3,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 114688, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 64,
      "num_stages": 3,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 114688, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 128,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.001886138692498207,
      "compile_s": 0.7679447459959192,
      "inner": 1,
      "median_ms": 62.42758300504647,
      "q1_ms": 62.341884011402726,
      "q3_ms": 62.611544999526814
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 128,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.001886138692498207,
      "compile_s": 0.3821200050006155,
      "inner": 2,
      "median_ms": 24.22176799882436,
      "q1_ms": 24.13687350053806,
      "q3_ms": 24.369484002818353
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 128,
      "num_stages": 2,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 131072, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 128,
      "num_stages": 2,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 131072, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 128,
      "num_stages": 3,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 196608, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 128,
      "num_stages": 3,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 196608, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 16,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0027109296061098576,
      "compile_s": 0.6309708269982366,
      "inner": 1,
      "median_ms": 131.5859040041687,
      "q1_ms": 131.47569399734493,
      "q3_ms": 131.97795998712536
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 16,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0027109296061098576,
      "compile_s": 0.4628785870008869,
      "inner": 1,
      "median_ms": 168.9716820110334,
      "q1_ms": 168.89811599685345,
      "q3_ms": 169.59591300110333
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 16,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0027109296061098576,
      "compile_s": 0.6112430869980017,
      "inner": 1,
      "median_ms": 104.19945399917196,
      "q1_ms": 104.16574998816941,
      "q3_ms": 104.73936800553929
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 16,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0027109296061098576,
      "compile_s": 0.46228166899527423,
      "inner": 1,
      "median_ms": 157.32954499253538,
      "q1_ms": 156.977052000002,
      "q3_ms": 157.39183299592696
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 32,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0027109296061098576,
      "compile_s": 0.616393642994808,
      "inner": 1,
      "median_ms": 116.97495600674301,
      "q1_ms": 116.79491499671713,
      "q3_ms": 116.97739099327009
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 32,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0027109296061098576,
      "compile_s": 0.42675712300115265,
      "inner": 1,
      "median_ms": 128.59929000842385,
      "q1_ms": 128.33890800538938,
      "q3_ms": 128.67564499902073
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 32,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0027109296061098576,
      "compile_s": 0.5977491079975152,
      "inner": 1,
      "median_ms": 97.77572299935855,
      "q1_ms": 97.73254601168446,
      "q3_ms": 97.82388601161074
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 32,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0027109296061098576,
      "compile_s": 0.42360286800249014,
      "inner": 1,
      "median_ms": 121.50764600664843,
      "q1_ms": 121.48564700328279,
      "q3_ms": 121.58557599468622
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 64,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0027109296061098576,
      "compile_s": 0.6254399269964779,
      "inner": 1,
      "median_ms": 125.30163300107233,
      "q1_ms": 125.30083300953265,
      "q3_ms": 125.96307399508078
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 64,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0027109296061098576,
      "compile_s": 0.450518170997384,
      "inner": 1,
      "median_ms": 115.75090099358931,
      "q1_ms": 115.55006199341733,
      "q3_ms": 115.76797399902716
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 64,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0027109296061098576,
      "compile_s": 0.6012033529987093,
      "inner": 1,
      "median_ms": 90.48230299958959,
      "q1_ms": 90.29257400834467,
      "q3_ms": 90.57698100514244
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 64,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0027109296061098576,
      "compile_s": 0.4436527480138466,
      "inner": 1,
      "median_ms": 111.20891000609845,
      "q1_ms": 111.12262798997108,
      "q3_ms": 111.47797899320722
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 128,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0027109296061098576,
      "compile_s": 0.7410042059927946,
      "inner": 1,
      "median_ms": 124.63023299642373,
      "q1_ms": 124.38879699038807,
      "q3_ms": 125.95610100834165
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 128,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0027109296061098576,
      "compile_s": 0.5764057719934499,
      "inner": 1,
      "median_ms": 121.57017599383835,
      "q1_ms": 120.59993098955601,
      "q3_ms": 123.31683099910151
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 128,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0027109296061098576,
      "compile_s": 0.7324775199958822,
      "inner": 1,
      "median_ms": 100.63214799447451,
      "q1_ms": 100.55394099617843,
      "q3_ms": 100.63958200043999
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 32,
      "bwd_inner": 128,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0027109296061098576,
      "compile_s": 0.5797046430088812,
      "inner": 1,
      "median_ms": 112.33841600187588,
      "q1_ms": 111.3567790016532,
      "q3_ms": 113.31956399953924
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 16,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0027109296061098576,
      "compile_s": 0.6161675220064353,
      "inner": 1,
      "median_ms": 117.34338199312333,
      "q1_ms": 117.26741799793672,
      "q3_ms": 117.4165010015713
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 16,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0027109296061098576,
      "compile_s": 0.46521504600241315,
      "inner": 1,
      "median_ms": 156.4945669961162,
      "q1_ms": 156.46816600929014,
      "q3_ms": 156.94767300738022
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 16,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0027109296061098576,
      "compile_s": 0.5986193639982957,
      "inner": 1,
      "median_ms": 93.25607599748764,
      "q1_ms": 92.54478600632865,
      "q3_ms": 93.27211799973156
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 16,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0027109296061098576,
      "compile_s": 0.45753881901327986,
      "inner": 1,
      "median_ms": 136.74632800393738,
      "q1_ms": 136.3236689940095,
      "q3_ms": 137.0278210088145
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 32,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0027109296061098576,
      "compile_s": 0.6185760369990021,
      "inner": 1,
      "median_ms": 110.54414400132373,
      "q1_ms": 110.42934699798934,
      "q3_ms": 110.55401200428605
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 32,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0027109296061098576,
      "compile_s": 0.4446214610070456,
      "inner": 1,
      "median_ms": 107.21891600405797,
      "q1_ms": 106.48908499570098,
      "q3_ms": 107.2418980038492
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 32,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0027109296061098576,
      "compile_s": 0.5848914810048882,
      "inner": 1,
      "median_ms": 80.25689399801195,
      "q1_ms": 80.1627839973662,
      "q3_ms": 80.3510709956754
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 32,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0027109296061098576,
      "compile_s": 0.4537040760042146,
      "inner": 1,
      "median_ms": 98.45810699334834,
      "q1_ms": 98.19226499530487,
      "q3_ms": 98.62034099933226
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 64,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0027109296061098576,
      "compile_s": 0.7121910129935713,
      "inner": 1,
      "median_ms": 108.02684299414977,
      "q1_ms": 107.95838299964089,
      "q3_ms": 108.20630099624395
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 64,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0027109296061098576,
      "compile_s": 0.4888314400013769,
      "inner": 1,
      "median_ms": 85.169334008242,
      "q1_ms": 84.96034900599625,
      "q3_ms": 85.19143098965287
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 64,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0027109296061098576,
      "compile_s": 0.7083726010023383,
      "inner": 1,
      "median_ms": 81.73932800127659,
      "q1_ms": 81.62331799394451,
      "q3_ms": 81.95243099180516
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 64,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0027109296061098576,
      "compile_s": 0.5026038279902423,
      "inner": 1,
      "median_ms": 82.42974300810602,
      "q1_ms": 82.30010700935964,
      "q3_ms": 82.49076899664942
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 128,
      "num_stages": 1,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 114688, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 128,
      "num_stages": 1,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 114688, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 128,
      "num_stages": 2,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 115712, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 64,
      "bwd_inner": 128,
      "num_stages": 2,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 115712, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 16,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0027109296061098576,
      "compile_s": 1.1588145889982115,
      "inner": 1,
      "median_ms": 299.0410329948645,
      "q1_ms": 298.9108430047054,
      "q3_ms": 299.32027200993616
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 16,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0027109296061098576,
      "compile_s": 0.8330111320101423,
      "inner": 1,
      "median_ms": 225.9216470120009,
      "q1_ms": 225.24287599662784,
      "q3_ms": 226.65432500070892
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 16,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0027109296061098576,
      "compile_s": 1.132073537010001,
      "inner": 1,
      "median_ms": 348.03432000626344,
      "q1_ms": 347.87213600066025,
      "q3_ms": 348.15303499635775
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 16,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0027109296061098576,
      "compile_s": 0.8116582380025648,
      "inner": 1,
      "median_ms": 261.47551999019925,
      "q1_ms": 260.662560001947,
      "q3_ms": 261.6805769939674
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 32,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0027109296061098576,
      "compile_s": 1.1469581659912365,
      "inner": 1,
      "median_ms": 237.85045598924626,
      "q1_ms": 237.84365599567536,
      "q3_ms": 237.9121660051169
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 32,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0027109296061098576,
      "compile_s": 0.7288878530089278,
      "inner": 1,
      "median_ms": 135.82303099974524,
      "q1_ms": 135.72721199307125,
      "q3_ms": 136.6307999996934
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 32,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0027109296061098576,
      "compile_s": 1.1673495299910428,
      "inner": 1,
      "median_ms": 248.64887099829502,
      "q1_ms": 248.64284000068437,
      "q3_ms": 249.0202340122778
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 32,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0027109296061098576,
      "compile_s": 0.7875903199892491,
      "inner": 1,
      "median_ms": 177.7137059980305,
      "q1_ms": 177.46492399601266,
      "q3_ms": 178.283493994968
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 64,
      "num_stages": 1,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 114688, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 64,
      "num_stages": 1,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 114688, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 64,
      "num_stages": 2,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 115200, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 64,
      "num_stages": 2,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 115200, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 128,
      "num_stages": 1,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 163840, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 128,
      "num_stages": 1,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 163840, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 128,
      "num_stages": 2,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 164864, available: 101376, context: []"
    },
    {
      "dtype": "bfloat16",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 128,
      "fwd_block_k": 64,
      "bwd_outer": 128,
      "bwd_inner": 128,
      "num_stages": 2,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 164864, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "auto",
      "rel_err": 0.0007632712367922068,
      "compile_s": 0.2133856100117555,
      "inner": 38,
      "median_ms": 1.1797730790261227,
      "q1_ms": 1.1797266580335992,
      "q3_ms": 1.179801263022359
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "auto",
      "rel_err": 0.0007173035410232842,
      "compile_s": 0.34024216400575824,
      "inner": 9,
      "median_ms": 5.863863221343814,
      "q1_ms": 5.848504111377729,
      "q3_ms": 5.893819889428818
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 32,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0007632712367922068,
      "compile_s": 0.15578549599740654,
      "inner": 36,
      "median_ms": 1.3945666109470443,
      "q1_ms": 1.3901318335330062,
      "q3_ms": 1.3946434167640593
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 32,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0007632712367922068,
      "compile_s": 0.13805955200223252,
      "inner": 23,
      "median_ms": 2.2058521742861634,
      "q1_ms": 2.2055856085068584,
      "q3_ms": 2.2283625648529064
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 32,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0007632712367922068,
      "compile_s": 0.15767702300217934,
      "inner": 26,
      "median_ms": 1.4614643458536682,
      "q1_ms": 1.4471283849651138,
      "q3_ms": 1.4614735768051246
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 32,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0007632712367922068,
      "compile_s": 0.14295100299932528,
      "inner": 24,
      "median_ms": 2.1798217082202123,
      "q1_ms": 2.1706748332993206,
      "q3_ms": 2.1862914163648384
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 32,
      "num_stages": 3,
      "num_warps": 4,
      "rel_err": 0.0007632712367922068,
      "compile_s": 0.1686852220009314,
      "inner": 30,
      "median_ms": 1.6260184997615095,
      "q1_ms": 1.625977766525466,
      "q3_ms": 1.6308195665866758
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 32,
      "num_stages": 3,
      "num_warps": 8,
      "rel_err": 0.0007632712367922068,
      "compile_s": 0.15183491400966886,
      "inner": 22,
      "median_ms": 2.291187681707511,
      "q1_ms": 2.289555499869907,
      "q3_ms": 2.2936409996790728
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 64,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0007622598786838353,
      "compile_s": 0.1862558720022207,
      "inner": 38,
      "median_ms": 1.310067105422556,
      "q1_ms": 1.306729736633746,
      "q3_ms": 1.3117233948402205
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 64,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0007623230922035873,
      "compile_s": 0.15126521600177512,
      "inner": 31,
      "median_ms": 1.620881903011562,
      "q1_ms": 1.6200351613318367,
      "q3_ms": 1.6272523226366649
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 64,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0007622598786838353,
      "compile_s": 0.20012868801131845,
      "inner": 20,
      "median_ms": 2.1080938997329213,
      "q1_ms": 2.1080081998661626,
      "q3_ms": 2.111122600035742
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 64,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0007623230922035873,
      "compile_s": 0.15588665699760895,
      "inner": 20,
      "median_ms": 2.3515267501352355,
      "q1_ms": 2.349624149792362,
      "q3_ms": 2.355893049389124
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 64,
      "num_stages": 3,
      "num_warps": 4,
      "rel_err": 0.0007622598786838353,
      "compile_s": 0.2095632709970232,
      "inner": 27,
      "median_ms": 1.7830113334908944,
      "q1_ms": 1.7823708891497787,
      "q3_ms": 1.7887952595224812
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 64,
      "num_stages": 3,
      "num_warps": 8,
      "rel_err": 0.0007623230922035873,
      "compile_s": 0.16757409999263473,
      "inner": 22,
      "median_ms": 2.013894636051687,
      "q1_ms": 2.0043606815637425,
      "q3_ms": 2.015300454249055
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 128,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.000755180255509913,
      "compile_s": 0.25881351399584673,
      "inner": 24,
      "median_ms": 1.5478042920828254,
      "q1_ms": 1.5377527918947937,
      "q3_ms": 1.5562607501730479
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 128,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.000755180255509913,
      "compile_s": 0.18684052699245512,
      "inner": 30,
      "median_ms": 1.630975233274512,
      "q1_ms": 1.6292409001228711,
      "q3_ms": 1.6313021333189681
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 128,
      "num_stages": 2,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 106496, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 128,
      "num_stages": 2,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 106496, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 128,
      "num_stages": 3,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 172032, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 128,
      "num_stages": 3,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 172032, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 32,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0007632712367922068,
      "compile_s": 0.2113250730035361,
      "inner": 40,
      "median_ms": 1.1955352500081062,
      "q1_ms": 1.191882874991279,
      "q3_ms": 1.196686199909891
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 32,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0007632712367922068,
      "compile_s": 0.20098802400752902,
      "inner": 21,
      "median_ms": 2.2903558572233167,
      "q1_ms": 2.286982809198976,
      "q3_ms": 2.2905982380810506
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 32,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0007632712367922068,
      "compile_s": 0.2216144819976762,
      "inner": 38,
      "median_ms": 1.2400409997146773,
      "q1_ms": 1.2393544208240008,
      "q3_ms": 1.2401891581248492
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 32,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0007632712367922068,
      "compile_s": 0.21358480599883478,
      "inner": 21,
      "median_ms": 2.3315165717143658,
      "q1_ms": 2.3211560002805314,
      "q3_ms": 2.3347369521313035
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 32,
      "num_stages": 3,
      "num_warps": 4,
      "rel_err": 0.0007632712367922068,
      "compile_s": 0.2317482979997294,
      "inner": 35,
      "median_ms": 1.3453114569918916,
      "q1_ms": 1.3450867143027219,
      "q3_ms": 1.3457683143704862
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 32,
      "num_stages": 3,
      "num_warps": 8,
      "rel_err": 0.0007632712367922068,
      "compile_s": 0.22380021699063946,
      "inner": 20,
      "median_ms": 2.4532385999918915,
      "q1_ms": 2.452952099702088,
      "q3_ms": 2.4618258998089004
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 64,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0007622598786838353,
      "compile_s": 0.30794632600736804,
      "inner": 36,
      "median_ms": 1.2853686942738325,
      "q1_ms": 1.2816591944202729,
      "q3_ms": 1.2927159445502587
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 64,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0007622598786838353,
      "compile_s": 0.29911251900193747,
      "inner": 19,
      "median_ms": 2.491525210452413,
      "q1_ms": 2.491475579230801,
      "q3_ms": 2.4916074739609795
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 64,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0007622598786838353,
      "compile_s": 0.3261041160003515,
      "inner": 32,
      "median_ms": 1.4755236875316768,
      "q1_ms": 1.4748934063391061,
      "q3_ms": 1.4755913125554798
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 64,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0007622598786838353,
      "compile_s": 0.3133800419891486,
      "inner": 18,
      "median_ms": 2.485618111677468,
      "q1_ms": 2.484340054959628,
      "q3_ms": 2.502532333083865
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 64,
      "num_stages": 3,
      "num_warps": 4,
      "rel_err": 0.0007622598786838353,
      "compile_s": 0.3523790879989974,
      "inner": 36,
      "median_ms": 1.3074835830290492,
      "q1_ms": 1.3074179167031412,
      "q3_ms": 1.3079811942589004
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 64,
      "num_stages": 3,
      "num_warps": 8,
      "rel_err": 0.0007622598786838353,
      "compile_s": 0.3272291140019661,
      "inner": 20,
      "median_ms": 2.3495631001424044,
      "q1_ms": 2.338339799462119,
      "q3_ms": 2.349940299609443
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 128,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.000755180255509913,
      "compile_s": 0.57250687899068,
      "inner": 32,
      "median_ms": 1.4924106249054603,
      "q1_ms": 1.489275906351395,
      "q3_ms": 1.4925095629223506
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 128,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.000755180255509913,
      "compile_s": 0.5371565039968118,
      "inner": 16,
      "median_ms": 2.863111062652024,
      "q1_ms": 2.8581810629475513,
      "q3_ms": 2.874841375160031
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 128,
      "num_stages": 2,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 114688, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 128,
      "num_stages": 2,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 114688, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 128,
      "num_stages": 3,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 180224, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 128,
      "num_stages": 3,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 180224, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 32,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0007632712367922068,
      "compile_s": 0.2899972190061817,
      "inner": 38,
      "median_ms": 1.1942208948312327,
      "q1_ms": 1.194043184161848,
      "q3_ms": 1.1948916580877267
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 32,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0007632712367922068,
      "compile_s": 0.20481370900233742,
      "inner": 29,
      "median_ms": 1.4397642757855997,
      "q1_ms": 1.437889724158971,
      "q3_ms": 1.4398637933231055
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 32,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0007632712367922068,
      "compile_s": 0.3314680600014981,
      "inner": 31,
      "median_ms": 1.4319752907237757,
      "q1_ms": 1.431616193581102,
      "q3_ms": 1.4426157417361656
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 32,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0007632712367922068,
      "compile_s": 0.2230386860028375,
      "inner": 32,
      "median_ms": 1.4345539998430468,
      "q1_ms": 1.4345276872518298,
      "q3_ms": 1.434908093415288
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 32,
      "num_stages": 3,
      "num_warps": 4,
      "rel_err": 0.0007632712367922068,
      "compile_s": 0.32325108999793883,
      "inner": 34,
      "median_ms": 1.3060096470700742,
      "q1_ms": 1.3059439413208405,
      "q3_ms": 1.3061352061306346
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 32,
      "num_stages": 3,
      "num_warps": 8,
      "rel_err": 0.0007632712367922068,
      "compile_s": 0.2283149339928059,
      "inner": 36,
      "median_ms": 1.2916132776202478,
      "q1_ms": 1.2912317219565415,
      "q3_ms": 1.291706527505691
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 64,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0007622598786838353,
      "compile_s": 0.46077044001140166,
      "inner": 39,
      "median_ms": 1.1780860002498883,
      "q1_ms": 1.1779508461656336,
      "q3_ms": 1.1780949743040718
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 64,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0007622598786838353,
      "compile_s": 0.3031935470062308,
      "inner": 35,
      "median_ms": 1.3506804285238363,
      "q1_ms": 1.3500649714842439,
      "q3_ms": 1.350877371649923
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 64,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0007622598786838353,
      "compile_s": 0.49225734898936935,
      "inner": 34,
      "median_ms": 1.301969941243228,
      "q1_ms": 1.3017978823935503,
      "q3_ms": 1.3022269413340837
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 64,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0007622598786838353,
      "compile_s": 0.32744291798735503,
      "inner": 36,
      "median_ms": 1.2800683887487845,
      "q1_ms": 1.2772040835342422,
      "q3_ms": 1.2861607497648544
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 64,
      "num_stages": 3,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 114688, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 64,
      "num_stages": 3,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 114688, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 128,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.000755180255509913,
      "compile_s": 0.943534935999196,
      "inner": 24,
      "median_ms": 1.943989000210422,
      "q1_ms": 1.943878416568623,
      "q3_ms": 1.9461434997841327
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 128,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.000755180255509913,
      "compile_s": 0.549669157990138,
      "inner": 32,
      "median_ms": 1.531672968667408,
      "q1_ms": 1.5305643123610935,
      "q3_ms": 1.5332591246988159
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 128,
      "num_stages": 2,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 131072, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 128,
      "num_stages": 2,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 131072, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 128,
      "num_stages": 3,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 196608, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 128,
      "num_stages": 3,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 196608, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 16,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0007173035410232842,
      "compile_s": 0.28903726499993354,
      "inner": 8,
      "median_ms": 6.763335999494302,
      "q1_ms": 6.752936249540653,
      "q3_ms": 6.781836000300245
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 16,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0007173035410232842,
      "compile_s": 0.2782876050041523,
      "inner": 4,
      "median_ms": 11.906615749467164,
      "q1_ms": 11.849632497614948,
      "q3_ms": 11.938295749132521
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 16,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0007173035410232842,
      "compile_s": 0.3020869239990134,
      "inner": 7,
      "median_ms": 7.064366000122391,
      "q1_ms": 7.034377856013764,
      "q3_ms": 7.124648286013065
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 16,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0007173035410232842,
      "compile_s": 0.28310452500591055,
      "inner": 4,
      "median_ms": 11.860576247272547,
      "q1_ms": 11.798809249739861,
      "q3_ms": 11.868966499605449
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 32,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0007173035410232842,
      "compile_s": 0.3457565529970452,
      "inner": 9,
      "median_ms": 5.85518566691058,
      "q1_ms": 5.845765888807364,
      "q3_ms": 5.86722400071772
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 32,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0007173035410232842,
      "compile_s": 0.3028660679992754,
      "inner": 5,
      "median_ms": 9.386243400513195,
      "q1_ms": 9.34242460061796,
      "q3_ms": 9.389665999333374
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 32,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0007173035410232842,
      "compile_s": 0.3582343620073516,
      "inner": 9,
      "median_ms": 5.745768888219673,
      "q1_ms": 5.720054667714673,
      "q3_ms": 5.758546221639133
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 32,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0007173035410232842,
      "compile_s": 0.31452249200083315,
      "inner": 5,
      "median_ms": 9.252616800949909,
      "q1_ms": 9.247823600890115,
      "q3_ms": 9.29796059790533
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 64,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0007173035410232842,
      "compile_s": 0.4892847779992735,
      "inner": 8,
      "median_ms": 6.076994375689537,
      "q1_ms": 6.043424375093309,
      "q3_ms": 6.096434750361368
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 64,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0007173035410232842,
      "compile_s": 0.39553619900834747,
      "inner": 5,
      "median_ms": 9.558076001121663,
      "q1_ms": 9.557541398680769,
      "q3_ms": 9.559908401570283
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 64,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0007173035410232842,
      "compile_s": 0.5112253420084016,
      "inner": 9,
      "median_ms": 5.8160315545844945,
      "q1_ms": 5.808377334485865,
      "q3_ms": 5.826934444485232
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 64,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0007173035410232842,
      "compile_s": 0.41295533000084106,
      "inner": 5,
      "median_ms": 9.37724240066018,
      "q1_ms": 9.371854399796575,
      "q3_ms": 9.394593400065787
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 128,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0007173035410232842,
      "compile_s": 0.896458415008965,
      "inner": 6,
      "median_ms": 7.74572066681382,
      "q1_ms": 7.718824832409155,
      "q3_ms": 7.77280983311357
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 128,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0007173035410232842,
      "compile_s": 0.6775390789989615,
      "inner": 5,
      "median_ms": 9.245086798910052,
      "q1_ms": 9.230731398565695,
      "q3_ms": 9.245458600344136
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 128,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0007173035410232842,
      "compile_s": 0.9655650599888759,
      "inner": 7,
      "median_ms": 7.548934143934665,
      "q1_ms": 7.506453571425352,
      "q3_ms": 7.54960928627822
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 128,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0007173035410232842,
      "compile_s": 0.6941698709997581,
      "inner": 5,
      "median_ms": 9.424293399206363,
      "q1_ms": 9.419905001414008,
      "q3_ms": 9.438702801708132
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 16,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0007173035410232842,
      "compile_s": 0.4515363469981821,
      "inner": 5,
      "median_ms": 8.703530399361625,
      "q1_ms": 8.673481398727745,
      "q3_ms": 8.705622199340723
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 16,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0007173035410232842,
      "compile_s": 0.4325378869980341,
      "inner": 2,
      "median_ms": 18.423522502416745,
      "q1_ms": 18.415799000649713,
      "q3_ms": 18.606209501740523
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 16,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0007173035410232842,
      "compile_s": 0.4754334919998655,
      "inner": 5,
      "median_ms": 8.575183601351455,
      "q1_ms": 8.51340240042191,
      "q3_ms": 8.578816399676725
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 16,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0007173035410232842,
      "compile_s": 0.46189349399355706,
      "inner": 2,
      "median_ms": 18.32012799422955,
      "q1_ms": 18.308099497517105,
      "q3_ms": 18.36517250194447
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 32,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0007173035410232842,
      "compile_s": 0.5917736679984955,
      "inner": 7,
      "median_ms": 7.042991142952815,
      "q1_ms": 7.031160714437386,
      "q3_ms": 7.049383142397606
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 32,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0007173035410232842,
      "compile_s": 0.5626519310026197,
      "inner": 3,
      "median_ms": 15.11878999978459,
      "q1_ms": 15.092525999837866,
      "q3_ms": 15.131093333669318
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 32,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0007173035410232842,
      "compile_s": 0.6056992230005562,
      "inner": 7,
      "median_ms": 7.140541571841043,
      "q1_ms": 7.129744142210776,
      "q3_ms": 7.1414188569178805
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 32,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0007173035410232842,
      "compile_s": 0.5815469759982079,
      "inner": 3,
      "median_ms": 15.235861331651298,
      "q1_ms": 15.231937000256343,
      "q3_ms": 15.241842333731862
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 64,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0007173035410232842,
      "compile_s": 1.0339229399978649,
      "inner": 2,
      "median_ms": 17.351885500829667,
      "q1_ms": 17.34100099565694,
      "q3_ms": 17.37773499917239
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 64,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0007173035410232842,
      "compile_s": 0.982684197006165,
      "inner": 1,
      "median_ms": 32.621995007502846,
      "q1_ms": 32.621905993437394,
      "q3_ms": 32.704951998312026
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 64,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0007173035410232842,
      "compile_s": 1.04733684900566,
      "inner": 3,
      "median_ms": 13.595628998397538,
      "q1_ms": 13.576682996548092,
      "q3_ms": 13.619954998527342
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 64,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0007173035410232842,
      "compile_s": 0.9859009200008586,
      "inner": 2,
      "median_ms": 23.05053400050383,
      "q1_ms": 23.037388498778455,
      "q3_ms": 23.119523997593205
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 128,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0007173035410232842,
      "compile_s": 2.749669997007004,
      "inner": 1,
      "median_ms": 109.47739600669593,
      "q1_ms": 109.47145499812905,
      "q3_ms": 109.4809019996319
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 128,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0007173035410232842,
      "compile_s": 2.2463022120064124,
      "inner": 1,
      "median_ms": 161.69117399840616,
      "q1_ms": 161.64236899930984,
      "q3_ms": 161.76885401364416
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 128,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0007173035410232842,
      "compile_s": 2.682569101001718,
      "inner": 1,
      "median_ms": 97.22702800354455,
      "q1_ms": 97.1626649989048,
      "q3_ms": 97.37920699990354
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 128,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0007173035410232842,
      "compile_s": 2.2105638529901626,
      "inner": 1,
      "median_ms": 155.223109992221,
      "q1_ms": 155.14644600625616,
      "q3_ms": 155.39093100233003
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 16,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0007173035410232842,
      "compile_s": 0.7700840720062843,
      "inner": 4,
      "median_ms": 12.613006500032498,
      "q1_ms": 12.601374499354279,
      "q3_ms": 12.617539749044226
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 16,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0007173035410232842,
      "compile_s": 0.6169586169999093,
      "inner": 3,
      "median_ms": 16.808701999252662,
      "q1_ms": 16.720816003119882,
      "q3_ms": 16.828379331855103
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 16,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0007173035410232842,
      "compile_s": 0.7873566089983797,
      "inner": 3,
      "median_ms": 13.222249666190086,
      "q1_ms": 13.201884663430974,
      "q3_ms": 13.254843996643709
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 16,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0007173035410232842,
      "compile_s": 0.6387737650074996,
      "inner": 3,
      "median_ms": 16.904897667700425,
      "q1_ms": 16.84799699675447,
      "q3_ms": 16.98421766438211
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 32,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0007173035410232842,
      "compile_s": 0.9975815899961162,
      "inner": 3,
      "median_ms": 13.497981330147013,
      "q1_ms": 13.455207333511984,
      "q3_ms": 13.517754996428266
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 32,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0007173035410232842,
      "compile_s": 0.7606636459968286,
      "inner": 3,
      "median_ms": 15.165317667803416,
      "q1_ms": 15.162215330443965,
      "q3_ms": 15.167976001976058
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 32,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0007173035410232842,
      "compile_s": 1.0166142560046865,
      "inner": 3,
      "median_ms": 13.548893999541178,
      "q1_ms": 13.542471666975567,
      "q3_ms": 13.571236331093436
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 32,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0007173035410232842,
      "compile_s": 0.7652329280099366,
      "inner": 3,
      "median_ms": 14.948838666896336,
      "q1_ms": 14.934266999868365,
      "q3_ms": 14.965594998405626
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 64,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0007173035410232842,
      "compile_s": 1.7065226280101342,
      "inner": 3,
      "median_ms": 14.581794666203981,
      "q1_ms": 14.576083335365789,
      "q3_ms": 14.583155997873595
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 64,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0007173035410232842,
      "compile_s": 1.2111063149932306,
      "inner": 2,
      "median_ms": 21.53827850270318,
      "q1_ms": 21.49223149899626,
      "q3_ms": 21.587413000816014
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 64,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0007173035410232842,
      "compile_s": 1.7101341980014695,
      "inner": 3,
      "median_ms": 13.338810332546322,
      "q1_ms": 13.304468331625685,
      "q3_ms": 13.457604664533088
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 64,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0007173035410232842,
      "compile_s": 1.2257805870031007,
      "inner": 2,
      "median_ms": 17.42971400381066,
      "q1_ms": 17.418657000234816,
      "q3_ms": 17.443990996980574
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 128,
      "num_stages": 1,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 131072, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 128,
      "num_stages": 1,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 131072, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 128,
      "num_stages": 2,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 132096, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 128,
      "num_stages": 2,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 132096, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "auto",
      "rel_err": 0.0005927980528213084,
      "compile_s": 0.22563368400733452,
      "inner": 2,
      "median_ms": 17.97805100068217,
      "q1_ms": 17.977875002543442,
      "q3_ms": 18.003362500166986
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "auto",
      "rel_err": 0.0010316796833649278,
      "compile_s": 0.4371598990110215,
      "inner": 1,
      "median_ms": 100.62008799286559,
      "q1_ms": 100.47931599547155,
      "q3_ms": 100.62053200090304
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 32,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0005927980528213084,
      "compile_s": 0.17870029200275894,
      "inner": 2,
      "median_ms": 21.977901000354905,
      "q1_ms": 21.86933949997183,
      "q3_ms": 22.129942000901792
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 32,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0005927980528213084,
      "compile_s": 0.16632648999802768,
      "inner": 1,
      "median_ms": 34.90151998994406,
      "q1_ms": 34.86275998875499,
      "q3_ms": 35.09474299789872
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 32,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0005927980528213084,
      "compile_s": 0.17840120699838735,
      "inner": 2,
      "median_ms": 22.714748498401605,
      "q1_ms": 22.696042993629817,
      "q3_ms": 22.723133501131088
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 32,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0005927980528213084,
      "compile_s": 0.17341578799823765,
      "inner": 1,
      "median_ms": 34.36758299358189,
      "q1_ms": 34.36345999944024,
      "q3_ms": 34.51585001312196
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 32,
      "num_stages": 3,
      "num_warps": 4,
      "rel_err": 0.0005927980528213084,
      "compile_s": 0.19206844399741385,
      "inner": 2,
      "median_ms": 24.888674495741725,
      "q1_ms": 24.874883994925767,
      "q3_ms": 24.954785498266574
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 32,
      "num_stages": 3,
      "num_warps": 8,
      "rel_err": 0.0005927980528213084,
      "compile_s": 0.18175923799572047,
      "inner": 1,
      "median_ms": 36.24138799204957,
      "q1_ms": 36.21673400630243,
      "q3_ms": 36.44175200315658
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 64,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0005908641614951193,
      "compile_s": 0.2045788649993483,
      "inner": 2,
      "median_ms": 20.138551502896007,
      "q1_ms": 20.133848003752064,
      "q3_ms": 20.14327049982967
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 64,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0005908641614951193,
      "compile_s": 0.1733153879904421,
      "inner": 2,
      "median_ms": 25.67508149513742,
      "q1_ms": 25.555801497830544,
      "q3_ms": 25.715398995089345
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 64,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0005908641614951193,
      "compile_s": 0.23888424399774522,
      "inner": 1,
      "median_ms": 33.126890994026326,
      "q1_ms": 33.03464599594008,
      "q3_ms": 33.22471500723623
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 64,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0005908641614951193,
      "compile_s": 0.19492981600342318,
      "inner": 1,
      "median_ms": 37.01094699499663,
      "q1_ms": 36.8817830021726,
      "q3_ms": 37.1385970065603
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 64,
      "num_stages": 3,
      "num_warps": 4,
      "rel_err": 0.0005908641614951193,
      "compile_s": 0.2358348669949919,
      "inner": 1,
      "median_ms": 27.64003400807269,
      "q1_ms": 27.63265901012346,
      "q3_ms": 27.647408001939766
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 64,
      "num_stages": 3,
      "num_warps": 8,
      "rel_err": 0.0005908641614951193,
      "compile_s": 0.19644869898911566,
      "inner": 1,
      "median_ms": 31.272313004592434,
      "q1_ms": 31.269807994249277,
      "q3_ms": 31.335723004303873
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 128,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0005867774016223848,
      "compile_s": 0.2770620869996492,
      "inner": 2,
      "median_ms": 24.344636003661435,
      "q1_ms": 24.340062496776227,
      "q3_ms": 24.34994649956934
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 128,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0005867774016223848,
      "compile_s": 0.21253212601004634,
      "inner": 2,
      "median_ms": 25.53189150057733,
      "q1_ms": 25.508718499622773,
      "q3_ms": 25.541339498886373
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 128,
      "num_stages": 2,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 106496, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 128,
      "num_stages": 2,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 106496, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 128,
      "num_stages": 3,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 172032, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 128,
      "num_stages": 3,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 172032, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 32,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0005927980528213084,
      "compile_s": 0.2261091729887994,
      "inner": 2,
      "median_ms": 18.057881003187504,
      "q1_ms": 18.03966199804563,
      "q3_ms": 18.116969004040584
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 32,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0005927980528213084,
      "compile_s": 0.2305287320050411,
      "inner": 1,
      "median_ms": 34.80987100920174,
      "q1_ms": 34.79601599974558,
      "q3_ms": 34.83278601197526
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 32,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0005927980528213084,
      "compile_s": 0.23846856800082605,
      "inner": 2,
      "median_ms": 18.722543500189204,
      "q1_ms": 18.64739599841414,
      "q3_ms": 18.737972495728172
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 32,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0005927980528213084,
      "compile_s": 0.24104313600400928,
      "inner": 1,
      "median_ms": 35.673124002642,
      "q1_ms": 35.66701500676572,
      "q3_ms": 35.679710999829695
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 32,
      "num_stages": 3,
      "num_warps": 4,
      "rel_err": 0.0005927980528213084,
      "compile_s": 0.25159745699784253,
      "inner": 2,
      "median_ms": 20.625412500521634,
      "q1_ms": 20.61038900137646,
      "q3_ms": 20.652748498832807
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 32,
      "num_stages": 3,
      "num_warps": 8,
      "rel_err": 0.0005927980528213084,
      "compile_s": 0.2518006219906965,
      "inner": 1,
      "median_ms": 37.578606003080495,
      "q1_ms": 37.57284799939953,
      "q3_ms": 37.58811199804768
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 64,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0005908641614951193,
      "compile_s": 0.32858325500274077,
      "inner": 2,
      "median_ms": 19.80617699882714,
      "q1_ms": 19.678045995533466,
      "q3_ms": 19.837332503811922
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 64,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0005908641614951193,
      "compile_s": 0.3281098790030228,
      "inner": 1,
      "median_ms": 38.421764998929575,
      "q1_ms": 38.2537709956523,
      "q3_ms": 38.703573009115644
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 64,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0005908641614951193,
      "compile_s": 0.3450320429983549,
      "inner": 2,
      "median_ms": 22.73398949910188,
      "q1_ms": 22.73348849848844,
      "q3_ms": 22.741544002201408
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 64,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0005908641614951193,
      "compile_s": 0.3412879620009335,
      "inner": 1,
      "median_ms": 37.72595799819101,
      "q1_ms": 37.709438998717815,
      "q3_ms": 37.73453600297216
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 64,
      "num_stages": 3,
      "num_warps": 4,
      "rel_err": 0.0005908641614951193,
      "compile_s": 0.3713930980011355,
      "inner": 2,
      "median_ms": 19.702822501130868,
      "q1_ms": 19.672255002660677,
      "q3_ms": 19.712681001692545
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 64,
      "num_stages": 3,
      "num_warps": 8,
      "rel_err": 0.0005908641614951193,
      "compile_s": 0.3643613099993672,
      "inner": 1,
      "median_ms": 35.741926010814495,
      "q1_ms": 35.54853600508068,
      "q3_ms": 35.77140199195128
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 128,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0005867044092155993,
      "compile_s": 0.5758071409945842,
      "inner": 2,
      "median_ms": 22.591351000301074,
      "q1_ms": 22.509054499096237,
      "q3_ms": 22.66543549922062
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 128,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0005867044092155993,
      "compile_s": 0.5693236890074331,
      "inner": 1,
      "median_ms": 44.77148299338296,
      "q1_ms": 44.771192988264374,
      "q3_ms": 44.786421000026166
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 128,
      "num_stages": 2,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 114688, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 128,
      "num_stages": 2,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 114688, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 128,
      "num_stages": 3,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 180224, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 128,
      "num_stages": 3,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 180224, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 32,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0005927980528213084,
      "compile_s": 0.31057386500469875,
      "inner": 2,
      "median_ms": 17.0638624986168,
      "q1_ms": 17.0506314971135,
      "q3_ms": 17.070514004444703
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 32,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0005927980528213084,
      "compile_s": 0.22955046099377796,
      "inner": 2,
      "median_ms": 20.54218950070208,
      "q1_ms": 20.52806899882853,
      "q3_ms": 20.615517496480606
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 32,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0005927980528213084,
      "compile_s": 0.3388470259960741,
      "inner": 2,
      "median_ms": 20.63077249476919,
      "q1_ms": 20.584749996487517,
      "q3_ms": 20.63482450466836
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 32,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0005927980528213084,
      "compile_s": 0.24016153000411578,
      "inner": 2,
      "median_ms": 20.6264389998978,
      "q1_ms": 20.624750999559183,
      "q3_ms": 20.677264998084866
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 32,
      "num_stages": 3,
      "num_warps": 4,
      "rel_err": 0.0005927980528213084,
      "compile_s": 0.3479042850085534,
      "inner": 2,
      "median_ms": 18.726305504969787,
      "q1_ms": 18.674392005777918,
      "q3_ms": 18.7566045060521
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 32,
      "num_stages": 3,
      "num_warps": 8,
      "rel_err": 0.0005927980528213084,
      "compile_s": 0.2468213740066858,
      "inner": 2,
      "median_ms": 18.5619445037446,
      "q1_ms": 18.537628500780556,
      "q3_ms": 18.563953999546356
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 64,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0005908641614951193,
      "compile_s": 0.48271413100883365,
      "inner": 3,
      "median_ms": 16.65630099887494,
      "q1_ms": 16.635267670305137,
      "q3_ms": 16.717439333054546
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 64,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0005908641614951193,
      "compile_s": 0.3271318000042811,
      "inner": 2,
      "median_ms": 19.48565200291341,
      "q1_ms": 19.426013503107242,
      "q3_ms": 19.52939000329934
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 64,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0005908641614951193,
      "compile_s": 0.5050011800049106,
      "inner": 2,
      "median_ms": 18.52207900083158,
      "q1_ms": 18.519318495236803,
      "q3_ms": 18.527715998061467
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 64,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0005908641614951193,
      "compile_s": 0.34059055300895125,
      "inner": 2,
      "median_ms": 18.36986150010489,
      "q1_ms": 18.36191649636021,
      "q3_ms": 18.409365497063845
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 64,
      "num_stages": 3,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 114688, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 64,
      "num_stages": 3,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 114688, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 128,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0005867044092155993,
      "compile_s": 0.974407264002366,
      "inner": 1,
      "median_ms": 27.52585700363852,
      "q1_ms": 27.298207001877017,
      "q3_ms": 27.548322002985515
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 128,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0005867044092155993,
      "compile_s": 0.5651236359990435,
      "inner": 2,
      "median_ms": 22.409466502722353,
      "q1_ms": 22.405087998777162,
      "q3_ms": 22.42861400009133
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 128,
      "num_stages": 2,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 131072, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 128,
      "num_stages": 2,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 131072, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 128,
      "num_stages": 3,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 196608, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 128,
      "num_stages": 3,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 196608, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 16,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0010316796833649278,
      "compile_s": 0.38435718599066604,
      "inner": 1,
      "median_ms": 113.0518059944734,
      "q1_ms": 112.96997099998407,
      "q3_ms": 113.21826200583018
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 16,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0010316796833649278,
      "compile_s": 0.45297173100698274,
      "inner": 1,
      "median_ms": 197.1025849925354,
      "q1_ms": 196.25661498866975,
      "q3_ms": 197.44079498923384
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 16,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0010316796833649278,
      "compile_s": 0.41096283300430514,
      "inner": 1,
      "median_ms": 118.433775001904,
      "q1_ms": 118.21677400439512,
      "q3_ms": 118.46055500791408
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 16,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0010316796833649278,
      "compile_s": 0.464432285996736,
      "inner": 1,
      "median_ms": 197.35811799182557,
      "q1_ms": 197.01972900656983,
      "q3_ms": 197.62975299090613
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 32,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0010316796833649278,
      "compile_s": 0.43255942899850197,
      "inner": 1,
      "median_ms": 100.6625659938436,
      "q1_ms": 100.59865500079468,
      "q3_ms": 100.6820729962783
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 32,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0010316796833649278,
      "compile_s": 0.444462637999095,
      "inner": 1,
      "median_ms": 155.92728200135753,
      "q1_ms": 155.81278799800202,
      "q3_ms": 155.95085801032837
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 32,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0010316796833649278,
      "compile_s": 0.449846069997875,
      "inner": 1,
      "median_ms": 99.92067200073507,
      "q1_ms": 99.58235200610943,
      "q3_ms": 99.9706970033003
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 32,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0010316796833649278,
      "compile_s": 0.45435817299585324,
      "inner": 1,
      "median_ms": 154.48728000046685,
      "q1_ms": 154.45822400215548,
      "q3_ms": 154.49059499951545
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 64,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0010316796833649278,
      "compile_s": 0.5789246869971976,
      "inner": 1,
      "median_ms": 103.86061400640756,
      "q1_ms": 103.78136399958748,
      "q3_ms": 104.18310399109032
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 64,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0010316796833649278,
      "compile_s": 0.5311849599966081,
      "inner": 1,
      "median_ms": 149.07973499794025,
      "q1_ms": 149.00663400476333,
      "q3_ms": 149.21247500751633
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 64,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0010316796833649278,
      "compile_s": 0.6007547350018285,
      "inner": 1,
      "median_ms": 101.60291500505991,
      "q1_ms": 101.50969799724407,
      "q3_ms": 101.6708630049834
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 64,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0010316796833649278,
      "compile_s": 0.548241128999507,
      "inner": 1,
      "median_ms": 148.1161710107699,
      "q1_ms": 148.019218002446,
      "q3_ms": 148.25106599892024
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 128,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0010316796833649278,
      "compile_s": 1.015755832006107,
      "inner": 1,
      "median_ms": 132.8470440057572,
      "q1_ms": 132.74277599703055,
      "q3_ms": 132.85806200292427
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 128,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0010316796833649278,
      "compile_s": 0.8059041259984951,
      "inner": 1,
      "median_ms": 148.97672799997963,
      "q1_ms": 148.96833400416654,
      "q3_ms": 149.80756999284495
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 128,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0010316796833649278,
      "compile_s": 1.085873329997412,
      "inner": 1,
      "median_ms": 128.95661599759478,
      "q1_ms": 128.6906830064254,
      "q3_ms": 129.80989000061527
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 128,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0010316796833649278,
      "compile_s": 0.8325999449880328,
      "inner": 1,
      "median_ms": 158.18956200382672,
      "q1_ms": 157.6464750105515,
      "q3_ms": 158.89141999650747
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 16,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0010316796833649278,
      "compile_s": 0.5810277150012553,
      "inner": 1,
      "median_ms": 136.56935399922077,
      "q1_ms": 136.51907900930382,
      "q3_ms": 136.89225299458485
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 16,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0010316796833649278,
      "compile_s": 0.707154798990814,
      "inner": 1,
      "median_ms": 289.6660039987182,
      "q1_ms": 289.45325200038496,
      "q3_ms": 289.9125299882144
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 16,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0010316796833649278,
      "compile_s": 0.5887936210056068,
      "inner": 1,
      "median_ms": 129.79403999634087,
      "q1_ms": 129.2128300119657,
      "q3_ms": 130.64238199149258
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 16,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0010316796833649278,
      "compile_s": 0.7172781439876417,
      "inner": 1,
      "median_ms": 284.23544399265666,
      "q1_ms": 283.567058009794,
      "q3_ms": 285.07346801052336
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 32,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0010316796833649278,
      "compile_s": 0.6916897909977706,
      "inner": 1,
      "median_ms": 111.18085699854419,
      "q1_ms": 110.95490099978633,
      "q3_ms": 111.35710099188145
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 32,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0010316796833649278,
      "compile_s": 0.7702927369973622,
      "inner": 1,
      "median_ms": 225.16264200385194,
      "q1_ms": 224.29207799723372,
      "q3_ms": 225.6852720020106
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 32,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0010316796833649278,
      "compile_s": 0.7046777739888057,
      "inner": 1,
      "median_ms": 108.93703199690208,
      "q1_ms": 108.88097000133712,
      "q3_ms": 108.98939400794916
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 32,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0010316796833649278,
      "compile_s": 0.8031281559960917,
      "inner": 1,
      "median_ms": 235.89074400661048,
      "q1_ms": 235.49062399251852,
      "q3_ms": 236.73711699666455
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 64,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0010316796833649278,
      "compile_s": 1.275155144991004,
      "inner": 1,
      "median_ms": 265.90742099506315,
      "q1_ms": 265.8496080111945,
      "q3_ms": 265.97418599703815
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 64,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0010316796833649278,
      "compile_s": 1.4984325429977616,
      "inner": 1,
      "median_ms": 543.851876995177,
      "q1_ms": 543.746816998464,
      "q3_ms": 544.1681039956165
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 64,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0010316796833649278,
      "compile_s": 1.2396200020011747,
      "inner": 1,
      "median_ms": 195.572971002548,
      "q1_ms": 195.47169300494716,
      "q3_ms": 195.76325299567543
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 64,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0010316796833649278,
      "compile_s": 1.3227032790018711,
      "inner": 1,
      "median_ms": 368.1687500065891,
      "q1_ms": 367.9648929974064,
      "q3_ms": 368.3225010026945
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 128,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0010316796833649278,
      "compile_s": 4.405805037997197,
      "inner": 1,
      "median_ms": 1750.7368059887085,
      "q1_ms": 1750.6574469880434,
      "q3_ms": 1751.3721089926548
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 128,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0010316796833649278,
      "compile_s": 4.65966295999533,
      "inner": 1,
      "median_ms": 2574.858606007183,
      "q1_ms": 2574.726032995386,
      "q3_ms": 2575.094061990967
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 128,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0010316796833649278,
      "compile_s": 4.167813120991923,
      "inner": 1,
      "median_ms": 1567.847124999389,
      "q1_ms": 1567.8427329985425,
      "q3_ms": 1568.0933170078788
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 128,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0010316796833649278,
      "compile_s": 4.5326352619886165,
      "inner": 1,
      "median_ms": 2454.482168002869,
      "q1_ms": 2454.046341998037,
      "q3_ms": 2454.5807739923475
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 16,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0010316796833649278,
      "compile_s": 0.9430413139925804,
      "inner": 1,
      "median_ms": 189.48817500495352,
      "q1_ms": 189.4739099952858,
      "q3_ms": 189.7206650028238
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 16,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0010316796833649278,
      "compile_s": 0.8593178730079671,
      "inner": 1,
      "median_ms": 254.30870499985758,
      "q1_ms": 254.2082760046469,
      "q3_ms": 254.34095499804243
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 16,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0010316796833649278,
      "compile_s": 0.9566973709879676,
      "inner": 1,
      "median_ms": 192.65273900236934,
      "q1_ms": 192.25027799257077,
      "q3_ms": 192.7937159925932
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 16,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0010316796833649278,
      "compile_s": 0.8589330570102902,
      "inner": 1,
      "median_ms": 249.24581000232138,
      "q1_ms": 249.06993699551094,
      "q3_ms": 249.32543899922166
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 32,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0010316796833649278,
      "compile_s": 1.1885101899970323,
      "inner": 1,
      "median_ms": 203.65156899788417,
      "q1_ms": 203.25241700629704,
      "q3_ms": 204.36224600416608
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 32,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0010316796833649278,
      "compile_s": 0.9638662670040503,
      "inner": 1,
      "median_ms": 220.50977998878807,
      "q1_ms": 220.31615699233953,
      "q3_ms": 221.28560100100003
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 32,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0010316796833649278,
      "compile_s": 1.1955470689863432,
      "inner": 1,
      "median_ms": 195.30090798798483,
      "q1_ms": 194.78773800074123,
      "q3_ms": 195.51293799304403
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 32,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0010316796833649278,
      "compile_s": 0.9720429719891399,
      "inner": 1,
      "median_ms": 216.5074939985061,
      "q1_ms": 216.24673200130928,
      "q3_ms": 216.5707329986617
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 64,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0010316796833649278,
      "compile_s": 1.9553276039951015,
      "inner": 1,
      "median_ms": 221.67285400792025,
      "q1_ms": 221.6080609941855,
      "q3_ms": 221.93399399111513
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 64,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0010316796833649278,
      "compile_s": 1.5320476170018082,
      "inner": 1,
      "median_ms": 333.7261220003711,
      "q1_ms": 333.2617819978623,
      "q3_ms": 334.2890360072488
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 64,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0010316796833649278,
      "compile_s": 1.877872388009564,
      "inner": 1,
      "median_ms": 198.5840580018703,
      "q1_ms": 198.5013310040813,
      "q3_ms": 199.2069849948166
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 64,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0010316796833649278,
      "compile_s": 1.4775680539896712,
      "inner": 1,
      "median_ms": 264.4184729870176,
      "q1_ms": 264.30114200047683,
      "q3_ms": 264.4427820050623
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 128,
      "num_stages": 1,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 131072, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 128,
      "num_stages": 1,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 131072, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 128,
      "num_stages": 2,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 132096, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 64,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 128,
      "num_stages": 2,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 132096, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "auto",
      "rel_err": 0.0006174648879095912,
      "compile_s": 0.3122858120041201,
      "inner": 17,
      "median_ms": 2.7114385293851444,
      "q1_ms": 2.710161412216942,
      "q3_ms": 2.7165493529002345
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "auto",
      "rel_err": 0.0008860013913363218,
      "compile_s": 0.7919850120088086,
      "inner": 4,
      "median_ms": 12.116514997615013,
      "q1_ms": 12.058850999892456,
      "q3_ms": 12.177097250969382
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 32,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0006174648879095912,
      "compile_s": 0.2057588159950683,
      "inner": 19,
      "median_ms": 2.702942894661407,
      "q1_ms": 2.6992659473571146,
      "q3_ms": 2.708336842356642
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 32,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0006174648879095912,
      "compile_s": 0.17034437900292687,
      "inner": 13,
      "median_ms": 4.0619906156699965,
      "q1_ms": 4.060342999694582,
      "q3_ms": 4.073846076677841
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 32,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0006174648879095912,
      "compile_s": 0.21779115201206878,
      "inner": 12,
      "median_ms": 4.078950832384483,
      "q1_ms": 4.076877834449988,
      "q3_ms": 4.081776250435117
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 32,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0006174648879095912,
      "compile_s": 0.17537040400202386,
      "inner": 9,
      "median_ms": 5.2426194451982155,
      "q1_ms": 5.240331888974954,
      "q3_ms": 5.247146888804209
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 32,
      "num_stages": 3,
      "num_warps": 4,
      "rel_err": 0.0006174648879095912,
      "compile_s": 0.22714751699822955,
      "inner": 14,
      "median_ms": 3.3521538571221754,
      "q1_ms": 3.3459185714101687,
      "q3_ms": 3.3542565704140412
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 32,
      "num_stages": 3,
      "num_warps": 8,
      "rel_err": 0.0006174648879095912,
      "compile_s": 0.18350004999956582,
      "inner": 11,
      "median_ms": 4.514033453729512,
      "q1_ms": 4.5135287271643225,
      "q3_ms": 4.521061182127927
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 64,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0006191389402374625,
      "compile_s": 0.2646754110028269,
      "inner": 16,
      "median_ms": 3.005222937645158,
      "q1_ms": 3.0045935627640574,
      "q3_ms": 3.0086424994806293
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 64,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0006191389402374625,
      "compile_s": 0.19187618099385872,
      "inner": 16,
      "median_ms": 3.184268374752719,
      "q1_ms": 3.1835946247156244,
      "q3_ms": 3.1870769380475394
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 64,
      "num_stages": 2,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 114688, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 64,
      "num_stages": 2,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 114688, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 64,
      "num_stages": 3,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 180224, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 64,
      "num_stages": 3,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 180224, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 128,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0006157264579087496,
      "compile_s": 0.40194182300183456,
      "inner": 12,
      "median_ms": 3.8123841671525347,
      "q1_ms": 3.789777250252276,
      "q3_ms": 3.8158540834653345
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 128,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0006157264579087496,
      "compile_s": 0.25611504100379534,
      "inner": 13,
      "median_ms": 3.799330770002248,
      "q1_ms": 3.7958834615822594,
      "q3_ms": 3.803519384326557
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 128,
      "num_stages": 2,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 212992, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 128,
      "num_stages": 2,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 212992, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 128,
      "num_stages": 3,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 344064, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 128,
      "num_stages": 3,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 344064, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 32,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0006174648879095912,
      "compile_s": 0.3127465639990987,
      "inner": 17,
      "median_ms": 2.5791360000285377,
      "q1_ms": 2.57669370587203,
      "q3_ms": 2.584091234466006
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 32,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0006174648879095912,
      "compile_s": 0.24671622599998955,
      "inner": 14,
      "median_ms": 3.595998642725005,
      "q1_ms": 3.5911840002102378,
      "q3_ms": 3.6043207863778144
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 32,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0006174648879095912,
      "compile_s": 0.31678456100053154,
      "inner": 13,
      "median_ms": 3.212055461167788,
      "q1_ms": 3.2088078464417213,
      "q3_ms": 3.220251691975416
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 32,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0006174648879095912,
      "compile_s": 0.2534726730082184,
      "inner": 10,
      "median_ms": 4.432501801056787,
      "q1_ms": 4.428783799812663,
      "q3_ms": 4.4478630006778985
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 32,
      "num_stages": 3,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 114688, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 32,
      "num_stages": 3,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 114688, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 64,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0006191389402374625,
      "compile_s": 0.4059983339975588,
      "inner": 15,
      "median_ms": 3.1045360665302724,
      "q1_ms": 3.103058466998239,
      "q3_ms": 3.1139611994149163
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 64,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0006191389402374625,
      "compile_s": 0.2917446840001503,
      "inner": 13,
      "median_ms": 3.2958701530543086,
      "q1_ms": 3.2923435389350812,
      "q3_ms": 3.297336691712889
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 64,
      "num_stages": 2,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 131072, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 64,
      "num_stages": 2,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 131072, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 64,
      "num_stages": 3,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 196608, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 64,
      "num_stages": 3,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 196608, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 128,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0006157264579087496,
      "compile_s": 0.6751253029942745,
      "inner": 13,
      "median_ms": 3.5803427700347337,
      "q1_ms": 3.579693077391801,
      "q3_ms": 3.580809077203202
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 128,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0006157264579087496,
      "compile_s": 0.38389322799048387,
      "inner": 15,
      "median_ms": 3.0233405326725915,
      "q1_ms": 3.022798799793236,
      "q3_ms": 3.0317957998098186
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 128,
      "num_stages": 2,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 229376, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 128,
      "num_stages": 2,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 229376, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 128,
      "num_stages": 3,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 360448, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 128,
      "num_stages": 3,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 360448, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 32,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0006174648879095912,
      "compile_s": 0.5304147340066265,
      "inner": 17,
      "median_ms": 2.811180059592623,
      "q1_ms": 2.8045517058618477,
      "q3_ms": 2.8187078826115743
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 32,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0006174648879095912,
      "compile_s": 0.32190395300858654,
      "inner": 18,
      "median_ms": 2.6620445561194277,
      "q1_ms": 2.65960600033092,
      "q3_ms": 2.6768419443720224
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 32,
      "num_stages": 2,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 114688, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 32,
      "num_stages": 2,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 114688, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 32,
      "num_stages": 3,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 147456, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 32,
      "num_stages": 3,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 147456, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 64,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0006191389402374625,
      "compile_s": 0.8943254299956607,
      "inner": 12,
      "median_ms": 3.993064167540676,
      "q1_ms": 3.988485666923225,
      "q3_ms": 4.005312500036477
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 64,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0006191389402374625,
      "compile_s": 0.5294432060036343,
      "inner": 15,
      "median_ms": 3.3780753331181286,
      "q1_ms": 3.3731126663042232,
      "q3_ms": 3.378106133701901
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 64,
      "num_stages": 2,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 163840, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 64,
      "num_stages": 2,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 163840, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 64,
      "num_stages": 3,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 229376, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 64,
      "num_stages": 3,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 229376, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 128,
      "num_stages": 1,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 131072, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 128,
      "num_stages": 1,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 131072, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 128,
      "num_stages": 2,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 262144, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 128,
      "num_stages": 2,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 262144, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 128,
      "num_stages": 3,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 393216, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 128,
      "num_stages": 3,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 393216, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 16,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0008860013913363218,
      "compile_s": 0.4210764199960977,
      "inner": 3,
      "median_ms": 14.714668999658898,
      "q1_ms": 14.670992333170338,
      "q3_ms": 14.734286334714852
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 16,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0008860013913363218,
      "compile_s": 0.3535266580001917,
      "inner": 2,
      "median_ms": 24.090375503874384,
      "q1_ms": 23.997924996365327,
      "q3_ms": 24.1279715046403
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 16,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0008860013913363218,
      "compile_s": 0.4311803489981685,
      "inner": 3,
      "median_ms": 14.460379999945872,
      "q1_ms": 14.427278671064414,
      "q3_ms": 14.46459666476585
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 16,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0008860013913363218,
      "compile_s": 0.37748356600059196,
      "inner": 2,
      "median_ms": 25.06460300355684,
      "q1_ms": 25.034965998202097,
      "q3_ms": 25.078395003220066
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 32,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0008860013913363218,
      "compile_s": 0.5100226680078777,
      "inner": 3,
      "median_ms": 14.303515000695674,
      "q1_ms": 14.293671668080302,
      "q3_ms": 14.304766669132126
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 32,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0008860013913363218,
      "compile_s": 0.41675589900114574,
      "inner": 2,
      "median_ms": 18.714122001256328,
      "q1_ms": 18.698787003813777,
      "q3_ms": 18.726410999079235
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 32,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0008860013913363218,
      "compile_s": 0.5636779210035456,
      "inner": 3,
      "median_ms": 14.65933733076478,
      "q1_ms": 14.610334997996688,
      "q3_ms": 14.675972333255535
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 32,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0008860013913363218,
      "compile_s": 0.4328883090056479,
      "inner": 2,
      "median_ms": 19.7820125031285,
      "q1_ms": 19.77199850080069,
      "q3_ms": 19.85137350129662
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 64,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0008860013913363218,
      "compile_s": 0.8370779030083213,
      "inner": 3,
      "median_ms": 14.675985662809884,
      "q1_ms": 14.648469999277344,
      "q3_ms": 14.720703329658136
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 64,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0008860013913363218,
      "compile_s": 0.5935297579999315,
      "inner": 3,
      "median_ms": 16.522196669635985,
      "q1_ms": 16.504219999963727,
      "q3_ms": 16.653813669108786
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 64,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0008860013913363218,
      "compile_s": 0.8934265110001434,
      "inner": 3,
      "median_ms": 14.948003663448617,
      "q1_ms": 14.916515002065958,
      "q3_ms": 14.948045665126605
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 64,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0008860013913363218,
      "compile_s": 0.6151522909931373,
      "inner": 2,
      "median_ms": 18.201598497398663,
      "q1_ms": 18.15265099867247,
      "q3_ms": 18.275864502356853
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 128,
      "num_stages": 1,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 163840, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 128,
      "num_stages": 1,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 163840, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 128,
      "num_stages": 2,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 164864, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 128,
      "num_stages": 2,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 164864, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 16,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0008860013913363218,
      "compile_s": 0.6146008989890106,
      "inner": 3,
      "median_ms": 15.629244667555515,
      "q1_ms": 15.623990000070384,
      "q3_ms": 15.66422566732702
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 16,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0008860013913363218,
      "compile_s": 0.4561358359933365,
      "inner": 2,
      "median_ms": 19.382626996957697,
      "q1_ms": 19.38031749887159,
      "q3_ms": 19.47084249695763
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 16,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0008860013913363218,
      "compile_s": 0.6405127850011922,
      "inner": 3,
      "median_ms": 15.05213399650529,
      "q1_ms": 15.041720670221062,
      "q3_ms": 15.052140331439054
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 16,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0008860013913363218,
      "compile_s": 0.4725760370056378,
      "inner": 2,
      "median_ms": 20.30445550190052,
      "q1_ms": 20.20605900179362,
      "q3_ms": 20.334181499492843
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 32,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0008860013913363218,
      "compile_s": 0.7953657460020622,
      "inner": 4,
      "median_ms": 12.054232000082266,
      "q1_ms": 12.040598998282803,
      "q3_ms": 12.067362749803578
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 32,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0008860013913363218,
      "compile_s": 0.5503189370065229,
      "inner": 3,
      "median_ms": 14.527457999065518,
      "q1_ms": 14.514824336705109,
      "q3_ms": 14.62538966734428
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 32,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0008860013913363218,
      "compile_s": 0.8147074750013417,
      "inner": 3,
      "median_ms": 12.833537999540567,
      "q1_ms": 12.812146999446364,
      "q3_ms": 12.846015000832267
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 32,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0008860013913363218,
      "compile_s": 0.566380552991177,
      "inner": 3,
      "median_ms": 15.149401001205357,
      "q1_ms": 15.050814331819614,
      "q3_ms": 15.170721667042622
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 64,
      "num_stages": 1,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 131072, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 64,
      "num_stages": 1,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 131072, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 64,
      "num_stages": 2,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 131584, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 64,
      "num_stages": 2,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 131584, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 128,
      "num_stages": 1,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 196608, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 128,
      "num_stages": 1,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 196608, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 128,
      "num_stages": 2,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 197632, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 128,
      "num_stages": 2,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 197632, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 16,
      "num_stages": 1,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 147456, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 16,
      "num_stages": 1,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 147456, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 16,
      "num_stages": 2,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 147584, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 16,
      "num_stages": 2,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 147584, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 32,
      "num_stages": 1,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 163840, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 32,
      "num_stages": 1,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 163840, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 32,
      "num_stages": 2,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 164096, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 32,
      "num_stages": 2,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 164096, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 64,
      "num_stages": 1,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 196608, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 64,
      "num_stages": 1,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 196608, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 64,
      "num_stages": 2,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 197120, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 64,
      "num_stages": 2,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 197120, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 128,
      "num_stages": 1,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 262144, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 128,
      "num_stages": 1,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 262144, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 128,
      "num_stages": 2,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 263168, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 2048,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 128,
      "num_stages": 2,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 263168, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "auto",
      "rel_err": 0.0005074176006019115,
      "compile_s": 0.3562594779941719,
      "inner": 1,
      "median_ms": 39.68391800299287,
      "q1_ms": 39.62576999037992,
      "q3_ms": 39.844177998020314
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "auto",
      "rel_err": 0.0007364146877080202,
      "compile_s": 0.9593953879957553,
      "inner": 1,
      "median_ms": 185.04835999920033,
      "q1_ms": 184.91046701092273,
      "q3_ms": 185.22209599905182
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 32,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0005074176006019115,
      "compile_s": 0.2401683329953812,
      "inner": 1,
      "median_ms": 42.0845110056689,
      "q1_ms": 42.07918100291863,
      "q3_ms": 42.37425100291148
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 32,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0005074176006019115,
      "compile_s": 0.22505711400299333,
      "inner": 1,
      "median_ms": 64.22745800227858,
      "q1_ms": 64.18367799778935,
      "q3_ms": 64.27513998642098
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 32,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0005074176006019115,
      "compile_s": 0.27802927599987015,
      "inner": 1,
      "median_ms": 65.10540000454057,
      "q1_ms": 64.29347400262486,
      "q3_ms": 65.10979900485836
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 32,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0005074176006019115,
      "compile_s": 0.25185021600918844,
      "inner": 1,
      "median_ms": 83.45056198595557,
      "q1_ms": 83.13362399348989,
      "q3_ms": 83.64837999397423
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 32,
      "num_stages": 3,
      "num_warps": 4,
      "rel_err": 0.0005074176006019115,
      "compile_s": 0.27890432099229656,
      "inner": 1,
      "median_ms": 53.38816699804738,
      "q1_ms": 53.33232400880661,
      "q3_ms": 53.646989006665535
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 32,
      "num_stages": 3,
      "num_warps": 8,
      "rel_err": 0.0005074176006019115,
      "compile_s": 0.24801212699094322,
      "inner": 1,
      "median_ms": 71.22701799380593,
      "q1_ms": 71.18474399612751,
      "q3_ms": 71.57495300634764
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 64,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0005091050988994539,
      "compile_s": 0.3060245719971135,
      "inner": 1,
      "median_ms": 47.17015000642277,
      "q1_ms": 47.0612240023911,
      "q3_ms": 47.553583994158544
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 64,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0005091050988994539,
      "compile_s": 0.24108439500560053,
      "inner": 1,
      "median_ms": 50.30708100821357,
      "q1_ms": 50.29330600518733,
      "q3_ms": 50.45105300087016
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 64,
      "num_stages": 2,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 114688, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 64,
      "num_stages": 2,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 114688, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 64,
      "num_stages": 3,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 180224, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 64,
      "num_stages": 3,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 180224, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 128,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0005113048246130347,
      "compile_s": 0.4492369280051207,
      "inner": 1,
      "median_ms": 59.69477799953893,
      "q1_ms": 59.26480199559592,
      "q3_ms": 59.706939005991444
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 128,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0005113048246130347,
      "compile_s": 0.3079695500055095,
      "inner": 1,
      "median_ms": 59.874238999327645,
      "q1_ms": 59.8365250043571,
      "q3_ms": 59.96203799440991
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 128,
      "num_stages": 2,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 212992, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 128,
      "num_stages": 2,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 212992, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 128,
      "num_stages": 3,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 344064, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 32,
      "block_k": 128,
      "num_stages": 3,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 344064, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 32,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0005074176006019115,
      "compile_s": 0.34759439900517464,
      "inner": 1,
      "median_ms": 39.562731006299146,
      "q1_ms": 39.54039799282327,
      "q3_ms": 39.86330899351742
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 32,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0005074176006019115,
      "compile_s": 0.2943108790059341,
      "inner": 1,
      "median_ms": 54.61455800104886,
      "q1_ms": 54.43063999700826,
      "q3_ms": 54.66440499003511
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 32,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0005074176006019115,
      "compile_s": 0.36402861200622283,
      "inner": 1,
      "median_ms": 48.968530987622216,
      "q1_ms": 48.538848001044244,
      "q3_ms": 49.01889999746345
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 32,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0005074176006019115,
      "compile_s": 0.3171002979943296,
      "inner": 1,
      "median_ms": 67.59700100519694,
      "q1_ms": 67.5328100041952,
      "q3_ms": 67.61634699068964
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 32,
      "num_stages": 3,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 114688, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 32,
      "num_stages": 3,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 114688, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 64,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0005091050988994539,
      "compile_s": 0.4472592099918984,
      "inner": 1,
      "median_ms": 47.02235900913365,
      "q1_ms": 46.930644995882176,
      "q3_ms": 47.505927010206506
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 64,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0005091050988994539,
      "compile_s": 0.3315377129911212,
      "inner": 1,
      "median_ms": 50.17243599286303,
      "q1_ms": 49.83569399337284,
      "q3_ms": 50.228280990268104
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 64,
      "num_stages": 2,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 131072, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 64,
      "num_stages": 2,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 131072, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 64,
      "num_stages": 3,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 196608, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 64,
      "num_stages": 3,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 196608, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 128,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0005113048246130347,
      "compile_s": 0.7241652700031409,
      "inner": 1,
      "median_ms": 50.24657599278726,
      "q1_ms": 50.13551599404309,
      "q3_ms": 50.25267900782637
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 128,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0005113048246130347,
      "compile_s": 0.42242165700008627,
      "inner": 1,
      "median_ms": 45.94360900227912,
      "q1_ms": 45.93267900054343,
      "q3_ms": 46.09831201378256
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 128,
      "num_stages": 2,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 229376, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 128,
      "num_stages": 2,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 229376, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 128,
      "num_stages": 3,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 360448, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 64,
      "block_k": 128,
      "num_stages": 3,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 360448, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 32,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0005074176006019115,
      "compile_s": 0.5647120970097603,
      "inner": 1,
      "median_ms": 39.67299099895172,
      "q1_ms": 39.65516699827276,
      "q3_ms": 39.69751599652227
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 32,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0005074176006019115,
      "compile_s": 0.36536709099891596,
      "inner": 1,
      "median_ms": 38.10537498793565,
      "q1_ms": 38.10075799992774,
      "q3_ms": 38.15549999126233
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 32,
      "num_stages": 2,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 114688, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 32,
      "num_stages": 2,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 114688, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 32,
      "num_stages": 3,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 147456, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 32,
      "num_stages": 3,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 147456, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 64,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0005091050988994539,
      "compile_s": 0.9470435430121142,
      "inner": 1,
      "median_ms": 61.197036993689835,
      "q1_ms": 61.17603598977439,
      "q3_ms": 61.340619999100454
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 64,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0005091050988994539,
      "compile_s": 0.5803071410045959,
      "inner": 1,
      "median_ms": 51.17547400004696,
      "q1_ms": 50.89192799641751,
      "q3_ms": 51.2619579967577
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 64,
      "num_stages": 2,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 163840, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 64,
      "num_stages": 2,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 163840, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 64,
      "num_stages": 3,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 229376, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 64,
      "num_stages": 3,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 229376, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 128,
      "num_stages": 1,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 131072, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 128,
      "num_stages": 1,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 131072, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 128,
      "num_stages": 2,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 262144, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 128,
      "num_stages": 2,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 262144, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 128,
      "num_stages": 3,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 393216, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd",
      "config": "forced",
      "block_q": 128,
      "block_k": 128,
      "num_stages": 3,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 393216, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 16,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0007364146877080202,
      "compile_s": 0.6310242289910093,
      "inner": 1,
      "median_ms": 234.25336500804406,
      "q1_ms": 234.13505298958626,
      "q3_ms": 234.3594359990675
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 16,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0007364146877080202,
      "compile_s": 0.71185100999719,
      "inner": 1,
      "median_ms": 393.7595479947049,
      "q1_ms": 393.11111099959817,
      "q3_ms": 393.98567599710077
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 16,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0007364146877080202,
      "compile_s": 0.6591639779944671,
      "inner": 1,
      "median_ms": 245.03638000169303,
      "q1_ms": 244.5703770063119,
      "q3_ms": 245.5737849959405
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 16,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0007364146877080202,
      "compile_s": 0.7499861230025999,
      "inner": 1,
      "median_ms": 413.1664119922789,
      "q1_ms": 411.20295799919404,
      "q3_ms": 413.57629500271287
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 32,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0007364146877080202,
      "compile_s": 0.7231751570070628,
      "inner": 1,
      "median_ms": 238.15141699742526,
      "q1_ms": 237.52219699963462,
      "q3_ms": 239.994884992484
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 32,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0007364146877080202,
      "compile_s": 0.6983248890028335,
      "inner": 1,
      "median_ms": 297.47754998970777,
      "q1_ms": 296.7068200086942,
      "q3_ms": 297.669877007138
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 32,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0007364146877080202,
      "compile_s": 0.7725751240068348,
      "inner": 1,
      "median_ms": 235.4785209900001,
      "q1_ms": 234.55142600869294,
      "q3_ms": 237.36421299690846
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 32,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0007364146877080202,
      "compile_s": 0.7304381940048188,
      "inner": 1,
      "median_ms": 313.88675000926014,
      "q1_ms": 313.4790499898372,
      "q3_ms": 314.90306499472354
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 64,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0007364146877080202,
      "compile_s": 1.0638456619926728,
      "inner": 1,
      "median_ms": 255.36308900336735,
      "q1_ms": 255.20667400269303,
      "q3_ms": 255.9141210076632
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 64,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0007364146877080202,
      "compile_s": 0.8324270370067097,
      "inner": 1,
      "median_ms": 259.8825860040961,
      "q1_ms": 259.7757740004454,
      "q3_ms": 261.29084698914085
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 64,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0007364146877080202,
      "compile_s": 1.1523771139909513,
      "inner": 1,
      "median_ms": 273.1330459937453,
      "q1_ms": 272.72030401218217,
      "q3_ms": 273.1941820093198
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 64,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0007364146877080202,
      "compile_s": 0.8960475479980232,
      "inner": 1,
      "median_ms": 297.70045600889716,
      "q1_ms": 297.64980399340857,
      "q3_ms": 297.98243098775856
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 128,
      "num_stages": 1,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 163840, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 128,
      "num_stages": 1,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 163840, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 128,
      "num_stages": 2,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 164864, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 32,
      "bwd_inner": 128,
      "num_stages": 2,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 164864, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 16,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0007364146877080202,
      "compile_s": 0.8359611899941228,
      "inner": 1,
      "median_ms": 227.77029500866774,
      "q1_ms": 227.46329499932472,
      "q3_ms": 229.20914700080175
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 16,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0007364146877080202,
      "compile_s": 0.730474851996405,
      "inner": 1,
      "median_ms": 301.1244549998082,
      "q1_ms": 301.09953800274525,
      "q3_ms": 301.22176599979866
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 16,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0007364146877080202,
      "compile_s": 0.8538609129900578,
      "inner": 1,
      "median_ms": 237.98870999598876,
      "q1_ms": 236.72551100025885,
      "q3_ms": 238.2664259930607
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 16,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0007364146877080202,
      "compile_s": 0.7719154350052122,
      "inner": 1,
      "median_ms": 322.5443660048768,
      "q1_ms": 322.3852309893118,
      "q3_ms": 322.77765599428676
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 32,
      "num_stages": 1,
      "num_warps": 4,
      "rel_err": 0.0007364146877080202,
      "compile_s": 0.9650682339997729,
      "inner": 1,
      "median_ms": 186.35185300081503,
      "q1_ms": 185.92845200328156,
      "q3_ms": 186.4539380039787
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 32,
      "num_stages": 1,
      "num_warps": 8,
      "rel_err": 0.0007364146877080202,
      "compile_s": 0.7662315860070521,
      "inner": 1,
      "median_ms": 229.13415099901613,
      "q1_ms": 227.8249500086531,
      "q3_ms": 229.46519899414852
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 32,
      "num_stages": 2,
      "num_warps": 4,
      "rel_err": 0.0007364146877080202,
      "compile_s": 1.0076374390046112,
      "inner": 1,
      "median_ms": 202.98538100905716,
      "q1_ms": 201.5100499993423,
      "q3_ms": 203.192371991463
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 32,
      "num_stages": 2,
      "num_warps": 8,
      "rel_err": 0.0007364146877080202,
      "compile_s": 0.7869348210078897,
      "inner": 1,
      "median_ms": 234.21736501040868,
      "q1_ms": 234.10618699563202,
      "q3_ms": 234.37159199966118
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 64,
      "num_stages": 1,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 131072, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 64,
      "num_stages": 1,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 131072, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 64,
      "num_stages": 2,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 131584, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 64,
      "num_stages": 2,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 131584, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 128,
      "num_stages": 1,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 196608, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 128,
      "num_stages": 1,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 196608, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 128,
      "num_stages": 2,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 197632, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 64,
      "bwd_inner": 128,
      "num_stages": 2,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 197632, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 16,
      "num_stages": 1,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 147456, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 16,
      "num_stages": 1,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 147456, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 16,
      "num_stages": 2,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 147584, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 16,
      "num_stages": 2,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 147584, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 32,
      "num_stages": 1,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 163840, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 32,
      "num_stages": 1,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 163840, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 32,
      "num_stages": 2,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 164096, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 32,
      "num_stages": 2,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 164096, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 64,
      "num_stages": 1,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 196608, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 64,
      "num_stages": 1,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 196608, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 64,
      "num_stages": 2,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 197120, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 64,
      "num_stages": 2,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 197120, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 128,
      "num_stages": 1,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 262144, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 128,
      "num_stages": 1,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 262144, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 128,
      "num_stages": 2,
      "num_warps": 4,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 263168, available: 101376, context: []"
    },
    {
      "dtype": "float32",
      "head_dim": 128,
      "seq": 8192,
      "batch": 4,
      "heads": 8,
      "tag": null,
      "mode": "fwd_bwd",
      "config": "forced",
      "fwd_block_q": 64,
      "fwd_block_k": 32,
      "bwd_outer": 128,
      "bwd_inner": 128,
      "num_stages": 2,
      "num_warps": 8,
      "error": "JaxRuntimeError: RESOURCE_EXHAUSTED: Shared memory size limit exceeded: requested 263168, available: 101376, context: []"
    }
  ]
}