{
 "measuredOn": "2026-10-05",
 "model": "deepseek-flash (DeepSeek-V4.1-Flash), temperature 0, 120-turn cap",
 "setup": "Synthetic data server (10,000 orders, 500 files, 40 documents, deterministic) connected to a Gatana organization. Direct mode: the four data tools exposed to the model as functions. Code mode: Gatana's two code mode tools (search_tools, execute_code); the model's code runs in the gateway sandbox and calls the same four tools. Correctness checked against ground truth computed outside the model.",
 "pricesUsdPerMillion": {
  "inputCacheMiss": 0.3,
  "inputCacheHit": 0.006,
  "output": 1.2,
  "note": "DeepSeek list prices, standard hours"
 },
 "summaryMedians": {
  "T1_revenue_by_region|codemode": {
   "runs": 5,
   "correct": 5,
   "prompt_median": 30458,
   "completion_median": 1274,
   "tool_calls_median": 6,
   "seconds_median": 12.1,
   "usd_median": 0.0033,
   "prompt_max": 75081,
   "seconds_max": 15.8
  },
  "T1_revenue_by_region|direct": {
   "runs": 5,
   "correct": 2,
   "prompt_median": 1330460,
   "completion_median": 127900,
   "tool_calls_median": 20,
   "seconds_median": 617.2,
   "usd_median": 0.3888,
   "prompt_max": 28855139,
   "seconds_max": 861.1
  },
  "T2_largest_files|codemode": {
   "runs": 5,
   "correct": 5,
   "prompt_median": 17670,
   "completion_median": 1134,
   "tool_calls_median": 5,
   "seconds_median": 9.4,
   "usd_median": 0.0026,
   "prompt_max": 525075,
   "seconds_max": 20.6
  },
  "T2_largest_files|direct": {
   "runs": 5,
   "correct": 5,
   "prompt_median": 31110,
   "completion_median": 905,
   "tool_calls_median": 2,
   "seconds_median": 6.3,
   "usd_median": 0.0055,
   "prompt_max": 38284,
   "seconds_max": 10.3
  },
  "T3_documents|codemode": {
   "runs": 5,
   "correct": 5,
   "prompt_median": 59153,
   "completion_median": 1802,
   "tool_calls_median": 6,
   "seconds_median": 18.6,
   "usd_median": 0.0061,
   "prompt_max": 108637,
   "seconds_max": 24.4
  },
  "T3_documents|direct": {
   "runs": 5,
   "correct": 0,
   "prompt_median": 242630,
   "completion_median": 67535,
   "tool_calls_median": 41,
   "seconds_median": 317.7,
   "usd_median": 0.1032,
   "prompt_max": 393528,
   "seconds_max": 406.1
  }
 },
 "runs": [
  {
   "task": "T1_revenue_by_region",
   "mode": "direct",
   "rep": 0,
   "ok": false,
   "turns": 5,
   "tool_calls": 11,
   "tool_result_bytes": 357617,
   "prompt": 332264,
   "completion": 103730,
   "cache_hit": 211200,
   "cache_miss": 121064,
   "seconds": 283.3
  },
  {
   "task": "T1_revenue_by_region",
   "mode": "codemode",
   "rep": 0,
   "ok": true,
   "turns": 6,
   "tool_calls": 7,
   "tool_result_bytes": 32270,
   "prompt": 43523,
   "completion": 1183,
   "cache_hit": 34048,
   "cache_miss": 9475,
   "seconds": 13.1
  },
  {
   "task": "T2_largest_files",
   "mode": "direct",
   "rep": 0,
   "ok": true,
   "turns": 2,
   "tool_calls": 1,
   "tool_result_bytes": 42332,
   "prompt": 15831,
   "completion": 905,
   "cache_hit": 1280,
   "cache_miss": 14551,
   "seconds": 5.0
  },
  {
   "task": "T2_largest_files",
   "mode": "codemode",
   "rep": 0,
   "ok": true,
   "turns": 4,
   "tool_calls": 3,
   "tool_result_bytes": 8497,
   "prompt": 10992,
   "completion": 742,
   "cache_hit": 8192,
   "cache_miss": 2800,
   "seconds": 6.8
  },
  {
   "task": "T3_documents",
   "mode": "direct",
   "rep": 0,
   "ok": false,
   "turns": 6,
   "tool_calls": 42,
   "tool_result_bytes": 452633,
   "prompt": 254354,
   "completion": 82048,
   "cache_hit": 173184,
   "cache_miss": 81170,
   "seconds": 363.4
  },
  {
   "task": "T3_documents",
   "mode": "codemode",
   "rep": 0,
   "ok": true,
   "turns": 7,
   "tool_calls": 6,
   "tool_result_bytes": 36065,
   "prompt": 57849,
   "completion": 1782,
   "cache_hit": 47488,
   "cache_miss": 10361,
   "seconds": 18.6
  },
  {
   "task": "T1_revenue_by_region",
   "mode": "direct",
   "rep": 1,
   "ok": true,
   "turns": 21,
   "tool_calls": 20,
   "tool_result_bytes": 1060872,
   "prompt": 6470200,
   "completion": 283234,
   "cache_hit": 6093436,
   "cache_miss": 376764,
   "seconds": 861.1
  },
  {
   "task": "T1_revenue_by_region",
   "mode": "codemode",
   "rep": 1,
   "ok": true,
   "turns": 6,
   "tool_calls": 6,
   "tool_result_bytes": 11144,
   "prompt": 23559,
   "completion": 1552,
   "cache_hit": 19584,
   "cache_miss": 3975,
   "seconds": 12.1
  },
  {
   "task": "T2_largest_files",
   "mode": "direct",
   "rep": 1,
   "ok": true,
   "turns": 3,
   "tool_calls": 2,
   "tool_result_bytes": 42395,
   "prompt": 31110,
   "completion": 1997,
   "cache_hit": 16384,
   "cache_miss": 14726,
   "seconds": 10.3
  },
  {
   "task": "T2_largest_files",
   "mode": "codemode",
   "rep": 1,
   "ok": true,
   "turns": 5,
   "tool_calls": 5,
   "tool_result_bytes": 12363,
   "prompt": 20004,
   "completion": 1092,
   "cache_hit": 15744,
   "cache_miss": 4260,
   "seconds": 9.4
  },
  {
   "task": "T3_documents",
   "mode": "direct",
   "rep": 1,
   "ok": false,
   "turns": 6,
   "tool_calls": 41,
   "tool_result_bytes": 410301,
   "prompt": 174691,
   "completion": 66988,
   "cache_hit": 108032,
   "cache_miss": 66659,
   "seconds": 317.7
  },
  {
   "task": "T3_documents",
   "mode": "codemode",
   "rep": 1,
   "ok": true,
   "turns": 8,
   "tool_calls": 7,
   "tool_result_bytes": 44545,
   "prompt": 74178,
   "completion": 1802,
   "cache_hit": 62464,
   "cache_miss": 11714,
   "seconds": 17.7
  },
  {
   "task": "T1_revenue_by_region",
   "mode": "direct",
   "rep": 2,
   "ok": false,
   "turns": 4,
   "tool_calls": 21,
   "tool_result_bytes": 1069379,
   "prompt": 613960,
   "completion": 66877,
   "cache_hit": 235644,
   "cache_miss": 378316,
   "seconds": 209.5
  },
  {
   "task": "T1_revenue_by_region",
   "mode": "codemode",
   "rep": 2,
   "ok": true,
   "turns": 6,
   "tool_calls": 6,
   "tool_result_bytes": 16186,
   "prompt": 30458,
   "completion": 1274,
   "cache_hit": 24960,
   "cache_miss": 5498,
   "seconds": 15.8
  },
  {
   "task": "T2_largest_files",
   "mode": "direct",
   "rep": 2,
   "ok": true,
   "turns": 2,
   "tool_calls": 1,
   "tool_result_bytes": 42332,
   "prompt": 15827,
   "completion": 490,
   "cache_hit": 1280,
   "cache_miss": 14547,
   "seconds": 3.8
  },
  {
   "task": "T2_largest_files",
   "mode": "codemode",
   "rep": 2,
   "ok": true,
   "turns": 5,
   "tool_calls": 4,
   "tool_result_bytes": 10275,
   "prompt": 17670,
   "completion": 1216,
   "cache_hit": 14080,
   "cache_miss": 3590,
   "seconds": 9.7
  },
  {
   "task": "T3_documents",
   "mode": "direct",
   "rep": 2,
   "ok": false,
   "turns": 6,
   "tool_calls": 41,
   "tool_result_bytes": 410301,
   "prompt": 174870,
   "completion": 57920,
   "cache_hit": 107904,
   "cache_miss": 66966,
   "seconds": 232.4
  },
  {
   "task": "T3_documents",
   "mode": "codemode",
   "rep": 2,
   "ok": true,
   "turns": 7,
   "tool_calls": 6,
   "tool_result_bytes": 40878,
   "prompt": 57786,
   "completion": 1273,
   "cache_hit": 46848,
   "cache_miss": 10938,
   "seconds": 17.1
  },
  {
   "task": "T1_revenue_by_region",
   "mode": "direct",
   "rep": 3,
   "ok": true,
   "turns": 6,
   "tool_calls": 20,
   "tool_result_bytes": 1060872,
   "prompt": 1330460,
   "completion": 246579,
   "cache_hit": 954876,
   "cache_miss": 375584,
   "seconds": 730.6
  },
  {
   "task": "T1_revenue_by_region",
   "mode": "codemode",
   "rep": 3,
   "ok": true,
   "turns": 5,
   "tool_calls": 4,
   "tool_result_bytes": 64520,
   "prompt": 75081,
   "completion": 1034,
   "cache_hit": 52352,
   "cache_miss": 22729,
   "seconds": 11.6
  },
  {
   "task": "T2_largest_files",
   "mode": "direct",
   "rep": 3,
   "ok": true,
   "turns": 3,
   "tool_calls": 5,
   "tool_result_bytes": 60930,
   "prompt": 38284,
   "completion": 1156,
   "cache_hit": 17408,
   "cache_miss": 20876,
   "seconds": 7.1
  },
  {
   "task": "T2_largest_files",
   "mode": "codemode",
   "rep": 3,
   "ok": true,
   "turns": 5,
   "tool_calls": 5,
   "tool_result_bytes": 9642,
   "prompt": 16535,
   "completion": 1134,
   "cache_hit": 13312,
   "cache_miss": 3223,
   "seconds": 8.9
  },
  {
   "task": "T3_documents",
   "mode": "direct",
   "rep": 3,
   "ok": false,
   "turns": 6,
   "tool_calls": 41,
   "tool_result_bytes": 410301,
   "prompt": 242630,
   "completion": 89855,
   "cache_hit": 175744,
   "cache_miss": 66886,
   "seconds": 406.1
  },
  {
   "task": "T3_documents",
   "mode": "codemode",
   "rep": 3,
   "ok": true,
   "turns": 7,
   "tool_calls": 6,
   "tool_result_bytes": 95492,
   "prompt": 108637,
   "completion": 2697,
   "cache_hit": 76800,
   "cache_miss": 31837,
   "seconds": 24.4
  },
  {
   "task": "T1_revenue_by_region",
   "mode": "direct",
   "rep": 4,
   "ok": false,
   "turns": 120,
   "tool_calls": 124,
   "tool_result_bytes": 615341,
   "prompt": 28855139,
   "completion": 127900,
   "cache_hit": 28643452,
   "cache_miss": 211687,
   "seconds": 617.2
  },
  {
   "task": "T1_revenue_by_region",
   "mode": "codemode",
   "rep": 4,
   "ok": true,
   "turns": 5,
   "tool_calls": 4,
   "tool_result_bytes": 12306,
   "prompt": 20518,
   "completion": 1548,
   "cache_hit": 16384,
   "cache_miss": 4134,
   "seconds": 11.0
  },
  {
   "task": "T2_largest_files",
   "mode": "direct",
   "rep": 4,
   "ok": true,
   "turns": 3,
   "tool_calls": 2,
   "tool_result_bytes": 42395,
   "prompt": 31334,
   "completion": 789,
   "cache_hit": 16640,
   "cache_miss": 14694,
   "seconds": 6.3
  },
  {
   "task": "T2_largest_files",
   "mode": "codemode",
   "rep": 4,
   "ok": true,
   "turns": 7,
   "tool_calls": 7,
   "tool_result_bytes": 425374,
   "prompt": 525075,
   "completion": 2071,
   "cache_hit": 365696,
   "cache_miss": 159379,
   "seconds": 20.6
  },
  {
   "task": "T3_documents",
   "mode": "direct",
   "rep": 4,
   "ok": false,
   "turns": 11,
   "tool_calls": 56,
   "tool_result_bytes": 410811,
   "prompt": 393528,
   "completion": 67535,
   "cache_hit": 326272,
   "cache_miss": 67256,
   "seconds": 309.2
  },
  {
   "task": "T3_documents",
   "mode": "codemode",
   "rep": 4,
   "ok": true,
   "turns": 7,
   "tool_calls": 7,
   "tool_result_bytes": 34504,
   "prompt": 59153,
   "completion": 2811,
   "cache_hit": 49536,
   "cache_miss": 9617,
   "seconds": 23.6
  }
 ]
}