{
 "generated_at": "2026-09-10T16:52:38+0800",
 "run": "raw/ledger_capture_480x832_v1",
 "clock_offset_ns": 3014606511824156,
 "anchors": 4,
 "gpus": [
  0,
  1,
  2,
  3
 ],
 "dit_pids": [
  3168673,
  3168674,
  3168675,
  3168676
 ],
 "steady_median": {
  "dit_wall_ms": 677.5455489999999,
  "request_dispatch_ms": 26.12016,
  "prologue_to_first_kernel_ms": 0.050628000000000006,
  "host_enqueue_done_to_driver_ms": 104.578772,
  "gpu_tail_after_host_enqueue_ms": 98.5741935,
  "result_return_after_last_kernel_ms": 6.0045785,
  "gap_dit_to_submit_ms": 4.7399035000000005,
  "vae_wall_ms": 169.26465050000002,
  "ipc_submit_to_worker_ms": 6.6658645,
  "vae_h2d_ms": 0.525181,
  "vae_barrier_ms": 2.0583215,
  "vae_decode_ms": 141.5076885,
  "vae_barrier2_ms": 0.5620665,
  "vae_convert_ms": 0.1588615,
  "vae_d2h_ms": 1.185548,
  "ipc_result_latency_ms": 15.8165895,
  "dit_all_gpus_idle_ms": 58.2805695,
  "vae_d2h_memcpy_ms": 0.973491,
  "dit_gpu_busy_ms_median_per_gpu": {
   "0": 602.77121,
   "1": 602.3830995000001,
   "2": 601.7894845,
   "3": 602.8283905000001
  },
  "dit_exposed_nccl_ms_median_per_gpu": {
   "0": 52.0855175,
   "1": 50.465976999999995,
   "2": 50.240915,
   "3": 43.781545
  },
  "vae_gpu_busy_ms_median_per_gpu": {
   "0": 134.059338,
   "1": 133.1208575,
   "2": 134.6853945,
   "3": 127.2104975
  }
 },
 "phase_median": {
  "first_chunk": {
   "dit_wall_ms": 2377.05125,
   "vae_wall_ms": 260.724382,
   "vae_decode_ms": 231.700594,
   "request_dispatch_ms": 1888.932004,
   "prologue_to_first_kernel_ms": 0.016634,
   "gpu_tail_after_host_enqueue_ms": 59.271077,
   "result_return_after_last_kernel_ms": 4.993265,
   "ipc_result_latency_ms": 10.522705,
   "dit_all_gpus_idle_ms": 99.293669
  },
  "cache_growth": {
   "dit_wall_ms": 588.7590975,
   "vae_wall_ms": 159.0455725,
   "vae_decode_ms": 137.54188449999998,
   "request_dispatch_ms": 28.124941,
   "prologue_to_first_kernel_ms": 0.050048999999999996,
   "gpu_tail_after_host_enqueue_ms": 80.520996,
   "result_return_after_last_kernel_ms": 6.589866499999999,
   "ipc_result_latency_ms": 13.29583,
   "dit_all_gpus_idle_ms": 59.817358
  },
  "cache_full": {
   "dit_wall_ms": 678.152624,
   "vae_wall_ms": 168.576185,
   "vae_decode_ms": 139.535328,
   "request_dispatch_ms": 26.146106,
   "prologue_to_first_kernel_ms": 0.061143,
   "gpu_tail_after_host_enqueue_ms": 98.541957,
   "result_return_after_last_kernel_ms": 6.019221,
   "ipc_result_latency_ms": 15.747248,
   "dit_all_gpus_idle_ms": 58.70301
  }
 },
 "chunks": [
  {
   "chunk": 0,
   "host_ns": {
    "dit_request": 253744074262,
    "dit_done": 256121125512,
    "vae_submit": 256125810379,
    "vae_result": 256386534761,
    "ipc_send": 256376012056
   },
   "request_dispatch_ms": 1888.932004,
   "host_enqueue_done_to_driver_ms": 64.264342,
   "gpu_tail_after_host_enqueue_ms": 59.271077,
   "result_return_after_last_kernel_ms": 4.993265,
   "prologue_to_first_kernel_ms": 0.016634,
   "dit_wall_ms": 2377.05125,
   "gap_dit_to_submit_ms": 4.684867,
   "vae_wall_ms": 260.724382,
   "ipc_result_latency_ms": 10.522705,
   "vae_h2d_ms": 0.623951,
   "ipc_submit_to_worker_ms": 8.97206,
   "vae_barrier_ms": 6.536912,
   "vae_decode_ms": 231.700594,
   "vae_barrier2_ms": 1.171786,
   "vae_convert_ms": 0.222128,
   "vae_d2h_ms": 0.922734,
   "dit_gpu": {
    "0": {
     "busy_ms": 2195.652942,
     "compute_ms": 2129.464831,
     "nccl_ms": 81.830276,
     "exposed_nccl_ms": 66.188111,
     "idle_ms": 181.398308
    },
    "1": {
     "busy_ms": 2181.836378,
     "compute_ms": 2132.169035,
     "nccl_ms": 66.830901,
     "exposed_nccl_ms": 49.667343,
     "idle_ms": 195.214872
    },
    "2": {
     "busy_ms": 2194.633426,
     "compute_ms": 2134.50072,
     "nccl_ms": 76.334443,
     "exposed_nccl_ms": 60.132706,
     "idle_ms": 182.417824
    },
    "3": {
     "busy_ms": 2191.597123,
     "compute_ms": 2137.922136,
     "nccl_ms": 69.642634,
     "exposed_nccl_ms": 53.674987,
     "idle_ms": 185.454127
    }
   },
   "dit_all_gpus_idle_ms": 99.293669,
   "dit_idle_gaps": [
    {
     "start_ms": 0.0,
     "len_ms": 27.538156,
     "host_state": {
      "3168673": {
       "nvtx": [],
       "osrt": [
        "poll (27384us)"
       ]
      }
     }
    },
    {
     "start_ms": 73.736297,
     "len_ms": 6.703384,
     "host_state": {
      "3168673": {
       "nvtx": [],
       "osrt": [
        "ioctl (4157us)"
       ]
      }
     }
    },
    {
     "start_ms": 2372.057985,
     "len_ms": 4.993265,
     "host_state": {
      "3168673": {
       "nvtx": [],
       "osrt": [
        "poll (294872us)"
       ]
      }
     }
    },
    {
     "start_ms": 92.813029,
     "len_ms": 4.375554,
     "host_state": {
      "3168673": {
       "nvtx": [],
       "osrt": [
        "ioctl (4177us)"
       ]
      }
     }
    },
    {
     "start_ms": 97.487367,
     "len_ms": 3.475886,
     "host_state": {
      "3168673": {
       "nvtx": [],
       "osrt": [
        "ioctl (3321us)"
       ]
      }
     }
    },
    {
     "start_ms": 108.262428,
     "len_ms": 2.613828,
     "host_state": {
      "3168673": {
       "nvtx": [],
       "osrt": [
        "ioctl (3562us)"
       ]
      }
     }
    }
   ],
   "vae_gpu": {
    "0": {
     "busy_ms": 199.879487,
     "compute_ms": 93.735006,
     "nccl_ms": 106.144481,
     "exposed_nccl_ms": 106.144481,
     "idle_ms": 31.821107
    },
    "1": {
     "busy_ms": 97.006218,
     "compute_ms": 94.652996,
     "nccl_ms": 2.353222,
     "exposed_nccl_ms": 2.353222,
     "idle_ms": 134.694376
    },
    "2": {
     "busy_ms": 164.427983,
     "compute_ms": 94.320757,
     "nccl_ms": 70.107226,
     "exposed_nccl_ms": 70.107226,
     "idle_ms": 67.272611
    },
    "3": {
     "busy_ms": 182.503946,
     "compute_ms": 94.220417,
     "nccl_ms": 88.283529,
     "exposed_nccl_ms": 88.283529,
     "idle_ms": 49.196648
    }
   },
   "vae_d2h_memcpy_ms": 0.779906
  },
  {
   "chunk": 1,
   "host_ns": {
    "dit_request": 256388935959,
    "dit_done": 256932965531,
    "vae_submit": 256942170656,
    "vae_result": 257101061057,
    "ipc_send": 257088063454
   },
   "request_dispatch_ms": 29.078012,
   "host_enqueue_done_to_driver_ms": 77.681964,
   "gpu_tail_after_host_enqueue_ms": 70.415496,
   "result_return_after_last_kernel_ms": 7.266468,
   "prologue_to_first_kernel_ms": 0.017296,
   "dit_wall_ms": 544.029572,
   "gap_dit_to_submit_ms": 9.205125,
   "vae_wall_ms": 158.890401,
   "ipc_result_latency_ms": 12.997603,
   "vae_h2d_ms": 0.516028,
   "ipc_submit_to_worker_ms": 4.235502,
   "vae_barrier_ms": 2.269726,
   "vae_decode_ms": 136.920107,
   "vae_barrier2_ms": 0.491959,
   "vae_convert_ms": 0.148583,
   "vae_d2h_ms": 1.12866,
   "dit_gpu": {
    "0": {
     "busy_ms": 466.208922,
     "compute_ms": 415.960941,
     "nccl_ms": 65.794938,
     "exposed_nccl_ms": 50.247981,
     "idle_ms": 77.82065
    },
    "1": {
     "busy_ms": 467.077288,
     "compute_ms": 413.565281,
     "nccl_ms": 72.028871,
     "exposed_nccl_ms": 53.512007,
     "idle_ms": 76.952284
    },
    "2": {
     "busy_ms": 466.260733,
     "compute_ms": 418.055919,
     "nccl_ms": 64.914791,
     "exposed_nccl_ms": 48.204814,
     "idle_ms": 77.768839
    },
    "3": {
     "busy_ms": 466.158949,
     "compute_ms": 419.909993,
     "nccl_ms": 63.198984,
     "exposed_nccl_ms": 46.248956,
     "idle_ms": 77.870623
    }
   },
   "dit_all_gpus_idle_ms": 60.036676,
   "dit_idle_gaps": [
    {
     "start_ms": 0.0,
     "len_ms": 26.543663,
     "host_state": {
      "3168673": {
       "nvtx": [],
       "osrt": [
        "poll (294872us)"
       ]
      }
     }
    },
    {
     "start_ms": 536.763104,
     "len_ms": 7.266468,
     "host_state": {
      "3168673": {
       "nvtx": [],
       "osrt": []
      }
     }
    },
    {
     "start_ms": 29.879875,
     "len_ms": 1.612365,
     "host_state": {
      "3168673": {
       "nvtx": [
        "gen/f=3"
       ],
       "osrt": [
        "close (1648us)"
       ]
      }
     }
    },
    {
     "start_ms": 334.835028,
     "len_ms": 0.845782,
     "host_state": {
      "3168673": {
       "nvtx": [
        "gen/f=3",
        "tfm/f=3/c=0/s=3",
        "blk/0"
       ],
       "osrt": []
      }
     }
    },
    {
     "start_ms": 436.079914,
     "len_ms": 0.819536,
     "host_state": {
      "3168673": {
       "nvtx": [
        "gen/f=3",
        "tfm/f=3/c=1/s=None",
        "blk/0"
       ],
       "osrt": []
      }
     }
    },
    {
     "start_ms": 132.480968,
     "len_ms": 0.73504,
     "host_state": {
      "3168673": {
       "nvtx": [
        "gen/f=3",
        "tfm/f=3/c=0/s=1",
        "blk/0"
       ],
       "osrt": []
      }
     }
    }
   ],
   "vae_gpu": {
    "0": {
     "busy_ms": 129.787329,
     "compute_ms": 120.048804,
     "nccl_ms": 9.738525,
     "exposed_nccl_ms": 9.738525,
     "idle_ms": 7.132778
    },
    "1": {
     "busy_ms": 124.364975,
     "compute_ms": 120.610118,
     "nccl_ms": 3.754857,
     "exposed_nccl_ms": 3.754857,
     "idle_ms": 12.555132
    },
    "2": {
     "busy_ms": 128.759556,
     "compute_ms": 120.050718,
     "nccl_ms": 8.708838,
     "exposed_nccl_ms": 8.708838,
     "idle_ms": 8.160551
    },
    "3": {
     "busy_ms": 124.528385,
     "compute_ms": 120.892635,
     "nccl_ms": 3.63575,
     "exposed_nccl_ms": 3.63575,
     "idle_ms": 12.391722
    }
   },
   "vae_d2h_memcpy_ms": 0.984226
  },
  {
   "chunk": 2,
   "host_ns": {
    "dit_request": 257103671971,
    "dit_done": 257674812171,
    "vae_submit": 257680302733,
    "vae_result": 257838756603,
    "ipc_send": 257825538290
   },
   "request_dispatch_ms": 27.048696,
   "host_enqueue_done_to_driver_ms": 83.13764,
   "gpu_tail_after_host_enqueue_ms": 76.289079,
   "result_return_after_last_kernel_ms": 6.848561,
   "prologue_to_first_kernel_ms": 0.089911,
   "dit_wall_ms": 571.1402,
   "gap_dit_to_submit_ms": 5.490562,
   "vae_wall_ms": 158.45387,
   "ipc_result_latency_ms": 13.218313,
   "vae_h2d_ms": 0.503752,
   "ipc_submit_to_worker_ms": 4.364849,
   "vae_barrier_ms": 1.49056,
   "vae_decode_ms": 136.862648,
   "vae_barrier2_ms": 0.488198,
   "vae_convert_ms": 0.148028,
   "vae_d2h_ms": 1.138892,
   "dit_gpu": {
    "0": {
     "busy_ms": 496.772241,
     "compute_ms": 444.651041,
     "nccl_ms": 68.010978,
     "exposed_nccl_ms": 52.1212,
     "idle_ms": 74.367959
    },
    "1": {
     "busy_ms": 496.599652,
     "compute_ms": 445.68046,
     "nccl_ms": 69.028691,
     "exposed_nccl_ms": 50.919192,
     "idle_ms": 74.540548
    },
    "2": {
     "busy_ms": 496.403714,
     "compute_ms": 446.745584,
     "nccl_ms": 67.497374,
     "exposed_nccl_ms": 49.65813,
     "idle_ms": 74.736486
    },
    "3": {
     "busy_ms": 496.076543,
     "compute_ms": 453.882855,
     "nccl_ms": 57.649025,
     "exposed_nccl_ms": 42.193688,
     "idle_ms": 75.063657
    }
   },
   "dit_all_gpus_idle_ms": 57.907461,
   "dit_idle_gaps": [
    {
     "start_ms": 0.0,
     "len_ms": 24.42516,
     "host_state": {
      "3168673": {
       "nvtx": [],
       "osrt": []
      }
     }
    },
    {
     "start_ms": 564.291639,
     "len_ms": 6.848561,
     "host_state": {
      "3168673": {
       "nvtx": [],
       "osrt": []
      }
     }
    },
    {
     "start_ms": 27.371151,
     "len_ms": 1.939716,
     "host_state": {
      "3168673": {
       "nvtx": [
        "gen/f=6"
       ],
       "osrt": [
        "close (1243us)"
       ]
      }
     }
    },
    {
     "start_ms": 244.268765,
     "len_ms": 0.809241,
     "host_state": {
      "3168673": {
       "nvtx": [
        "gen/f=6",
        "tfm/f=6/c=0/s=2",
        "blk/0"
       ],
       "osrt": []
      }
     }
    },
    {
     "start_ms": 351.116623,
     "len_ms": 0.761467,
     "host_state": {
      "3168673": {
       "nvtx": [
        "gen/f=6",
        "tfm/f=6/c=0/s=3",
        "blk/0"
       ],
       "osrt": []
      }
     }
    },
    {
     "start_ms": 458.443003,
     "len_ms": 0.747438,
     "host_state": {
      "3168673": {
       "nvtx": [
        "gen/f=6",
        "tfm/f=6/c=1/s=None",
        "blk/0"
       ],
       "osrt": []
      }
     }
    }
   ],
   "vae_gpu": {
    "0": {
     "busy_ms": 128.385324,
     "compute_ms": 119.623665,
     "nccl_ms": 8.761659,
     "exposed_nccl_ms": 8.761659,
     "idle_ms": 8.477324
    },
    "1": {
     "busy_ms": 126.023564,
     "compute_ms": 121.263173,
     "nccl_ms": 4.760391,
     "exposed_nccl_ms": 4.760391,
     "idle_ms": 10.839084
    },
    "2": {
     "busy_ms": 126.086861,
     "compute_ms": 120.23764,
     "nccl_ms": 5.849221,
     "exposed_nccl_ms": 5.849221,
     "idle_ms": 10.775787
    },
    "3": {
     "busy_ms": 125.409968,
     "compute_ms": 121.22846,
     "nccl_ms": 4.181508,
     "exposed_nccl_ms": 4.181508,
     "idle_ms": 11.45268
    }
   },
   "vae_d2h_memcpy_ms": 0.987267
  },
  {
   "chunk": 3,
   "host_ns": {
    "dit_request": 257841361368,
    "dit_done": 258447739363,
    "vae_submit": 258452522315,
    "vae_result": 258614174166,
    "ipc_send": 258600800819
   },
   "request_dispatch_ms": 27.17187,
   "host_enqueue_done_to_driver_ms": 90.89923,
   "gpu_tail_after_host_enqueue_ms": 84.752913,
   "result_return_after_last_kernel_ms": 6.146317,
   "prologue_to_first_kernel_ms": 0.036811,
   "dit_wall_ms": 606.377995,
   "gap_dit_to_submit_ms": 4.782952,
   "vae_wall_ms": 161.651851,
   "ipc_result_latency_ms": 13.373347,
   "vae_h2d_ms": 0.559818,
   "ipc_submit_to_worker_ms": 3.984787,
   "vae_barrier_ms": 1.8888,
   "vae_decode_ms": 138.657832,
   "vae_barrier2_ms": 0.979322,
   "vae_convert_ms": 0.316612,
   "vae_d2h_ms": 1.156131,
   "dit_gpu": {
    "0": {
     "busy_ms": 531.60526,
     "compute_ms": 480.469611,
     "nccl_ms": 67.271554,
     "exposed_nccl_ms": 51.135649,
     "idle_ms": 74.772735
    },
    "1": {
     "busy_ms": 531.367414,
     "compute_ms": 479.225271,
     "nccl_ms": 72.429608,
     "exposed_nccl_ms": 52.142143,
     "idle_ms": 75.010581
    },
    "2": {
     "busy_ms": 530.771437,
     "compute_ms": 484.026494,
     "nccl_ms": 63.85649,
     "exposed_nccl_ms": 46.744943,
     "idle_ms": 75.606558
    },
    "3": {
     "busy_ms": 531.040062,
     "compute_ms": 487.090468,
     "nccl_ms": 60.530097,
     "exposed_nccl_ms": 43.949594,
     "idle_ms": 75.337933
    }
   },
   "dit_all_gpus_idle_ms": 59.59804,
   "dit_idle_gaps": [
    {
     "start_ms": 0.0,
     "len_ms": 24.77194,
     "host_state": {
      "3168673": {
       "nvtx": [],
       "osrt": []
      }
     }
    },
    {
     "start_ms": 600.231678,
     "len_ms": 6.146317,
     "host_state": {
      "3168673": {
       "nvtx": [],
       "osrt": []
      }
     }
    },
    {
     "start_ms": 27.43124,
     "len_ms": 2.445904,
     "host_state": {
      "3168673": {
       "nvtx": [
        "gen/f=9"
       ],
       "osrt": [
        "close (1267us)"
       ]
      }
     }
    },
    {
     "start_ms": 258.723962,
     "len_ms": 1.056783,
     "host_state": {
      "3168673": {
       "nvtx": [
        "gen/f=9",
        "tfm/f=9/c=0/s=2",
        "blk/0"
       ],
       "osrt": []
      }
     }
    },
    {
     "start_ms": 144.185506,
     "len_ms": 1.053022,
     "host_state": {
      "3168673": {
       "nvtx": [
        "gen/f=9",
        "tfm/f=9/c=0/s=1",
        "blk/0"
       ],
       "osrt": []
      }
     }
    },
    {
     "start_ms": 373.043071,
     "len_ms": 0.849089,
     "host_state": {
      "3168673": {
       "nvtx": [
        "gen/f=9",
        "tfm/f=9/c=0/s=3",
        "blk/0"
       ],
       "osrt": []
      }
     }
    }
   ],
   "vae_gpu": {
    "0": {
     "busy_ms": 131.29682,
     "compute_ms": 119.709781,
     "nccl_ms": 11.587039,
     "exposed_nccl_ms": 11.587039,
     "idle_ms": 7.361012
    },
    "1": {
     "busy_ms": 130.895807,
     "compute_ms": 120.837225,
     "nccl_ms": 10.058582,
     "exposed_nccl_ms": 10.058582,
     "idle_ms": 7.762025
    },
    "2": {
     "busy_ms": 124.852411,
     "compute_ms": 120.357401,
     "nccl_ms": 4.49501,
     "exposed_nccl_ms": 4.49501,
     "idle_ms": 13.805421
    },
    "3": {
     "busy_ms": 131.450291,
     "compute_ms": 121.728037,
     "nccl_ms": 9.722254,
     "exposed_nccl_ms": 9.722254,
     "idle_ms": 7.207541
    }
   },
   "vae_d2h_memcpy_ms": 0.973122
  },
  {
   "chunk": 4,
   "host_ns": {
    "dit_request": 258616953026,
    "dit_done": 259269737670,
    "vae_submit": 259274352851,
    "vae_result": 259433553595,
    "ipc_send": 259420110757
   },
   "request_dispatch_ms": 33.493242,
   "host_enqueue_done_to_driver_ms": 97.791563,
   "gpu_tail_after_host_enqueue_ms": 91.460391,
   "result_return_after_last_kernel_ms": 6.331172,
   "prologue_to_first_kernel_ms": 0.063287,
   "dit_wall_ms": 652.784644,
   "gap_dit_to_submit_ms": 4.615181,
   "vae_wall_ms": 159.200744,
   "ipc_result_latency_ms": 13.442838,
   "vae_h2d_ms": 0.628151,
   "ipc_submit_to_worker_ms": 3.767894,
   "vae_barrier_ms": 1.117918,
   "vae_decode_ms": 138.163662,
   "vae_barrier2_ms": 0.548441,
   "vae_convert_ms": 0.16431,
   "vae_d2h_ms": 1.146904,
   "dit_gpu": {
    "0": {
     "busy_ms": 569.049987,
     "compute_ms": 517.479629,
     "nccl_ms": 67.908447,
     "exposed_nccl_ms": 51.570358,
     "idle_ms": 83.734657
    },
    "1": {
     "busy_ms": 567.305621,
     "compute_ms": 516.200309,
     "nccl_ms": 72.300142,
     "exposed_nccl_ms": 51.105312,
     "idle_ms": 85.479023
    },
    "2": {
     "busy_ms": 569.509432,
     "compute_ms": 520.819318,
     "nccl_ms": 65.540999,
     "exposed_nccl_ms": 48.690114,
     "idle_ms": 83.275212
    },
    "3": {
     "busy_ms": 569.915553,
     "compute_ms": 522.472238,
     "nccl_ms": 65.061921,
     "exposed_nccl_ms": 47.443315,
     "idle_ms": 82.869091
    }
   },
   "dit_all_gpus_idle_ms": 65.961545,
   "dit_idle_gaps": [
    {
     "start_ms": 0.0,
     "len_ms": 30.64849,
     "host_state": {
      "3168673": {
       "nvtx": [],
       "osrt": []
      }
     }
    },
    {
     "start_ms": 646.453472,
     "len_ms": 6.331172,
     "host_state": {
      "3168673": {
       "nvtx": [],
       "osrt": []
      }
     }
    },
    {
     "start_ms": 33.695049,
     "len_ms": 2.574085,
     "host_state": {
      "3168673": {
       "nvtx": [
        "gen/f=12"
       ],
       "osrt": [
        "close (3467us)"
       ]
      }
     }
    },
    {
     "start_ms": 158.985285,
     "len_ms": 1.143109,
     "host_state": {
      "3168673": {
       "nvtx": [
        "gen/f=12",
        "tfm/f=12/c=0/s=1",
        "blk/0"
       ],
       "osrt": []
      }
     }
    },
    {
     "start_ms": 281.313617,
     "len_ms": 1.090967,
     "host_state": {
      "3168673": {
       "nvtx": [
        "gen/f=12",
        "tfm/f=12/c=0/s=2",
        "blk/0"
       ],
       "osrt": []
      }
     }
    },
    {
     "start_ms": 403.239927,
     "len_ms": 0.73926,
     "host_state": {
      "3168673": {
       "nvtx": [
        "gen/f=12",
        "tfm/f=12/c=0/s=3",
        "blk/0"
       ],
       "osrt": []
      }
     }
    }
   ],
   "vae_gpu": {
    "0": {
     "busy_ms": 130.168982,
     "compute_ms": 119.546707,
     "nccl_ms": 10.622275,
     "exposed_nccl_ms": 10.622275,
     "idle_ms": 7.99468
    },
    "1": {
     "busy_ms": 126.130708,
     "compute_ms": 121.86734,
     "nccl_ms": 4.263368,
     "exposed_nccl_ms": 4.263368,
     "idle_ms": 12.032954
    },
    "2": {
     "busy_ms": 130.682653,
     "compute_ms": 120.899188,
     "nccl_ms": 9.783465,
     "exposed_nccl_ms": 9.783465,
     "idle_ms": 7.481009
    },
    "3": {
     "busy_ms": 127.918151,
     "compute_ms": 122.040066,
     "nccl_ms": 5.878085,
     "exposed_nccl_ms": 5.878085,
     "idle_ms": 10.245511
    }
   },
   "vae_d2h_memcpy_ms": 0.989059
  },
  {
   "chunk": 5,
   "host_ns": {
    "dit_request": 259435870002,
    "dit_done": 260114022626,
    "vae_submit": 260118762030,
    "vae_result": 260382171225,
    "ipc_send": 260366887869
   },
   "request_dispatch_ms": 26.094214,
   "host_enqueue_done_to_driver_ms": 105.435249,
   "gpu_tail_after_host_enqueue_ms": 99.253774,
   "result_return_after_last_kernel_ms": 6.181475,
   "prologue_to_first_kernel_ms": 0.061143,
   "dit_wall_ms": 678.152624,
   "gap_dit_to_submit_ms": 4.739404,
   "vae_wall_ms": 263.409195,
   "ipc_result_latency_ms": 15.283356,
   "vae_h2d_ms": 0.503492,
   "ipc_submit_to_worker_ms": 7.174058,
   "vae_barrier_ms": 8.70664,
   "vae_decode_ms": 229.907031,
   "vae_barrier2_ms": 1.551762,
   "vae_convert_ms": 0.133222,
   "vae_d2h_ms": 1.228429,
   "dit_gpu": {
    "0": {
     "busy_ms": 604.743265,
     "compute_ms": 550.252689,
     "nccl_ms": 73.271507,
     "exposed_nccl_ms": 54.490576,
     "idle_ms": 73.409359
    },
    "1": {
     "busy_ms": 604.011793,
     "compute_ms": 550.76742,
     "nccl_ms": 73.486294,
     "exposed_nccl_ms": 53.244373,
     "idle_ms": 74.140831
    },
    "2": {
     "busy_ms": 603.161291,
     "compute_ms": 552.720568,
     "nccl_ms": 71.228217,
     "exposed_nccl_ms": 50.440723,
     "idle_ms": 74.991333
    },
    "3": {
     "busy_ms": 604.785466,
     "compute_ms": 560.664716,
     "nccl_ms": 59.451307,
     "exposed_nccl_ms": 44.12075,
     "idle_ms": 73.367158
    }
   },
   "dit_all_gpus_idle_ms": 57.358839,
   "dit_idle_gaps": [
    {
     "start_ms": 0.0,
     "len_ms": 23.704952,
     "host_state": {
      "3168673": {
       "nvtx": [],
       "osrt": []
      }
     }
    },
    {
     "start_ms": 671.971149,
     "len_ms": 6.181475,
     "host_state": {
      "3168673": {
       "nvtx": [],
       "osrt": []
      }
     }
    },
    {
     "start_ms": 26.730809,
     "len_ms": 2.316102,
     "host_state": {
      "3168673": {
       "nvtx": [
        "gen/f=15"
       ],
       "osrt": [
        "close (1655us)"
       ]
      }
     }
    },
    {
     "start_ms": 416.653936,
     "len_ms": 1.045825,
     "host_state": {
      "3168673": {
       "nvtx": [
        "gen/f=15",
        "tfm/f=15/c=0/s=3",
        "blk/0"
       ],
       "osrt": []
      }
     }
    },
    {
     "start_ms": 287.498981,
     "len_ms": 0.987059,
     "host_state": {
      "3168673": {
       "nvtx": [
        "gen/f=15",
        "tfm/f=15/c=0/s=2",
        "blk/0"
       ],
       "osrt": []
      }
     }
    },
    {
     "start_ms": 157.782651,
     "len_ms": 0.724571,
     "host_state": {
      "3168673": {
       "nvtx": [
        "gen/f=15",
        "tfm/f=15/c=0/s=1",
        "blk/0"
       ],
       "osrt": []
      }
     }
    }
   ],
   "vae_gpu": {
    "0": {
     "busy_ms": 177.968522,
     "compute_ms": 119.58881,
     "nccl_ms": 58.379712,
     "exposed_nccl_ms": 58.379712,
     "idle_ms": 51.938509
    },
    "1": {
     "busy_ms": 147.700601,
     "compute_ms": 120.800349,
     "nccl_ms": 26.900252,
     "exposed_nccl_ms": 26.900252,
     "idle_ms": 82.20643
    },
    "2": {
     "busy_ms": 157.024692,
     "compute_ms": 119.848326,
     "nccl_ms": 37.176366,
     "exposed_nccl_ms": 37.176366,
     "idle_ms": 72.882339
    },
    "3": {
     "busy_ms": 203.288855,
     "compute_ms": 119.701179,
     "nccl_ms": 83.587676,
     "exposed_nccl_ms": 83.587676,
     "idle_ms": 26.618176
    }
   },
   "vae_d2h_memcpy_ms": 0.971875
  },
  {
   "chunk": 6,
   "host_ns": {
    "dit_request": 260384564253,
    "dit_done": 261064173802,
    "vae_submit": 261068914205,
    "vae_result": 261234217996,
    "ipc_send": 261218470748
   },
   "request_dispatch_ms": 26.628679,
   "host_enqueue_done_to_driver_ms": 104.596366,
   "gpu_tail_after_host_enqueue_ms": 98.60643,
   "result_return_after_last_kernel_ms": 5.989936,
   "prologue_to_first_kernel_ms": 0.040113,
   "dit_wall_ms": 679.609549,
   "gap_dit_to_submit_ms": 4.740403,
   "vae_wall_ms": 165.303791,
   "ipc_result_latency_ms": 15.747248,
   "vae_h2d_ms": 0.523823,
   "ipc_submit_to_worker_ms": 6.157671,
   "vae_barrier_ms": 0.79669,
   "vae_decode_ms": 139.535328,
   "vae_barrier2_ms": 0.532145,
   "vae_convert_ms": 0.173409,
   "vae_d2h_ms": 1.580074,
   "dit_gpu": {
    "0": {
     "busy_ms": 602.022332,
     "compute_ms": 550.124514,
     "nccl_ms": 69.586754,
     "exposed_nccl_ms": 51.897818,
     "idle_ms": 77.587217
    },
    "1": {
     "busy_ms": 601.094864,
     "compute_ms": 550.615033,
     "nccl_ms": 70.046643,
     "exposed_nccl_ms": 50.479831,
     "idle_ms": 78.514685
    },
    "2": {
     "busy_ms": 600.992841,
     "compute_ms": 550.951734,
     "nccl_ms": 69.929797,
     "exposed_nccl_ms": 50.041107,
     "idle_ms": 78.616708
    },
    "3": {
     "busy_ms": 601.114162,
     "compute_ms": 557.817291,
     "nccl_ms": 59.269722,
     "exposed_nccl_ms": 43.296871,
     "idle_ms": 78.495387
    }
   },
   "dit_all_gpus_idle_ms": 62.475118,
   "dit_idle_gaps": [
    {
     "start_ms": 0.0,
     "len_ms": 24.284659,
     "host_state": {
      "3168673": {
       "nvtx": [],
       "osrt": []
      }
     }
    },
    {
     "start_ms": 673.619613,
     "len_ms": 5.989936,
     "host_state": {
      "3168673": {
       "nvtx": [],
       "osrt": []
      }
     }
    },
    {
     "start_ms": 286.696789,
     "len_ms": 3.825685,
     "host_state": {
      "3168673": {
       "nvtx": [
        "gen/f=18",
        "tfm/f=18/c=0/s=2",
        "blk/0"
       ],
       "osrt": []
      }
     }
    },
    {
     "start_ms": 26.8275,
     "len_ms": 2.380018,
     "host_state": {
      "3168673": {
       "nvtx": [
        "gen/f=18"
       ],
       "osrt": [
        "close (1355us)"
       ]
      }
     }
    },
    {
     "start_ms": 285.206767,
     "len_ms": 1.108673,
     "host_state": {
      "3168673": {
       "nvtx": [
        "gen/f=18",
        "tfm/f=18/c=0/s=2",
        "blk/0"
       ],
       "osrt": []
      }
     }
    },
    {
     "start_ms": 418.870524,
     "len_ms": 0.970006,
     "host_state": {
      "3168673": {
       "nvtx": [
        "gen/f=18",
        "tfm/f=18/c=0/s=3",
        "blk/0"
       ],
       "osrt": []
      }
     }
    }
   ],
   "vae_gpu": {
    "0": {
     "busy_ms": 131.69646,
     "compute_ms": 120.295111,
     "nccl_ms": 11.401349,
     "exposed_nccl_ms": 11.401349,
     "idle_ms": 7.838868
    },
    "1": {
     "busy_ms": 130.580438,
     "compute_ms": 121.609536,
     "nccl_ms": 8.970902,
     "exposed_nccl_ms": 8.970902,
     "idle_ms": 8.95489
    },
    "2": {
     "busy_ms": 132.177972,
     "compute_ms": 120.893963,
     "nccl_ms": 11.284009,
     "exposed_nccl_ms": 11.284009,
     "idle_ms": 7.357356
    },
    "3": {
     "busy_ms": 124.920443,
     "compute_ms": 121.328566,
     "nccl_ms": 3.591877,
     "exposed_nccl_ms": 3.591877,
     "idle_ms": 14.614885
    }
   },
   "vae_d2h_memcpy_ms": 0.971298
  },
  {
   "chunk": 7,
   "host_ns": {
    "dit_request": 261236463450,
    "dit_done": 261912255614,
    "vae_submit": 261916767651,
    "vae_result": 262086720767,
    "ipc_send": 262070834836
   },
   "request_dispatch_ms": 26.146106,
   "host_enqueue_done_to_driver_ms": 104.271602,
   "gpu_tail_after_host_enqueue_ms": 98.446543,
   "result_return_after_last_kernel_ms": 5.825059,
   "prologue_to_first_kernel_ms": 0.061399,
   "dit_wall_ms": 675.792164,
   "gap_dit_to_submit_ms": 4.512037,
   "vae_wall_ms": 169.953116,
   "ipc_result_latency_ms": 15.885931,
   "vae_h2d_ms": 0.531211,
   "ipc_submit_to_worker_ms": 11.909903,
   "vae_barrier_ms": 2.110272,
   "vae_decode_ms": 137.338403,
   "vae_barrier2_ms": 0.591988,
   "vae_convert_ms": 0.144314,
   "vae_d2h_ms": 1.137239,
   "dit_gpu": {
    "0": {
     "busy_ms": 600.838798,
     "compute_ms": 550.392473,
     "nccl_ms": 68.930418,
     "exposed_nccl_ms": 50.446325,
     "idle_ms": 74.953366
    },
    "1": {
     "busy_ms": 601.911677,
     "compute_ms": 551.459554,
     "nccl_ms": 69.196045,
     "exposed_nccl_ms": 50.452123,
     "idle_ms": 73.880487
    },
    "2": {
     "busy_ms": 601.518753,
     "compute_ms": 551.662855,
     "nccl_ms": 70.142982,
     "exposed_nccl_ms": 49.855898,
     "idle_ms": 74.273411
    },
    "3": {
     "busy_ms": 602.21337,
     "compute_ms": 557.431475,
     "nccl_ms": 60.769805,
     "exposed_nccl_ms": 44.781895,
     "idle_ms": 73.578794
    }
   },
   "dit_all_gpus_idle_ms": 58.70301,
   "dit_idle_gaps": [
    {
     "start_ms": 0.0,
     "len_ms": 23.748492,
     "host_state": {
      "3168673": {
       "nvtx": [],
       "osrt": []
      }
     }
    },
    {
     "start_ms": 669.967105,
     "len_ms": 5.825059,
     "host_state": {
      "3168673": {
       "nvtx": [],
       "osrt": []
      }
     }
    },
    {
     "start_ms": 26.349589,
     "len_ms": 2.414244,
     "host_state": {
      "3168673": {
       "nvtx": [
        "gen/f=21"
       ],
       "osrt": [
        "close (1399us)"
       ]
      }
     }
    },
    {
     "start_ms": 412.92802,
     "len_ms": 1.061264,
     "host_state": {
      "3168673": {
       "nvtx": [
        "gen/f=21",
        "tfm/f=21/c=0/s=3",
        "blk/0"
       ],
       "osrt": []
      }
     }
    },
    {
     "start_ms": 155.555832,
     "len_ms": 1.027059,
     "host_state": {
      "3168673": {
       "nvtx": [
        "gen/f=21",
        "tfm/f=21/c=0/s=1",
        "blk/0"
       ],
       "osrt": []
      }
     }
    },
    {
     "start_ms": 30.817775,
     "len_ms": 0.833854,
     "host_state": {
      "3168673": {
       "nvtx": [
        "gen/f=21",
        "tfm/f=21/c=0/s=0",
        "blk/0"
       ],
       "osrt": []
      }
     }
    }
   ],
   "vae_gpu": {
    "0": {
     "busy_ms": 129.624872,
     "compute_ms": 119.973514,
     "nccl_ms": 9.651358,
     "exposed_nccl_ms": 9.651358,
     "idle_ms": 7.713531
    },
    "1": {
     "busy_ms": 123.800155,
     "compute_ms": 121.341945,
     "nccl_ms": 2.45821,
     "exposed_nccl_ms": 2.45821,
     "idle_ms": 13.538248
    },
    "2": {
     "busy_ms": 129.892611,
     "compute_ms": 120.112924,
     "nccl_ms": 9.779687,
     "exposed_nccl_ms": 9.779687,
     "idle_ms": 7.445792
    },
    "3": {
     "busy_ms": 129.146287,
     "compute_ms": 119.916157,
     "nccl_ms": 9.23013,
     "exposed_nccl_ms": 9.23013,
     "idle_ms": 8.192116
    }
   },
   "vae_d2h_memcpy_ms": 0.975107
  },
  {
   "chunk": 8,
   "host_ns": {
    "dit_request": 262089524656,
    "dit_done": 262766463130,
    "vae_submit": 262772260720,
    "vae_result": 262940836905,
    "ipc_send": 262924939047
   },
   "request_dispatch_ms": 25.242447,
   "host_enqueue_done_to_driver_ms": 104.561178,
   "gpu_tail_after_host_enqueue_ms": 98.541957,
   "result_return_after_last_kernel_ms": 6.019221,
   "prologue_to_first_kernel_ms": 0.039337,
   "dit_wall_ms": 676.938474,
   "gap_dit_to_submit_ms": 5.79759,
   "vae_wall_ms": 168.576185,
   "ipc_result_latency_ms": 15.897858,
   "vae_h2d_ms": 0.526539,
   "ipc_submit_to_worker_ms": 4.697293,
   "vae_barrier_ms": 2.006371,
   "vae_decode_ms": 143.480049,
   "vae_barrier2_ms": 0.46058,
   "vae_convert_ms": 0.17487,
   "vae_d2h_ms": 1.142667,
   "dit_gpu": {
    "0": {
     "busy_ms": 603.520088,
     "compute_ms": 551.246871,
     "nccl_ms": 70.370797,
     "exposed_nccl_ms": 52.273217,
     "idle_ms": 73.418386
    },
    "1": {
     "busy_ms": 602.854522,
     "compute_ms": 553.019381,
     "nccl_ms": 68.463101,
     "exposed_nccl_ms": 49.835141,
     "idle_ms": 74.083952
    },
    "2": {
     "busy_ms": 602.060216,
     "compute_ms": 550.93573,
     "nccl_ms": 72.251053,
     "exposed_nccl_ms": 51.124486,
     "idle_ms": 74.878258
    },
    "3": {
     "busy_ms": 603.443411,
     "compute_ms": 560.001071,
     "nccl_ms": 58.947272,
     "exposed_nccl_ms": 43.44234,
     "idle_ms": 73.495063
    }
   },
   "dit_all_gpus_idle_ms": 57.858129,
   "dit_idle_gaps": [
    {
     "start_ms": 0.0,
     "len_ms": 22.721459,
     "host_state": {
      "3168673": {
       "nvtx": [],
       "osrt": []
      }
     }
    },
    {
     "start_ms": 670.919253,
     "len_ms": 6.019221,
     "host_state": {
      "3168673": {
       "nvtx": [],
       "osrt": []
      }
     }
    },
    {
     "start_ms": 25.609927,
     "len_ms": 2.394888,
     "host_state": {
      "3168673": {
       "nvtx": [
        "gen/f=24"
       ],
       "osrt": [
        "close (1315us)"
       ]
      }
     }
    },
    {
     "start_ms": 286.062335,
     "len_ms": 1.045614,
     "host_state": {
      "3168673": {
       "nvtx": [
        "gen/f=24",
        "tfm/f=24/c=0/s=2",
        "blk/0"
       ],
       "osrt": []
      }
     }
    },
    {
     "start_ms": 543.32811,
     "len_ms": 0.995765,
     "host_state": {
      "3168673": {
       "nvtx": [
        "gen/f=24",
        "tfm/f=24/c=1/s=None",
        "blk/0"
       ],
       "osrt": []
      }
     }
    },
    {
     "start_ms": 414.560966,
     "len_ms": 0.987238,
     "host_state": {
      "3168673": {
       "nvtx": [
        "gen/f=24",
        "tfm/f=24/c=0/s=3",
        "blk/0"
       ],
       "osrt": []
      }
     }
    }
   ],
   "vae_gpu": {
    "0": {
     "busy_ms": 136.422216,
     "compute_ms": 119.882834,
     "nccl_ms": 16.539382,
     "exposed_nccl_ms": 16.539382,
     "idle_ms": 7.057833
    },
    "1": {
     "busy_ms": 135.661277,
     "compute_ms": 123.080254,
     "nccl_ms": 12.581023,
     "exposed_nccl_ms": 12.581023,
     "idle_ms": 7.818772
    },
    "2": {
     "busy_ms": 137.192817,
     "compute_ms": 120.200227,
     "nccl_ms": 16.99259,
     "exposed_nccl_ms": 16.99259,
     "idle_ms": 6.287232
    },
    "3": {
     "busy_ms": 125.274708,
     "compute_ms": 120.7779,
     "nccl_ms": 4.496808,
     "exposed_nccl_ms": 4.496808,
     "idle_ms": 18.205341
    }
   },
   "vae_d2h_memcpy_ms": 0.978371
  },
  {
   "chunk": 9,
   "host_ns": {
    "dit_request": 262943339708,
    "dit_done": 263662721395,
    "vae_submit": 263667136833,
    "vae_result": 263830056050,
    "ipc_send": 263814547298
   },
   "request_dispatch_ms": 26.999932,
   "host_enqueue_done_to_driver_ms": 144.887345,
   "gpu_tail_after_host_enqueue_ms": 98.177898,
   "result_return_after_last_kernel_ms": 46.709447,
   "prologue_to_first_kernel_ms": 0.06848,
   "dit_wall_ms": 719.381687,
   "gap_dit_to_submit_ms": 4.415438,
   "vae_wall_ms": 162.919217,
   "ipc_result_latency_ms": 15.508752,
   "vae_h2d_ms": 0.480343,
   "ipc_submit_to_worker_ms": 4.421591,
   "vae_barrier_ms": 1.961475,
   "vae_decode_ms": 137.781676,
   "vae_barrier2_ms": 0.685865,
   "vae_convert_ms": 0.20114,
   "vae_d2h_ms": 1.1544,
   "dit_gpu": {
    "0": {
     "busy_ms": 603.986657,
     "compute_ms": 553.498575,
     "nccl_ms": 67.07246,
     "exposed_nccl_ms": 50.488082,
     "idle_ms": 115.39503
    },
    "1": {
     "busy_ms": 603.504002,
     "compute_ms": 553.120839,
     "nccl_ms": 69.258209,
     "exposed_nccl_ms": 50.383163,
     "idle_ms": 115.877685
    },
    "2": {
     "busy_ms": 602.390773,
     "compute_ms": 550.574653,
     "nccl_ms": 72.677739,
     "exposed_nccl_ms": 51.81612,
     "idle_ms": 116.990914
    },
    "3": {
     "busy_ms": 603.728949,
     "compute_ms": 558.74239,
     "nccl_ms": 61.814599,
     "exposed_nccl_ms": 44.986559,
     "idle_ms": 115.652738
    }
   },
   "dit_all_gpus_idle_ms": 99.563046,
   "dit_idle_gaps": [
    {
     "start_ms": 672.67224,
     "len_ms": 46.709447,
     "host_state": {
      "3168673": {
       "nvtx": [],
       "osrt": []
      }
     }
    },
    {
     "start_ms": 0.0,
     "len_ms": 24.529396,
     "host_state": {
      "3168673": {
       "nvtx": [],
       "osrt": []
      }
     }
    },
    {
     "start_ms": 27.489551,
     "len_ms": 2.333525,
     "host_state": {
      "3168673": {
       "nvtx": [
        "gen/f=27"
       ],
       "osrt": [
        "close (1308us)"
       ]
      }
     }
    },
    {
     "start_ms": 545.214657,
     "len_ms": 1.06976,
     "host_state": {
      "3168673": {
       "nvtx": [
        "gen/f=27",
        "tfm/f=27/c=1/s=None",
        "blk/0"
       ],
       "osrt": []
      }
     }
    },
    {
     "start_ms": 287.355056,
     "len_ms": 0.936899,
     "host_state": {
      "3168673": {
       "nvtx": [
        "gen/f=27",
        "tfm/f=27/c=0/s=2",
        "blk/0"
       ],
       "osrt": []
      }
     }
    },
    {
     "start_ms": 416.35681,
     "len_ms": 0.82783,
     "host_state": {
      "3168673": {
       "nvtx": [
        "gen/f=27",
        "tfm/f=27/c=0/s=3",
        "blk/0"
       ],
       "osrt": []
      }
     }
    }
   ],
   "vae_gpu": {
    "0": {
     "busy_ms": 131.041883,
     "compute_ms": 119.640568,
     "nccl_ms": 11.401315,
     "exposed_nccl_ms": 11.401315,
     "idle_ms": 6.739793
    },
    "1": {
     "busy_ms": 129.354667,
     "compute_ms": 121.276896,
     "nccl_ms": 8.077771,
     "exposed_nccl_ms": 8.077771,
     "idle_ms": 8.427009
    },
    "2": {
     "busy_ms": 130.522217,
     "compute_ms": 120.069077,
     "nccl_ms": 10.45314,
     "exposed_nccl_ms": 10.45314,
     "idle_ms": 7.259459
    },
    "3": {
     "busy_ms": 124.555645,
     "compute_ms": 120.00049,
     "nccl_ms": 4.555155,
     "exposed_nccl_ms": 4.555155,
     "idle_ms": 13.226031
    }
   },
   "vae_d2h_memcpy_ms": 0.987619
  }
 ],
 "seconds": 20.764084577560425,
 "scope": "Epoch-1 chunks of the diagnostic ledger run (nsys+NVTX; two VAE chunks also carry dispatch-mode overhead). Host stage times from driver/worker marks and ranges; GPU busy = union of kernel intervals per physical GPU inside the stage window; exposed NCCL = NCCL kernel time not overlapped by compute kernels on the same GPU. Not clean timing; the CPU-ready boundary excludes encoding/network."
}