{ "generated_at": "2026-09-10T05:28:32.259525+00:00", "status": "counter_output_checks_passed_with_scheduler_note", "jobs": [ { "stage": "dit", "status": "passed", "scheduler_note": [], "output": "/data/zhoutaichang/feature/Lingbot_world_realtime/runs/analysis_report_20260908/raw/ncu_missing704_255d8b6427b0b5a7_v1", "elapsed_seconds": 12.073920965194702, "allocated": "0", "input_sha256": "8e60010c4ab22e3ad90412ff00e301e28227cf54db05320e456d91009e843787", "correctness": { "op": "aten.linear.default", "stage": "dit", "shape": [ 1, 2640, 5120 ], "exact": true, "max_abs": 0.0, "rmse": 0.0, "finite": true, "math_policy": { "matmul_precision": "highest", "matmul_allow_tf32": false, "cudnn_allow_tf32": true, "cudnn_benchmark": false, "cudnn_deterministic": false }, "scope": "Original ATen op on saved real tensors; isolated diagnostic" }, "kernel_count": 1, "kernels": [ { "id": "0", "name": "nvjet_sm90_tst_192x192_64x4_1x2_h_bz_coopB_bias_TNN", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 185.344, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 87.977072, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 20.855918, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 42.173961, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 14.524666, "unit": "%" }, "dram__bytes_read.sum": { "value": 161.596672, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 24.357888, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 1.003294, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 168.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 214.228, "unit": "Kbyte/block" } } } ], "sha256": { "guard_own_pid_history_v2.json": "ab40312fb958467b92adc5b9d8b1094da70da562f84d22d62dde79eb8ca5a55c", "replay_comparison.json": "78fe5259ed70cf9417213fcd5b9df6f069872d21ae8d64dd8d8440ffc3e6192b", "metrics.csv": "43416f95805c211eba76b7145dc8697030d8356b8fd9957b5818538975bfbbe0", "replay_comparison.output.pt": "df718ffb6296582f2673361973e0b05950220ff961e8e3e4379cd352d6c20e6b", "operator.ncu-rep": "91e4a74619fe7519a26a6ec8f46c422ddb38223de452e0af42f64e2e58c5e640", "run.log": "3b35d3aaf356bc64b399919c4774d805b49d2458f419cf614de15d1e293495d2", "identity.json": "d04e0129c99c875ed53f5139d08c5044759f9ed8765661eaf471dd971417d648", "replay_operator.py": "cbea82d5e280ab2be0b1ce8dfebce6c1e80888c1d9fe727638e97a2652578d9b", "manifest.json": "cc49ffc329f59d7541062a1b8fc427d6414b90ec6d26937f7ec6f1167470abde", "guard_own_pid_history.json": "a066004ecfeac3feb9b5877c2f48c1df1921ba4c01af7519c3626f44d34574f4" } }, { "stage": "dit", "status": "passed", "scheduler_note": [], "output": "/data/zhoutaichang/feature/Lingbot_world_realtime/runs/analysis_report_20260908/raw/ncu_missing704_db47f7eb1f64a850_v1", "elapsed_seconds": 12.020690202713013, "allocated": "0", "input_sha256": "5486c3201c4910ee4de3cbdc2b72d468ede3117a03f1e731537cf900e60ecc9d", "correctness": { "op": "aten.linear.default", "stage": "dit", "shape": [ 1, 2640, 15360 ], "exact": true, "max_abs": 0.0, "rmse": 0.0, "finite": true, "math_policy": { "matmul_precision": "highest", "matmul_allow_tf32": false, "cudnn_allow_tf32": true, "cudnn_benchmark": false, "cudnn_deterministic": false }, "scope": "Original ATen op on saved real tensors; isolated diagnostic" }, "kernel_count": 1, "kernels": [ { "id": "0", "name": "nvjet_sm90_tst_192x208_64x4_2x1_v_bz_coopB_bias_TNT", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 518.112, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 92.989233, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 21.168798, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 47.743474, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 14.724952, "unit": "%" }, "dram__bytes_read.sum": { "value": 451.653888, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 76.100864, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 1.018611, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 168.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 222.42, "unit": "Kbyte/block" } } } ], "sha256": { "guard_own_pid_history_v2.json": "7cb57ffec83bb70cb5346dd3fea0659825dfdcdaacb59164b3d2f09649ac3f5b", "replay_comparison.json": "fdcb915d6d42f9678323493da94426d5fdbb4182f26d5e21c09e5ec0c9170581", "metrics.csv": "91c8546c250f9ef8e50ba379ebf0cab402c9e366c37d0735baebf50912d4c6d9", "replay_comparison.output.pt": "f9b6cb7bef02816978837a92cf254d8d28e2170300bcf6c79e402c3736c1bf58", "operator.ncu-rep": "3d05d7fd143ba111684d2af396e82a6a98ea4e7b936d522335df4a1ccb43fbf9", "run.log": "d88b194f04d77da7672fe449d4c731c22b01d261b2f1c405404dc01de91a9c92", "identity.json": "af90fd3f65ea6f4ac46e8894e590081acf7eaafd8a5f870124bfa0befe755a65", "replay_operator.py": "cbea82d5e280ab2be0b1ce8dfebce6c1e80888c1d9fe727638e97a2652578d9b", "manifest.json": "8cb0fa3569ec7653f95a468ca1575f0d77b0de394357a11bb9649a59a4485922", "guard_own_pid_history.json": "3889685902ec27ea99668838a80709060eb8a39295dff18cc2c340ecc2cf8f1f" } }, { "stage": "dit", "status": "passed", "scheduler_note": [], "output": "/data/zhoutaichang/feature/Lingbot_world_realtime/runs/analysis_report_20260908/raw/ncu_missing704_1b9e93116d235408_v1", "elapsed_seconds": 13.044363021850586, "allocated": "0", "input_sha256": "75bf41dc2a7a283f82bdcd0635998b9bc8c93a30b0ebb2b209ce44fd3b9e360c", "correctness": { "op": "aten.linear.default", "stage": "dit", "shape": [ 1, 2640, 5120 ], "exact": true, "max_abs": 0.0, "rmse": 0.0, "finite": true, "math_policy": { "matmul_precision": "highest", "matmul_allow_tf32": false, "cudnn_allow_tf32": true, "cudnn_benchmark": false, "cudnn_deterministic": false }, "scope": "Original ATen op on saved real tensors; isolated diagnostic" }, "kernel_count": 1, "kernels": [ { "id": "0", "name": "nvjet_sm90_tst_192x192_64x4_1x2_h_bz_coopB_bias_TNN", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 472.896, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 91.75556, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 20.160094, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 43.729417, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 14.576527, "unit": "%" }, "dram__bytes_read.sum": { "value": 436.227072, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 22.518272, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 970.0766, "unit": "Gbyte/s" }, "launch__registers_per_thread": { "value": 168.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 214.228, "unit": "Kbyte/block" } } } ], "sha256": { "guard_own_pid_history_v2.json": "fba609d4567bee023da31e772edaa53faba94c4e986da05987d13cff8a567434", "replay_comparison.json": "78fe5259ed70cf9417213fcd5b9df6f069872d21ae8d64dd8d8440ffc3e6192b", "metrics.csv": "4f8fc65b6645bc2ead7afa5c2da4ecd5b06e5b2753de811bbfb90f26ca6ef6fb", "replay_comparison.output.pt": "93c83a4b877b3d1b104dc40152b8ba147f5114489523985320e4ab7a628225b9", "operator.ncu-rep": "3e14bcfa8c3a75610aaf86dd35bf11cd4f3dfdffe03bc720fc75baec19936a27", "run.log": "4f2aeeac4bc261966dd1e90298b0cb9a82e234b7b2e06a0e0ebe6182fba25bf9", "identity.json": "1719aead35c19ae3429ac7b2fbdc4ab844adb00e977be62c751d88fe7eff444c", "replay_operator.py": "cbea82d5e280ab2be0b1ce8dfebce6c1e80888c1d9fe727638e97a2652578d9b", "manifest.json": "f70962e64471db3056d881c534da5cdbcc996a454fbeca34303ebf1122f24db9", "guard_own_pid_history.json": "a829ac48a20a0c7d5e87154fe6b1e6a6155c6f1cb5e29a7fa8c93931cf0ca269" } }, { "stage": "dit", "status": "passed", "scheduler_note": [], "output": "/data/zhoutaichang/feature/Lingbot_world_realtime/runs/analysis_report_20260908/raw/ncu_missing704_9ba227caa2b6dfb2_v1", "elapsed_seconds": 12.339098453521729, "allocated": "0", "input_sha256": "c08dc8cdb79c43328ef6849ee7e60049c20bbb243f50823bd9a80415eb91dc94", "correctness": { "op": "aten.linear.default", "stage": "dit", "shape": [ 1, 2640, 13824 ], "exact": true, "max_abs": 0.0, "rmse": 0.0, "finite": true, "math_policy": { "matmul_precision": "highest", "matmul_allow_tf32": false, "cudnn_allow_tf32": true, "cudnn_benchmark": false, "cudnn_deterministic": false }, "scope": "Original ATen op on saved real tensors; isolated diagnostic" }, "kernel_count": 1, "kernels": [ { "id": "0", "name": "nvjet_sm90_tst_128x240_64x4_2x1_v_bz_coopA_bias_TNT", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 451.04, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 94.507539, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 20.897931, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 53.350468, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 14.758237, "unit": "%" }, "dram__bytes_read.sum": { "value": 384.699648, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 68.826368, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 1.005512, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 168.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 205.988, "unit": "Kbyte/block" } } } ], "sha256": { "guard_own_pid_history_v2.json": "16348cdab0152d701f10060f016bf2e34f584b96c5fdb8c8fda3b3cdcef5194d", "replay_comparison.json": "8060ec2b5b72561e35775fe6c7d870ecc3d004d1f80637796bd5ce212ed0ec6e", "metrics.csv": "395668c3b933dfb7929f93b0ae5461ad6886bd1372fac00bb7fc09403084a16f", "replay_comparison.output.pt": "2618796ace96b7534246132dbd9b7d3277039bd76c38b5831c96ce72cf4ebf6c", "operator.ncu-rep": "82e60605a54146fa7218ed079743d928fb67eaa9cf295796272e74e5e9247415", "run.log": "b6193016e254e09742aad384925dcf372fc85ae6148091ca5e651228487799e8", "identity.json": "c5cdc15f116309c64c07788696c21e265fa2a0af8bd86685dc23f74b8401c737", "replay_operator.py": "cbea82d5e280ab2be0b1ce8dfebce6c1e80888c1d9fe727638e97a2652578d9b", "manifest.json": "178c200e1a6f6809a140e4c1718c76213be9c34b0851dddbe4f92ee89a7b9514", "guard_own_pid_history.json": "32edf4b5f16567b8a1155ee5c4b16fc659ff68d0cb99386c5a29afd7ad1b5835" } }, { "stage": "dit", "status": "passed", "scheduler_note": [], "output": "/data/zhoutaichang/feature/Lingbot_world_realtime/runs/analysis_report_20260908/raw/ncu_missing704_697ad043e70ad841_v1", "elapsed_seconds": 13.675179243087769, "allocated": "0", "input_sha256": "984423718cf2b4425d4e6c3c3cc45ee71db6a1b3d36906b61da17257f41a950c", "correctness": { "op": "aten.linear.default", "stage": "dit", "shape": [ 1, 2640, 5120 ], "exact": true, "max_abs": 0.0, "rmse": 0.0, "finite": true, "math_policy": { "matmul_precision": "highest", "matmul_allow_tf32": false, "cudnn_allow_tf32": true, "cudnn_benchmark": false, "cudnn_deterministic": false }, "scope": "Original ATen op on saved real tensors; isolated diagnostic" }, "kernel_count": 1, "kernels": [ { "id": "0", "name": "nvjet_sm90_tst_192x192_64x4_1x2_h_bz_coopB_bias_TNN", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 183.52, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 88.174905, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 20.924745, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 42.676095, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 14.558241, "unit": "%" }, "dram__bytes_read.sum": { "value": 161.599232, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 23.164672, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 1.006778, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 168.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 214.228, "unit": "Kbyte/block" } } } ], "sha256": { "guard_own_pid_history_v2.json": "f63c928441b7fa6d4ac7b3f03ee7153f036f59143f466f7808834396226e022f", "replay_comparison.json": "78fe5259ed70cf9417213fcd5b9df6f069872d21ae8d64dd8d8440ffc3e6192b", "metrics.csv": "a63f156a08877d29b1133170a0382e77a181045f822abb0b93c8ff78b947a09e", "replay_comparison.output.pt": "8aa83639f0269c95d932d8c5d25ffeda2ac0600d87cb384f522593e1a6f954c1", "operator.ncu-rep": "c43ae08beedd59c7a13719837f39dd4c0b4e6cefe484921bcbc7809b27c43117", "run.log": "c84514dadf833cacdd7292b955b40608f696d9711eec9f1e56091b4efddd1beb", "identity.json": "2551b4316d2af844ef8e3d4fcf5c9c9707c55b6a6e1b8e26b84dc715c912df11", "replay_operator.py": "cbea82d5e280ab2be0b1ce8dfebce6c1e80888c1d9fe727638e97a2652578d9b", "manifest.json": "b29bfdc19e0fe2a8fcb3fd203a52f45cbaa2fe1ad7e6dc8cedc30db28cbf099c", "guard_own_pid_history.json": "9a8e363117e153c5d6b6b2cb71119129ec15b391ac1fe7fe1972d400ce9f0acd" } }, { "stage": "vae", "status": "passed", "scheduler_note": [], "output": "/data/zhoutaichang/feature/Lingbot_world_realtime/runs/analysis_report_20260908/raw/ncu_missing704_0551e0aa26bb7201_v1", "elapsed_seconds": 20.73886227607727, "allocated": "0", "input_sha256": "361110b0df425093397551c589c4c8f8e119f195c1b7c8ce7fe463e598338f6e", "correctness": { "op": "aten.conv3d.default", "stage": "vae", "shape": [ 1, 96, 4, 704, 320 ], "exact": true, "max_abs": 0.0, "rmse": 0.0, "finite": true, "math_policy": { "matmul_precision": "highest", "matmul_allow_tf32": false, "cudnn_allow_tf32": true, "cudnn_benchmark": false, "cudnn_deterministic": false }, "scope": "Original ATen op on saved real tensors; isolated diagnostic" }, "kernel_count": 5, "kernels": [ { "id": "0", "name": "void cudnn::nchwToNhwcKernel(cudnn::nchw2nhwc_params_t, const T1 *, T2 *)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 0.316928, "unit": "ms" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 37.462203, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 67.515515, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 80.918825, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 94.629991, "unit": "%" }, "dram__bytes_read.sum": { "value": 0.523785, "unit": "Gbyte" }, "dram__bytes_write.sum": { "value": 505.793792, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 3.248621, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 32.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 5.248, "unit": "Kbyte/block" } } }, { "id": "1", "name": "void cudnn::nchwToNhwcKernel(cudnn::nchw2nhwc_params_t, const T1 *, T2 *)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 0.003712, "unit": "ms" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 11.825452, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 5.664136, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 14.672777, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 25.471919, "unit": "%" }, "dram__bytes_read.sum": { "value": 0.001004, "unit": "Gbyte" }, "dram__bytes_write.sum": { "value": 0.000256, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 0.270621, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 32.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 5.248, "unit": "Kbyte/block" } } }, { "id": "2", "name": "sm90_xmma_fprop_implicit_gemm_f32f32_tf32f32_f32_nhwckrsc_nhwc_tilesize256x32x32_warpgroupsize1x1x1_g1_execute_segment_k_off_kernel__5x_cudnn", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 2.926656, "unit": "ms" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 31.552625, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 9.867081, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 62.728996, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 18.36999, "unit": "%" }, "dram__bytes_read.sum": { "value": 1.049221, "unit": "Gbyte" }, "dram__bytes_write.sum": { "value": 340.380928, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 0.474809, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 168.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 232.448, "unit": "Kbyte/block" } } }, { "id": "3", "name": "void cudnn::nhwcToNchwKernel(cudnn::nhwc2nchw_params_t, const T1 *, T2 *)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 0.18816, "unit": "ms" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 40.001026, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 74.2135, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 83.138163, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 93.315156, "unit": "%" }, "dram__bytes_read.sum": { "value": 0.346048, "unit": "Gbyte" }, "dram__bytes_write.sum": { "value": 325.758976, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 3.570403, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 32.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 5.248, "unit": "Kbyte/block" } } }, { "id": "4", "name": "void at::elementwise_kernel<128, 2, void at::gpu_kernel_impl_nocast>(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 0.242496, "unit": "ms" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 59.75911, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 57.199903, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 64.354426, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 83.664485, "unit": "%" }, "dram__bytes_read.sum": { "value": 0.346046, "unit": "Gbyte" }, "dram__bytes_write.sum": { "value": 321.334528, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 2.75213, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 18.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 1.024, "unit": "Kbyte/block" } } } ], "sha256": { "guard_own_pid_history_v2.json": "2429391d306bbfd11338d46184dac9127536db7d518beced75d33e60395ce49c", "replay_comparison.json": "5d1dd24219140eeea3da6c1492d301443d2a7c562799c1688d0c36f6046c94d9", "metrics.csv": "2bc7b21500e77e41becaebf0c80515918ddb5a283e183d46b987e5d33f92a12c", "replay_comparison.output.pt": "d2619c4b5220cb14d45454d17b8070ad38df8acb2950b30bd6d479c425ebef16", "operator.ncu-rep": "fcb2ab259fe9dc9575e201a038cdc3f15a4da12319999f712ccbdf7a3c2fc50e", "run.log": "bda5e832ffed4073c8bdbc7fef8a2df060e14cde49238ed6fdefd94fef9a74f9", "identity.json": "32a52c645183434a40faf14b100c1294b60fec5dacf281cb9d66813b4189d6ad", "replay_operator.py": "cbea82d5e280ab2be0b1ce8dfebce6c1e80888c1d9fe727638e97a2652578d9b", "manifest.json": "c59dfdfb2cfb57e15257d5997bcbc3edf4b94ae74f9c6858cc482d49b64f92f6", "guard_own_pid_history.json": "8840412eacc3dcfe36aeab3c5abb44693d5fb18626e509a254288ee2cb9d043f" } }, { "stage": "vae", "status": "passed", "scheduler_note": [], "output": "/data/zhoutaichang/feature/Lingbot_world_realtime/runs/analysis_report_20260908/raw/ncu_missing704_1b34e3d0553bbca4_v1", "elapsed_seconds": 20.13989543914795, "allocated": "0", "input_sha256": "836c7ee7ca56334d7997004c7d80dc3a7742322bc7befd6ad4a0f29a4ec8ad0f", "correctness": { "op": "aten.conv3d.default", "stage": "vae", "shape": [ 1, 192, 4, 352, 160 ], "exact": true, "max_abs": 0.0, "rmse": 0.0, "finite": true, "math_policy": { "matmul_precision": "highest", "matmul_allow_tf32": false, "cudnn_allow_tf32": true, "cudnn_benchmark": false, "cudnn_deterministic": false }, "scope": "Original ATen op on saved real tensors; isolated diagnostic" }, "kernel_count": 5, "kernels": [ { "id": "0", "name": "void cudnn::nchwToNhwcKernel(cudnn::nchw2nhwc_params_t, const T1 *, T2 *)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 0.154816, "unit": "ms" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 38.994795, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 68.579423, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 83.039358, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 92.634988, "unit": "%" }, "dram__bytes_read.sum": { "value": 264.27776, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 246.554624, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 3.29961, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 32.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 5.248, "unit": "Kbyte/block" } } }, { "id": "1", "name": "void cudnn::nchwToNhwcKernel(cudnn::nchw2nhwc_params_t, const T1 *, T2 *)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 0.004992, "unit": "ms" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 32.519013, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 16.697735, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 39.4128, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 74.119453, "unit": "%" }, "dram__bytes_read.sum": { "value": 3.990016, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 0.0, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 0.799282, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 32.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 5.248, "unit": "Kbyte/block" } } }, { "id": "2", "name": "sm90_xmma_fprop_implicit_gemm_f32f32_tf32f32_f32_nhwckrsc_nhwc_tilesize128x128x32_warpgroupsize1x1x1_g1_execute_segment_k_off_kernel__5x_cudnn", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 1.759712, "unit": "ms" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 70.298033, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 9.176507, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 65.803113, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 18.352035, "unit": "%" }, "dram__bytes_read.sum": { "value": 609.446144, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 167.589632, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 0.44157, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 168.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 232.448, "unit": "Kbyte/block" } } }, { "id": "3", "name": "void cudnn::nhwcToNchwKernel(cudnn::nhwc2nchw_params_t, const T1 *, T2 *)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 0.089984, "unit": "ms" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 39.593024, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 75.325003, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 85.992377, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 92.996436, "unit": "%" }, "dram__bytes_read.sum": { "value": 173.03296, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 153.057536, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 3.623872, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 32.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 5.248, "unit": "Kbyte/block" } } }, { "id": "4", "name": "void at::elementwise_kernel<128, 2, void at::gpu_kernel_impl_nocast>(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 0.121344, "unit": "ms" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 60.091202, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 55.068871, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 63.772259, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 83.859226, "unit": "%" }, "dram__bytes_read.sum": { "value": 173.032192, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 148.393984, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 2.648884, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 18.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 1.024, "unit": "Kbyte/block" } } } ], "sha256": { "guard_own_pid_history_v2.json": "a62526e4962e0f65e5041c81a1dda62c400ce8fdcfd6d586cb2d6d86c45d12be", "replay_comparison.json": "a1f0d520083e788f56bbb773a24810b1820a962c219484193d455186aaf7df2a", "metrics.csv": "b79f1f4706c0c6217b39b35e0b7bc0bfc021033c644d22457e81308e664617cb", "replay_comparison.output.pt": "f40dc85e6ee2452d23b952a2b52084cc69340f54a48c20b8804f9bdba20aa5e2", "operator.ncu-rep": "1a810cb03e7f751f6377cce6604680dd275f60c05c48688bb2a009126bbda720", "run.log": "046dd25d383cfafdaf0491199be563970874bad09a702eab2dfffae0bcfa4520", "identity.json": "3d330b633fa3ace8b30eda98c45fd1c51f41ac4658fd5fc754b5d24f68d9bb25", "replay_operator.py": "cbea82d5e280ab2be0b1ce8dfebce6c1e80888c1d9fe727638e97a2652578d9b", "manifest.json": "011ba0955f73ac530ed6f89751965d218f421d9d877b074e8d954050eaae1dcf", "guard_own_pid_history.json": "1550143a0f471437ada66fdb009b265fb87218d613539f9ff8de2e36e49bf937" } }, { "stage": "dit", "status": "passed", "scheduler_note": [], "output": "/data/zhoutaichang/feature/Lingbot_world_realtime/runs/analysis_report_20260908/raw/ncu_missing704_606cff6747219bac_v1", "elapsed_seconds": 12.97829818725586, "allocated": "0", "input_sha256": "70fed7084bfb383625de9935f9cc6ad7e398ccdf921a0e71276f5cf1bdbdcb0d", "correctness": { "op": "aten.linear.default", "stage": "dit", "shape": [ 1, 10560, 5120 ], "exact": true, "max_abs": 0.0, "rmse": 0.0, "finite": true, "math_policy": { "matmul_precision": "highest", "matmul_allow_tf32": false, "cudnn_allow_tf32": true, "cudnn_benchmark": false, "cudnn_deterministic": false }, "scope": "Original ATen op on saved real tensors; isolated diagnostic" }, "kernel_count": 1, "kernels": [ { "id": "0", "name": "nvjet_sm90_tst_256x128_64x4_1x2_h_bz_coopA_bias_TNT", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 650.336, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 93.499099, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 21.66624, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 55.006145, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 14.779993, "unit": "%" }, "dram__bytes_read.sum": { "value": 569.92512, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 108.10752, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 1.042588, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 168.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 214.212, "unit": "Kbyte/block" } } } ], "sha256": { "guard_own_pid_history_v2.json": "3b12a80ecbdb1fcff0e7e0ee42df97364724fed14e8efe6371890787b976322b", "replay_comparison.json": "bce45cd578dd03d947afe3bc6ab8425eaabae02ad610654b8ab8ceefc61ccefc", "metrics.csv": "4e58ec02c1d23b861e259aba7b6e394c38455e83002c15868d0b33c63e986707", "replay_comparison.output.pt": "d3e48448b1e42925dc741b8e2bf769d7011c8041538ff55c2b0a3f5aaa5d71d8", "operator.ncu-rep": "7b4eb0e851f02702da7568e2e161df38bfbb5ae4b2983726aaa38c4504e5ae9f", "run.log": "51ca74931da523e7a155df6536007fe0d8501722573cc09779b3e108fbbeb24b", "identity.json": "fcd183bc1eaec3e24a6cc7532512d2e4db91d7489ac21138b9ffa72dceadc31c", "replay_operator.py": "cbea82d5e280ab2be0b1ce8dfebce6c1e80888c1d9fe727638e97a2652578d9b", "manifest.json": "01060229caaada753853cb29f580851d028d71f319c34c72547ef152ce115c30", "guard_own_pid_history.json": "ef83f9ed5a5b15d25cd19aafb9697a5f35a11d5ced5c6a8e30e066df109b104f" } }, { "stage": "vae", "status": "passed", "scheduler_note": [], "output": "/data/zhoutaichang/feature/Lingbot_world_realtime/runs/analysis_report_20260908/raw/ncu_missing704_ac3438e4ac0a9485_v1", "elapsed_seconds": 18.498531818389893, "allocated": "0", "input_sha256": "ae178ca2bbf227553643572af23755eb3ed51289f32398e11a61f88fb81ea3a1", "correctness": { "op": "aten.conv3d.default", "stage": "vae", "shape": [ 1, 384, 2, 176, 80 ], "exact": true, "max_abs": 0.0, "rmse": 0.0, "finite": true, "math_policy": { "matmul_precision": "highest", "matmul_allow_tf32": false, "cudnn_allow_tf32": true, "cudnn_benchmark": false, "cudnn_deterministic": false }, "scope": "Original ATen op on saved real tensors; isolated diagnostic" }, "kernel_count": 5, "kernels": [ { "id": "0", "name": "void cudnn::nchwToNhwcKernel(cudnn::nchw2nhwc_params_t, const T1 *, T2 *)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 0.049568, "unit": "ms" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 40.797062, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 67.453876, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 82.632261, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 93.249602, "unit": "%" }, "dram__bytes_read.sum": { "value": 89.6896, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 71.156992, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 3.244968, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 32.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 5.248, "unit": "Kbyte/block" } } }, { "id": "1", "name": "void cudnn::nchwToNhwcKernel(cudnn::nchw2nhwc_params_t, const T1 *, T2 *)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 0.010144, "unit": "ms" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 62.904102, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 34.105876, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 69.333043, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 84.235024, "unit": "%" }, "dram__bytes_read.sum": { "value": 15.934464, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 0.611328, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 1.631091, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 32.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 5.248, "unit": "Kbyte/block" } } }, { "id": "2", "name": "sm90_xmma_fprop_implicit_gemm_f32f32_tf32f32_f32_nhwckrsc_nhwc_tilesize128x128x32_warpgroupsize1x1x1_g1_execute_segment_k_off_kernel__5x_cudnn", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 1.01648, "unit": "ms" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 60.635004, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 6.582824, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 57.383586, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 17.334903, "unit": "%" }, "dram__bytes_read.sum": { "value": 280.979456, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 41.000704, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 0.31676, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 168.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 232.448, "unit": "Kbyte/block" } } }, { "id": "3", "name": "void cudnn::nhwcToNchwKernel(cudnn::nhwc2nchw_params_t, const T1 *, T2 *)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 0.022016, "unit": "ms" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 39.533865, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 61.661659, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 83.086082, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 88.529565, "unit": "%" }, "dram__bytes_read.sum": { "value": 43.271168, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 21.8688, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 2.958756, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 32.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 5.248, "unit": "Kbyte/block" } } }, { "id": "4", "name": "void at::elementwise_kernel<128, 2, void at::gpu_kernel_impl_nocast>(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 0.031328, "unit": "ms" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 57.755959, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 40.419862, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 59.393079, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 80.931138, "unit": "%" }, "dram__bytes_read.sum": { "value": 43.27168, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 17.565184, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 1.941933, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 18.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 1.024, "unit": "Kbyte/block" } } } ], "sha256": { "guard_own_pid_history_v2.json": "013c7a54ec30a472987c80a1b9991f4a1327822a6e7a072265989d6265a1403a", "replay_comparison.json": "60930f2572471940d75b458e424510051d51cf7489567d81905a53cc60aeb4f5", "metrics.csv": "47a5b718228e567b8ac7e9550e0916d17be1aa4bb3f7a29b3a31058b0282232f", "replay_comparison.output.pt": "66d90f3f31491ed885d2b81beee7c2594b4cbc12b3c6abe127209b2472b820b7", "operator.ncu-rep": "49f743c42d5234932e24e9e4c8bc6a63aabceb02174d9cfe21de56e36f3ab398", "run.log": "91971fa7572749312c58ace3ef9bc63148249be0a49c97ba86044aaa5c1efb62", "identity.json": "03ee538ae3a2a49ed96f51bbb8f2dc60ba96a0dd9897f3106a6589acefa22b9d", "replay_operator.py": "cbea82d5e280ab2be0b1ce8dfebce6c1e80888c1d9fe727638e97a2652578d9b", "manifest.json": "6b79c5e4047ab36fb097f366f552081a1c3a65b8ba0f7b37f20a0d1fcd8c5a27", "guard_own_pid_history.json": "da39a901c5e9e90b8614f61e23669791e803b9c33c331747348dcf04ee261bf8" } }, { "stage": "vae", "status": "passed", "scheduler_note": [], "output": "/data/zhoutaichang/feature/Lingbot_world_realtime/runs/analysis_report_20260908/raw/ncu_missing704_a50d349fe3a8545f_v1", "elapsed_seconds": 19.154671907424927, "allocated": "0", "input_sha256": "fd2ac936cea7b4d5d60612a6045f5c612e27536de2424aa111e4bc689fc1e795", "correctness": { "op": "aten.conv3d.default", "stage": "vae", "shape": [ 1, 384, 1, 88, 40 ], "exact": true, "max_abs": 0.0, "rmse": 0.0, "finite": true, "math_policy": { "matmul_precision": "highest", "matmul_allow_tf32": false, "cudnn_allow_tf32": true, "cudnn_benchmark": false, "cudnn_deterministic": false }, "scope": "Original ATen op on saved real tensors; isolated diagnostic" }, "kernel_count": 5, "kernels": [ { "id": "0", "name": "void cudnn::nchwToNhwcKernel(cudnn::nchw2nhwc_params_t, const T1 *, T2 *)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 10.432, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 38.363356, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 37.9252, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 72.882164, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 81.69635, "unit": "%" }, "dram__bytes_read.sum": { "value": 17.428224, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 1.563648, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 1.82054, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 32.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 5.248, "unit": "Kbyte/block" } } }, { "id": "1", "name": "void cudnn::nchwToNhwcKernel(cudnn::nchw2nhwc_params_t, const T1 *, T2 *)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 10.272, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 61.807772, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 33.868591, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 68.398037, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 84.348962, "unit": "%" }, "dram__bytes_read.sum": { "value": 15.934464, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 0.778752, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 1.627065, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 32.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 5.248, "unit": "Kbyte/block" } } }, { "id": "2", "name": "sm90_xmma_fprop_implicit_gemm_f32f32_tf32f32_f32_nhwckrsc_nhwc_tilesize128x128x32_warpgroupsize1x1x1_g1_execute_segment_k_off_kernel__5x_cudnn", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 147.488, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 53.616532, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 5.270415, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 47.961199, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 11.166581, "unit": "%" }, "dram__bytes_read.sum": { "value": 33.41184, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 3.982336, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 0.25354, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 168.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 232.448, "unit": "Kbyte/block" } } }, { "id": "3", "name": "void cudnn::nhwcToNchwKernel(cudnn::nhwc2nchw_params_t, const T1 *, T2 *)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 5.568, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 20.483167, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 20.347906, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 45.328474, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 69.817103, "unit": "%" }, "dram__bytes_read.sum": { "value": 5.42208, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 0.0, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 0.973793, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 32.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 5.248, "unit": "Kbyte/block" } } }, { "id": "4", "name": "void at::elementwise_kernel<128, 2, void at::gpu_kernel_impl_nocast>(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 6.56, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 35.163881, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 17.263871, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 38.049735, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 68.626808, "unit": "%" }, "dram__bytes_read.sum": { "value": 5.422592, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 0.000256, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 0.826654, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 18.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 1.024, "unit": "Kbyte/block" } } } ], "sha256": { "guard_own_pid_history_v2.json": "5117b8f4e86e6e7c22828d4815ffddff1cb17fb498927f24b4671fece537a23c", "replay_comparison.json": "385b975e001198fe437d8490d91614dd8c52aec195eeeab5d94028edff5c5e7e", "metrics.csv": "aff4d277129a49b56a0b330a7043300d5a021b5d482de1d412b8ef46e3ea21c2", "replay_comparison.output.pt": "d506e4c0b6dffde1677d9c2afe237d5dec0e0bbf365f2fe3edde5f0874b0c124", "operator.ncu-rep": "de8f05ef08f94750db9bee362956be6d4b1f2e5675f05b6efc62d6baf98973a5", "run.log": "ba7ea79bbb1d7795a9efa39971046b6511170c6b1a441f211f228b9132b9b37d", "identity.json": "4c18f1f4c8c86d6ffb836bd1c01ed975055e1c1443d11dbf917594ccd66a61eb", "replay_operator.py": "cbea82d5e280ab2be0b1ce8dfebce6c1e80888c1d9fe727638e97a2652578d9b", "manifest.json": "ca8aecb486cef18a6df6c5a1114ed53a065d1dc00cf44ad586ed4a0be4a6c42c", "guard_own_pid_history.json": "7b289e499486195c0440de94c8ed8116e95b151c9d6c570fe063420e49587704" } }, { "stage": "dit", "status": "counter_output_passed_scheduler_note", "scheduler_note": [ { "gpu_id": 0, "pid": 2019660, "user": "unknown", "kind": "foreign", "holder": "zhoutaichang", "holder_account": "zhoutaichang", "command": "[No data]", "memory_mb": 2266, "first_seen": "2026-09-10T05:20:53.219014752Z", "last_seen": "2026-09-10T05:20:54.100960726Z", "sightings": 3, "warn_count": 0, "last_warned": null, "last_signal": null, "end_time": "2026-09-10T05:20:55.549497427Z", "resolution": "process exited" } ], "output": "/data/zhoutaichang/feature/Lingbot_world_realtime/runs/analysis_report_20260908/raw/ncu_missing704_4cfc56167ab9dd35_v1", "elapsed_seconds": 13.774619579315186, "allocated": "0", "input_sha256": "94f48ff6a98bf87d9ac8185c5db13ca77c238c34547a0c86dd5cac7b7d5b326f", "correctness": { "op": "aten.linear.default", "stage": "dit", "shape": [ 1, 10560, 5120 ], "exact": true, "max_abs": 0.0, "rmse": 0.0, "finite": true, "math_policy": { "matmul_precision": "highest", "matmul_allow_tf32": false, "cudnn_allow_tf32": true, "cudnn_benchmark": false, "cudnn_deterministic": false }, "scope": "Original ATen op on saved real tensors; isolated diagnostic" }, "kernel_count": 1, "kernels": [ { "id": "0", "name": "nvjet_sm90_tst_256x128_64x4_1x2_h_bz_coopA_bias_TNT", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 208.928, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 86.043035, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 16.886047, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 54.194965, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 14.740247, "unit": "%" }, "dram__bytes_read.sum": { "value": 77.00992, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 92.732928, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 812.446623, "unit": "Gbyte/s" }, "launch__registers_per_thread": { "value": 168.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 214.212, "unit": "Kbyte/block" } } } ], "sha256": { "guard_own_pid_history_v2.json": "885aed506ae04f9b6271265e60b27e98c99c79167f7b6b3d7b719848c757042b", "replay_comparison.json": "bce45cd578dd03d947afe3bc6ab8425eaabae02ad610654b8ab8ceefc61ccefc", "metrics.csv": "92b9c5fb515f4e9cd1e26b9dfd38bfbd18745ffdfb4c638173a116a9f450416d", "replay_comparison.output.pt": "672369aef55d7cc82b54a7785a87c5d100ebb472d3b36605077086fb529a6491", "operator.ncu-rep": "b8a80a027941cdf0f175b7804e017deca44a3b1722fef3977d9c3e09bc44ae6f", "run.log": "b1f7fa90ad0800e08518c9ebd01f5a26198835d34932d1f5cc3cf9af66c3b49b", "identity.json": "2917e8ab6155da4b0a2dc841bbe0510db2a323716f03eab3a56debd386d43f12", "replay_operator.py": "cbea82d5e280ab2be0b1ce8dfebce6c1e80888c1d9fe727638e97a2652578d9b", "manifest.json": "1949f1657b4cecd44cce2ac25b33e2a806081071e9de4eb29b59d72879ff211d" } }, { "stage": "vae", "status": "passed", "scheduler_note": [], "output": "/data/zhoutaichang/feature/Lingbot_world_realtime/runs/analysis_report_20260908/raw/ncu_missing704_053cda5cb0c0dcf6_v1", "elapsed_seconds": 18.673240900039673, "allocated": "0", "input_sha256": "146fb336db70349e18bee5b6332d0ab63940d20d95e88e53d065f0d87a4673e8", "correctness": { "op": "aten.conv3d.default", "stage": "vae", "shape": [ 1, 384, 2, 176, 80 ], "exact": true, "max_abs": 0.0, "rmse": 0.0, "finite": true, "math_policy": { "matmul_precision": "highest", "matmul_allow_tf32": false, "cudnn_allow_tf32": true, "cudnn_benchmark": false, "cudnn_deterministic": false }, "scope": "Original ATen op on saved real tensors; isolated diagnostic" }, "kernel_count": 5, "kernels": [ { "id": "0", "name": "void cudnn::nchwToNhwcKernel(cudnn::nchw2nhwc_params_t, const T1 *, T2 *)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 24.416, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 41.627845, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 60.371031, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 81.6219, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 90.34357, "unit": "%" }, "dram__bytes_read.sum": { "value": 44.848896, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 25.988096, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 2.901253, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 32.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 5.248, "unit": "Kbyte/block" } } }, { "id": "1", "name": "void cudnn::nchwToNhwcKernel(cudnn::nchw2nhwc_params_t, const T1 *, T2 *)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 6.752, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 47.45892, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 24.646973, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 54.025325, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 79.402757, "unit": "%" }, "dram__bytes_read.sum": { "value": 7.971328, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 0.0, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 1.180588, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 32.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 5.248, "unit": "Kbyte/block" } } }, { "id": "2", "name": "sm90_xmma_fprop_implicit_gemm_f32f32_tf32f32_f32_nhwckrsc_nhwc_tilesize128x128x32_warpgroupsize1x1x1_g1_execute_segment_k_off_kernel__5x_cudnn", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 519.68, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 59.65408, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 4.809225, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 55.831597, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 17.350418, "unit": "%" }, "dram__bytes_read.sum": { "value": 82.941952, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 37.313792, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 0.231403, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 168.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 232.448, "unit": "Kbyte/block" } } }, { "id": "3", "name": "void cudnn::nhwcToNchwKernel(cudnn::nhwc2nchw_params_t, const T1 *, T2 *)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 22.4, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 38.760723, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 60.568158, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 81.773657, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 89.061875, "unit": "%" }, "dram__bytes_read.sum": { "value": 43.271424, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 21.942528, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 2.911337, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 32.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 5.248, "unit": "Kbyte/block" } } }, { "id": "4", "name": "void at::elementwise_kernel<128, 2, void at::gpu_kernel_impl_nocast>(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 30.72, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 58.861714, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 41.254908, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 60.558228, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 78.51864, "unit": "%" }, "dram__bytes_read.sum": { "value": 43.272192, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 17.630208, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 1.9825, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 18.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 1.024, "unit": "Kbyte/block" } } } ], "sha256": { "guard_own_pid_history_v2.json": "8ed6d5d93dbda11ee9cf698ff69c351881134b2a691359b07b929a40fe95d47f", "replay_comparison.json": "60930f2572471940d75b458e424510051d51cf7489567d81905a53cc60aeb4f5", "metrics.csv": "2c331c70a4a01e031b488d5080c59a8d50c4cba7b78d20a59b50b420601ecfdf", "replay_comparison.output.pt": "59b3681fa6e53763b47b45beecbfdb97e70d469f47534c6aac474c5b1da9ffbb", "operator.ncu-rep": "6956efdebdc47978a19233844f04040ac161cb7e7a09224c09fe562abd24641d", "run.log": "4b4077d61e179e4a6a49dd74c55e9a35ea3141321c5735ea1be5c03e246470a2", "identity.json": "05af298dbb245bcfd8f7634ec9d22f3992de873d3969c5dc36316c11a55c08e9", "replay_operator.py": "cbea82d5e280ab2be0b1ce8dfebce6c1e80888c1d9fe727638e97a2652578d9b", "manifest.json": "be02910ef4f690318a0187df7da211a555cc794bc160ef568647341bd493f6fd" } }, { "stage": "vae", "status": "passed", "scheduler_note": [], "output": "/data/zhoutaichang/feature/Lingbot_world_realtime/runs/analysis_report_20260908/raw/ncu_missing704_4273bf5ee693569e_v1", "elapsed_seconds": 18.38431215286255, "allocated": "0", "input_sha256": "05ade70ee900de950adaf23171001399a4d6cc88017bf23770bc74ef28c01d3b", "correctness": { "op": "aten.conv3d.default", "stage": "vae", "shape": [ 1, 768, 2, 176, 80 ], "exact": true, "max_abs": 0.0, "rmse": 0.0, "finite": true, "math_policy": { "matmul_precision": "highest", "matmul_allow_tf32": false, "cudnn_allow_tf32": true, "cudnn_benchmark": false, "cudnn_deterministic": false }, "scope": "Original ATen op on saved real tensors; isolated diagnostic" }, "kernel_count": 5, "kernels": [ { "id": "0", "name": "void cudnn::nchwToNhwcKernel(cudnn::nchw2nhwc_params_t, const T1 *, T2 *)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 48.256, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 40.434914, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 66.100578, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 79.763297, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 92.503063, "unit": "%" }, "dram__bytes_read.sum": { "value": 86.532352, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 66.80576, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 3.177597, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 32.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 5.248, "unit": "Kbyte/block" } } }, { "id": "1", "name": "void cudnn::nchwToNhwcKernel(cudnn::nchw2nhwc_params_t, const T1 *, T2 *)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 15.2, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 82.436652, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 4.856264, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 12.976548, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 85.396044, "unit": "%" }, "dram__bytes_read.sum": { "value": 3.547904, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 0.0, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 0.233415, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 32.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 5.248, "unit": "Kbyte/block" } } }, { "id": "2", "name": "sm90_xmma_fprop_implicit_gemm_f32f32_tf32f32_f32_nhwckrsc_nhwc_tilesize128x128x32_warpgroupsize1x1x1_g1_execute_segment_k_off_kernel__5x_cudnn", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 129.952, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 88.980992, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 33.04372, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 73.52943, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 17.852009, "unit": "%" }, "dram__bytes_read.sum": { "value": 133.88288, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 72.706048, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 1.589733, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 168.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 232.448, "unit": "Kbyte/block" } } }, { "id": "3", "name": "void cudnn::nhwcToNchwKernel(cudnn::nhwc2nchw_params_t, const T1 *, T2 *)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 45.056, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 37.93825, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 70.172358, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 84.680591, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 89.034572, "unit": "%" }, "dram__bytes_read.sum": { "value": 86.524928, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 65.519104, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 3.374557, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 32.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 5.248, "unit": "Kbyte/block" } } }, { "id": "4", "name": "void at::elementwise_kernel<128, 2, void at::gpu_kernel_impl_nocast>(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 60.672, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 59.418102, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 50.323983, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 62.25211, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 83.980021, "unit": "%" }, "dram__bytes_read.sum": { "value": 86.52672, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 60.369152, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 2.421148, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 18.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 1.024, "unit": "Kbyte/block" } } } ], "sha256": { "guard_own_pid_history_v2.json": "3d59f5164389589a8676b1e0dfa4617aeeb85d8df92e0e456cec08cd86f40a01", "replay_comparison.json": "637816acb6f8cf62e4c2423466cc7c7fb56e0f2f3073f8eb1027edd04650821f", "metrics.csv": "ccaeb2e4346e742db251add3dca72c0d6af65604ae2b424566babba0c6ad70a2", "replay_comparison.output.pt": "3ff204e024ec28232b17e4bb628e82b1768d505dfe6974858ffd6c988fa7b5d9", "operator.ncu-rep": "bf493e347c99521efe559395be2cf4ca0aa234d49d5c7368949ab2ca1602fd9d", "run.log": "aa2f50cad9e9b68bddba580e97dfec24696460abbb15ba603442181b8bb5e51f", "identity.json": "63fd7409fc5d6c0fa4d125fde0ae821ef0674f772c622b43df84cacf44686590", "replay_operator.py": "cbea82d5e280ab2be0b1ce8dfebce6c1e80888c1d9fe727638e97a2652578d9b", "manifest.json": "1ea0ce74a904622d037525af9aa52f747749703fa1332fece7f7ea6fe373ec4f" } }, { "stage": "dit", "status": "passed", "scheduler_note": [], "output": "/data/zhoutaichang/feature/Lingbot_world_realtime/runs/analysis_report_20260908/raw/ncu_missing704_3d9ee58b5b1e2e00_v1", "elapsed_seconds": 11.874884605407715, "allocated": "0", "input_sha256": "31fe6110c4fdd22ab61ff0c539192126b4bc8da9a7355f9916c23de05f620d74", "correctness": { "op": "aten.linear.default", "stage": "dit", "shape": [ 10560, 5120 ], "exact": true, "max_abs": 0.0, "rmse": 0.0, "finite": true, "math_policy": { "matmul_precision": "highest", "matmul_allow_tf32": false, "cudnn_allow_tf32": true, "cudnn_benchmark": false, "cudnn_deterministic": false }, "scope": "Original ATen op on saved real tensors; isolated diagnostic" }, "kernel_count": 1, "kernels": [ { "id": "0", "name": "nvjet_sm90_tst_128x192_64x5_2x1_v_bz_coopB_bias_TNN", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 53.472, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 40.236191, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 33.664381, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 67.208801, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 14.739979, "unit": "%" }, "dram__bytes_read.sum": { "value": 4.755456, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 81.8496, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 1.619634, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 168.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 222.444, "unit": "Kbyte/block" } } } ], "sha256": { "guard_own_pid_history_v2.json": "bbcdcd1fc14391ab2da565ea05d839ba0586ef66b419acd50f7bf4434a7c638b", "replay_comparison.json": "30eb9a7af580240568562425986a1bfc0172cee0cc9907a3eb75f8d1d909809b", "metrics.csv": "637613bd6b7dca12517fab6c0d238c2ec8cf93de069ea6470ca3db5871a719e2", "replay_comparison.output.pt": "9986f48ce47b7f25394121223f1c9574782a638fc26aa16fbe0bdac352361d14", "operator.ncu-rep": "fec77761f7a379591c5fcc2eb2b593506b0a1df6baa8d0d4eb0e5f9ed39d6780", "run.log": "bccca658e404c895f1ac9cdfc3232d187355e71213aa4eb1da61778b990732ab", "identity.json": "03fd7080c4569e0911cdced998a34830d23eb577fc2b25db45ccbbf9b9d65f74", "replay_operator.py": "cbea82d5e280ab2be0b1ce8dfebce6c1e80888c1d9fe727638e97a2652578d9b", "manifest.json": "d9d41b11453b25f399e62abc9566e4a638ef889dbc95123715dfe3bb9d5f2434" } }, { "stage": "vae", "status": "passed", "scheduler_note": [], "output": "/data/zhoutaichang/feature/Lingbot_world_realtime/runs/analysis_report_20260908/raw/ncu_missing704_1ae7dbe8166a0b6b_v1", "elapsed_seconds": 19.34512758255005, "allocated": "0", "input_sha256": "85c0ba2d39aa4aa6ffbabaa237b9fa5d7721eefd855682c8cf482c661a5943d9", "correctness": { "op": "aten.conv3d.default", "stage": "vae", "shape": [ 1, 96, 1, 704, 320 ], "exact": true, "max_abs": 0.0, "rmse": 0.0, "finite": true, "math_policy": { "matmul_precision": "highest", "matmul_allow_tf32": false, "cudnn_allow_tf32": true, "cudnn_benchmark": false, "cudnn_deterministic": false }, "scope": "Original ATen op on saved real tensors; isolated diagnostic" }, "kernel_count": 5, "kernels": [ { "id": "0", "name": "void cudnn::nchwToNhwcKernel(cudnn::nchw2nhwc_params_t, const T1 *, T2 *)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 158.496, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 37.586965, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 66.627572, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 81.834161, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 93.257434, "unit": "%" }, "dram__bytes_read.sum": { "value": 261.903104, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 246.129408, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 3.205333, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 32.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 5.248, "unit": "Kbyte/block" } } }, { "id": "1", "name": "void cudnn::nchwToNhwcKernel(cudnn::nchw2nhwc_params_t, const T1 *, T2 *)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 3.488, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 12.603875, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 6.065995, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 15.459878, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 24.031763, "unit": "%" }, "dram__bytes_read.sum": { "value": 1.004288, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 0.0, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 0.287927, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 32.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 5.248, "unit": "Kbyte/block" } } }, { "id": "2", "name": "sm90_xmma_fprop_implicit_gemm_f32f32_tf32f32_f32_nhwckrsc_nhwc_tilesize256x32x32_warpgroupsize1x1x1_g1_execute_segment_k_off_kernel__5x_cudnn", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 738.976, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 31.264587, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 9.724301, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 59.885082, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 18.503275, "unit": "%" }, "dram__bytes_read.sum": { "value": 262.976, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 82.790656, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 0.4679, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 168.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 232.448, "unit": "Kbyte/block" } } }, { "id": "3", "name": "void cudnn::nhwcToNchwKernel(cudnn::nhwc2nchw_params_t, const T1 *, T2 *)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 44.384, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 41.746253, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 70.914106, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 85.734401, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 91.685438, "unit": "%" }, "dram__bytes_read.sum": { "value": 86.537728, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 64.758528, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 3.408802, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 32.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 5.248, "unit": "Kbyte/block" } } }, { "id": "4", "name": "void at::elementwise_kernel<128, 2, void at::gpu_kernel_impl_nocast>(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 60.672, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 59.459378, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 51.000054, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 62.565379, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 84.007886, "unit": "%" }, "dram__bytes_read.sum": { "value": 86.523904, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 62.30656, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 2.453034, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 18.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 1.024, "unit": "Kbyte/block" } } } ], "sha256": { "guard_own_pid_history_v2.json": "7489389749cdc79c036fce97791e52567d6c583a4b829ab5a2000eb9dd30d792", "replay_comparison.json": "92961129b1c65510a083bf09abbd0bf30e8db1236eeed603ac09abdd46c9154c", "metrics.csv": "aa0edd1e76279cf72710425be23ebfba1e4b4f7ffc6a94b0f49d12d954d51f8f", "replay_comparison.output.pt": "80fee6231d3d6405e965533268ec967f19bc05d938c1e0b80fb9c2429468f31c", "operator.ncu-rep": "51a9504490b7b302354d49cd81033b8883db791b6bb92e50a75c7554c16bdb8b", "run.log": "afefc581c64ac66695dcf52e6b7fed647ec67aca74ddc07b63dad92a663ff906", "identity.json": "43d1b3043b1b2fce31f99a73e7ec2615742f9117550b974503ff98e3d36bb12e", "replay_operator.py": "cbea82d5e280ab2be0b1ce8dfebce6c1e80888c1d9fe727638e97a2652578d9b", "manifest.json": "2f1575bd79d407772dc3b760ebe27bef6c7766fad363562acd9fde1e1593dae9" } }, { "stage": "vae", "status": "passed", "scheduler_note": [], "output": "/data/zhoutaichang/feature/Lingbot_world_realtime/runs/analysis_report_20260908/raw/ncu_missing704_996e506f74aef428_v1", "elapsed_seconds": 18.920615911483765, "allocated": "0", "input_sha256": "a9625d3e84c044a16224205940b2918538dcbebc7b9e36a51af82b718a774552", "correctness": { "op": "aten.conv3d.default", "stage": "vae", "shape": [ 1, 192, 1, 352, 160 ], "exact": true, "max_abs": 0.0, "rmse": 0.0, "finite": true, "math_policy": { "matmul_precision": "highest", "matmul_allow_tf32": false, "cudnn_allow_tf32": true, "cudnn_benchmark": false, "cudnn_deterministic": false }, "scope": "Original ATen op on saved real tensors; isolated diagnostic" }, "kernel_count": 5, "kernels": [ { "id": "0", "name": "void cudnn::nchwToNhwcKernel(cudnn::nchw2nhwc_params_t, const T1 *, T2 *)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 77.312, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 38.599213, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 66.201483, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 83.499739, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 94.606701, "unit": "%" }, "dram__bytes_read.sum": { "value": 132.146944, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 113.996032, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 3.183762, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 32.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 5.248, "unit": "Kbyte/block" } } }, { "id": "1", "name": "void cudnn::nchwToNhwcKernel(cudnn::nchw2nhwc_params_t, const T1 *, T2 *)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 5.152, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 31.641973, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 16.259436, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 38.61829, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 73.963813, "unit": "%" }, "dram__bytes_read.sum": { "value": 3.990272, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 0.0, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 0.774509, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 32.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 5.248, "unit": "Kbyte/block" } } }, { "id": "2", "name": "sm90_xmma_fprop_implicit_gemm_f32f32_tf32f32_f32_nhwckrsc_nhwc_tilesize128x128x32_warpgroupsize1x1x1_g1_execute_segment_k_off_kernel__5x_cudnn", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 427.04, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 73.067093, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 8.585092, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 64.293498, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 16.744155, "unit": "%" }, "dram__bytes_read.sum": { "value": 138.944, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 37.46048, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 0.413087, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 168.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 232.448, "unit": "Kbyte/block" } } }, { "id": "3", "name": "void cudnn::nhwcToNchwKernel(cudnn::nhwc2nchw_params_t, const T1 *, T2 *)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 22.464, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 39.39385, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 62.209485, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 81.155405, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 89.162527, "unit": "%" }, "dram__bytes_read.sum": { "value": 43.270656, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 23.85024, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 2.987932, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 32.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 5.248, "unit": "Kbyte/block" } } }, { "id": "4", "name": "void at::elementwise_kernel<128, 2, void at::gpu_kernel_impl_nocast>(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 30.528, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 59.221936, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 41.731601, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 60.937512, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 80.401756, "unit": "%" }, "dram__bytes_read.sum": { "value": 43.270912, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 17.987072, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 2.006616, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 18.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 1.024, "unit": "Kbyte/block" } } } ], "sha256": { "guard_own_pid_history_v2.json": "36fbfbe388e7640d63814cd7244f25fc1972dc1fd1da61bba99dbc13e2ab5a24", "replay_comparison.json": "1dd5e7e3599879d488e65f8ab4692234a7ab316a99f745243b45b55078ef4db7", "metrics.csv": "05a6544ddacff75d7aee5bdecf8ab706b391686960b1fb0fa93bc4dcf50575aa", "replay_comparison.output.pt": "bc640aafec9bf1100d48af2782767066120080f2ef816f2b88866d743f738d14", "operator.ncu-rep": "7d33d6014048b2293e53ca69ed282bf24c960c0f3fdb6713569a2f80239bda98", "run.log": "5c18cb39bc2b949081b6dff03a85bef3750373c3ca574341e3c688b3b8dc451a", "identity.json": "62dedf239f37b3bcf1abf44833855d566728eba2aec4a19a1f6a8c5afdc55a62", "replay_operator.py": "cbea82d5e280ab2be0b1ce8dfebce6c1e80888c1d9fe727638e97a2652578d9b", "manifest.json": "3773e8b03317dc862aad00e7d2598e0e24a93b0fd9015140a975a08b8adc3b3e" } }, { "stage": "vae", "status": "passed", "scheduler_note": [], "output": "/data/zhoutaichang/feature/Lingbot_world_realtime/runs/analysis_report_20260908/raw/ncu_missing704_e70c8df9d1c257cc_v1", "elapsed_seconds": 19.817643880844116, "allocated": "0", "input_sha256": "9b2cc0033aa13a80bc2dd24c544a6b1097e94f10f8c1f3748cbca111210d5fa1", "correctness": { "op": "aten.conv3d.default", "stage": "vae", "shape": [ 1, 384, 1, 176, 80 ], "exact": true, "max_abs": 0.0, "rmse": 0.0, "finite": true, "math_policy": { "matmul_precision": "highest", "matmul_allow_tf32": false, "cudnn_allow_tf32": true, "cudnn_benchmark": false, "cudnn_deterministic": false }, "scope": "Original ATen op on saved real tensors; isolated diagnostic" }, "kernel_count": 6, "kernels": [ { "id": "0", "name": "void cudnn::nchwToNhwcKernel(cudnn::nchw2nhwc_params_t, const T1 *, T2 *)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 37.088, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 40.939927, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 64.810161, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 85.309148, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 90.636878, "unit": "%" }, "dram__bytes_read.sum": { "value": 67.268864, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 48.30208, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 3.116128, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 32.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 5.248, "unit": "Kbyte/block" } } }, { "id": "1", "name": "void cudnn::nchwToNhwcKernel(cudnn::nchw2nhwc_params_t, const T1 *, T2 *)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 10.528, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 60.389793, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 33.309839, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 66.459544, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 86.205833, "unit": "%" }, "dram__bytes_read.sum": { "value": 15.934464, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 0.882432, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 1.59735, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 32.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 5.248, "unit": "Kbyte/block" } } }, { "id": "2", "name": "void cask_plugin__5x_cudnn::init_device_workspace_kernel>(T1, bool)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 4.8, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 0.043349, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 0.020058, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 0.337283, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 1.630035, "unit": "%" }, "dram__bytes_read.sum": { "value": 0.004608, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 0.0, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 0.00096, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 16.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 1.024, "unit": "Kbyte/block" } } }, { "id": "3", "name": "sm90_xmma_fprop_implicit_gemm_f32f32_tf32f32_f32_nhwckrsc_nhwc_tilesize128x128x32_warpgroupsize1x1x1_g1_execute_segment_k_on_kernel__5x_cudnn", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 504.928, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 60.616992, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 8.561562, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 55.778751, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 16.314718, "unit": "%" }, "dram__bytes_read.sum": { "value": 179.351552, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 28.662016, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 0.411967, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 168.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 232.448, "unit": "Kbyte/block" } } }, { "id": "4", "name": "void cudnn::nhwcToNchwKernel(cudnn::nhwc2nchw_params_t, const T1 *, T2 *)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 12.064, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 36.357604, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 43.6705, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 74.050607, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 86.737283, "unit": "%" }, "dram__bytes_read.sum": { "value": 21.643776, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 3.651072, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 2.096721, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 32.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 5.248, "unit": "Kbyte/block" } } }, { "id": "5", "name": "void at::elementwise_kernel<128, 2, void at::gpu_kernel_impl_nocast>(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 16.0, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 56.852889, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 28.834673, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 60.084634, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 77.665579, "unit": "%" }, "dram__bytes_read.sum": { "value": 21.643008, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 0.514816, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 1.384864, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 18.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 1.024, "unit": "Kbyte/block" } } } ], "sha256": { "guard_own_pid_history_v2.json": "b095a1b107e2a5287d3a3ff578c18c922effe86800514faa76ccf24bfe40e91e", "replay_comparison.json": "34f9cea92af76b4a871c2cf5043318de8bd6eecf99da49cbfe66c4035fbdffe8", "metrics.csv": "322ddd52edb945be7dee84324a3d329dc0d3ca65f88e55dcd3b9ff5a97134901", "replay_comparison.output.pt": "ddeaed51a924da4aad2f8168f1f03d49dee99b42a5a3a00e41ab22b89d920c55", "operator.ncu-rep": "934a92b96d1dfc08089f81dfbdfcd6a2713a4e973716396d458c89efd078b016", "run.log": "0340f08273838842dd0a5eb0fb7465d950ece8a82741fd5c5bb6d47c51376dbf", "identity.json": "9b6fef5ca0e79215b84cb2cdca9222ed537422f249a3db51c70bf92c0ba7f29f", "replay_operator.py": "cbea82d5e280ab2be0b1ce8dfebce6c1e80888c1d9fe727638e97a2652578d9b", "manifest.json": "2e1775893c6518995f43faa6d7c9e28811a622acff23e58a60994839f6c55acb" } }, { "stage": "vae", "status": "passed", "scheduler_note": [], "output": "/data/zhoutaichang/feature/Lingbot_world_realtime/runs/analysis_report_20260908/raw/ncu_missing704_0e33e5013ffbea1c_v1", "elapsed_seconds": 19.263991117477417, "allocated": "0", "input_sha256": "c09536053b980c5e962fadc017011cff0cd6c94409f5c2270984dfec83c493a3", "correctness": { "op": "aten.conv3d.default", "stage": "vae", "shape": [ 1, 3, 4, 704, 320 ], "exact": true, "max_abs": 0.0, "rmse": 0.0, "finite": true, "math_policy": { "matmul_precision": "highest", "matmul_allow_tf32": false, "cudnn_allow_tf32": true, "cudnn_benchmark": false, "cudnn_deterministic": false }, "scope": "Original ATen op on saved real tensors; isolated diagnostic" }, "kernel_count": 5, "kernels": [ { "id": "0", "name": "void cudnn::nchwToNhwcKernel(cudnn::nchw2nhwc_params_t, const T1 *, T2 *)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 0.31568, "unit": "ms" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 37.622393, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 67.741384, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 81.452424, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 93.745447, "unit": "%" }, "dram__bytes_read.sum": { "value": 0.523789, "unit": "Gbyte" }, "dram__bytes_write.sum": { "value": 505.148672, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 3.259431, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 32.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 5.248, "unit": "Kbyte/block" } } }, { "id": "1", "name": "void cudnn::nchwToNhwcKernel(cudnn::nchw2nhwc_params_t, const T1 *, T2 *)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 0.003296, "unit": "ms" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 0.356453, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 0.257799, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 1.149234, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 11.983411, "unit": "%" }, "dram__bytes_read.sum": { "value": 4e-05, "unit": "Gbyte" }, "dram__bytes_write.sum": { "value": 0.0, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 0.012272, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 32.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 5.248, "unit": "Kbyte/block" } } }, { "id": "2", "name": "sm80_xmma_fprop_implicit_gemm_indexed_wo_smem_tf32f32_tf32f32_f32_nhwckrsc_nhwc_tilesize128x16x64_stage1_warpsize4x1x1_g1_tensor16x8x8_alignc4_execute_kernel__5x_cudnn", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 1.148096, "unit": "ms" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 51.621077, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 22.10333, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 75.24804, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 30.370714, "unit": "%" }, "dram__bytes_read.sum": { "value": 1.207012, "unit": "Gbyte" }, "dram__bytes_write.sum": { "value": 14.110976, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 1.063607, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 94.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 5.12, "unit": "Kbyte/block" } } }, { "id": "3", "name": "void cudnn::nhwcToNchwKernel(cudnn::nhwc2nchw_params_t, const T1 *, T2 *)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 0.038368, "unit": "ms" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 81.301954, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 5.885784, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 14.580013, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 87.339402, "unit": "%" }, "dram__bytes_read.sum": { "value": 0.010829, "unit": "Gbyte" }, "dram__bytes_write.sum": { "value": 0.02048, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 0.282769, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 32.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 5.248, "unit": "Kbyte/block" } } }, { "id": "4", "name": "void at::elementwise_kernel<128, 2, void at::gpu_kernel_impl_nocast>(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 0.0096, "unit": "ms" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 47.685405, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 23.523262, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 51.542645, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 74.09594, "unit": "%" }, "dram__bytes_read.sum": { "value": 0.010828, "unit": "Gbyte" }, "dram__bytes_write.sum": { "value": 0.000768, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 1.128, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 18.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 1.024, "unit": "Kbyte/block" } } } ], "sha256": { "guard_own_pid_history_v2.json": "64b95c657525403929615c93149341fc3d0d22797bf8499ecb3a09b1151a6e09", "replay_comparison.json": "30ef8e299e0820a4cecc499e006a73fa7c00692bfecca889ab2c2df6bc90c83e", "metrics.csv": "74bab34b00580ebd194e817a92d4213eee0d5e825ff1db2c9c4feb436049855b", "replay_comparison.output.pt": "cf142d5edb817b240d3194e3af0bb19ed66676b6f5355695baa5e0b409fddb83", "operator.ncu-rep": "8551efb0deb2110d48dc20501f0d4e4e0715965ce094ee0e4be1319b266c0541", "run.log": "2fd634d4616f16605f11710d194e3d0b1e774e446859ce218e8de72e89671308", "identity.json": "3f0be979c4f9a4d2e8dbbb6209493039d6239d9e9ff7ba53c89e0ecb78369237", "replay_operator.py": "cbea82d5e280ab2be0b1ce8dfebce6c1e80888c1d9fe727638e97a2652578d9b", "manifest.json": "4b9c801a3ee53c7b69d72e25a96cdd7328f58ad081ff67a29742dae7868324e9" } }, { "stage": "dit", "status": "passed", "scheduler_note": [], "output": "/data/zhoutaichang/feature/Lingbot_world_realtime/runs/analysis_report_20260908/raw/ncu_missing704_5a8bc19ef82213ac_v1", "elapsed_seconds": 11.714650869369507, "allocated": "0", "input_sha256": "26ebef86e4633eac87d2d256ccdab318b493bdf4c53c9c855e742e4fe4f4f537", "correctness": { "op": "aten.linear.default", "stage": "dit", "shape": [ 1, 10560, 64 ], "exact": true, "max_abs": 0.0, "rmse": 0.0, "finite": true, "math_policy": { "matmul_precision": "highest", "matmul_allow_tf32": false, "cudnn_allow_tf32": true, "cudnn_benchmark": false, "cudnn_deterministic": false }, "scope": "Original ATen op on saved real tensors; isolated diagnostic" }, "kernel_count": 1, "kernels": [ { "id": "0", "name": "nvjet_sm90_tst_64x80_64x11_1x2_h_bz_bias_TNT", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 31.136, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 22.954364, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 74.503733, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 77.028846, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 13.512909, "unit": "%" }, "dram__bytes_read.sum": { "value": 108.811264, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 2.653696, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 3.579938, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 168.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 220.508, "unit": "Kbyte/block" } } } ], "sha256": { "guard_own_pid_history_v2.json": "6e8879ef5e7fbe1a13baa4d3f2363d26b12a623a1d61efb3cab6faf9565632c9", "replay_comparison.json": "4183bc70de84a790e96fd1637e638db0132bfcb477575feac5a68fbb97136628", "metrics.csv": "fc63f48c0abfa5decbde598374626ea692157d29e5189496d4224121807fd167", "replay_comparison.output.pt": "972affb397a1fde1377a8c120f7f45c07a74c11c4e8ec981e86b7ce20bc6cd50", "operator.ncu-rep": "7424557eeb04c8b9653061564971990ef2e5da20791a9572a49ad982236e635d", "run.log": "e5b770b1ff790c7dc28fd2846e0db68c6343969f73caa0258c1ae8a77d6aa418", "identity.json": "c3505d6cdd196169c707aff101ec68b19419c9c747cb046afab03828ad3764ba", "replay_operator.py": "cbea82d5e280ab2be0b1ce8dfebce6c1e80888c1d9fe727638e97a2652578d9b", "manifest.json": "d1b0cf745a033dcc723b4c53ec0a1884233d496fa48968e478d3daa290c8551b" } }, { "stage": "vae", "status": "passed", "scheduler_note": [], "output": "/data/zhoutaichang/feature/Lingbot_world_realtime/runs/analysis_report_20260908/raw/ncu_missing704_fd5ad4e08e7ff9d8_v1", "elapsed_seconds": 20.18788981437683, "allocated": "0", "input_sha256": "9dd2254d46cabc12367da79ca54c5220c508380e8b7fd1b985050405f5f81ee3", "correctness": { "op": "aten.conv3d.default", "stage": "vae", "shape": [ 1, 768, 1, 88, 40 ], "exact": true, "max_abs": 0.0, "rmse": 0.0, "finite": true, "math_policy": { "matmul_precision": "highest", "matmul_allow_tf32": false, "cudnn_allow_tf32": true, "cudnn_benchmark": false, "cudnn_deterministic": false }, "scope": "Original ATen op on saved real tensors; isolated diagnostic" }, "kernel_count": 6, "kernels": [ { "id": "0", "name": "void cudnn::nchwToNhwcKernel(cudnn::nchw2nhwc_params_t, const T1 *, T2 *)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 9.856, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 37.814765, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 36.660836, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 68.880476, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 82.425709, "unit": "%" }, "dram__bytes_read.sum": { "value": 16.229376, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 1.101568, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 1.758416, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 32.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 5.248, "unit": "Kbyte/block" } } }, { "id": "1", "name": "void cudnn::nchwToNhwcKernel(cudnn::nchw2nhwc_params_t, const T1 *, T2 *)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 15.168, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 82.72878, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 4.874997, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 13.05391, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 85.259736, "unit": "%" }, "dram__bytes_read.sum": { "value": 3.547904, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 0.0, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 0.233907, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 32.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 5.248, "unit": "Kbyte/block" } } }, { "id": "2", "name": "void cask_plugin__5x_cudnn::init_device_workspace_kernel>(T1, bool)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 4.384, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 0.039343, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 0.0209, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 0.26054, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 1.678345, "unit": "%" }, "dram__bytes_read.sum": { "value": 0.004352, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 0.0, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 0.000993, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 16.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 1.024, "unit": "Kbyte/block" } } }, { "id": "3", "name": "sm90_xmma_fprop_implicit_gemm_f32f32_tf32f32_f32_nhwckrsc_nhwc_tilesize128x128x32_warpgroupsize1x1x1_g1_execute_segment_k_on_kernel__5x_cudnn", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 27.584, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 46.849007, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 18.040615, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 52.334057, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 15.439062, "unit": "%" }, "dram__bytes_read.sum": { "value": 20.38144, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 3.5456, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 0.867425, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 168.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 232.448, "unit": "Kbyte/block" } } }, { "id": "4", "name": "void cudnn::nhwcToNchwKernel(cudnn::nhwc2nchw_params_t, const T1 *, T2 *)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 8.128, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 27.067728, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 27.769106, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 57.67761, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 82.181854, "unit": "%" }, "dram__bytes_read.sum": { "value": 10.8288, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 0.009984, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 1.333512, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 32.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 5.248, "unit": "Kbyte/block" } } }, { "id": "5", "name": "void at::elementwise_kernel<128, 2, void at::gpu_kernel_impl_nocast>(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 9.632, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 47.591335, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 23.496223, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 52.18756, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 74.70752, "unit": "%" }, "dram__bytes_read.sum": { "value": 10.830848, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 0.004352, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 1.124917, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 18.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 1.024, "unit": "Kbyte/block" } } } ], "sha256": { "guard_own_pid_history_v2.json": "65607266f31aabc93bc52e9c07b83276b012f6028f34b0942d24a2cb66f68839", "replay_comparison.json": "25cf421727af26051d841da0fe2b3cdba7c897da4f9d61d4eb747f76312924d6", "metrics.csv": "2c95712f9ee4b624884277c2fc5b24ed677fc2e372a01bee59bd317528f9aaf0", "replay_comparison.output.pt": "654aef21de5dc372db7c352cd407d8dca33a3a84dc2ba215382ed03926406277", "operator.ncu-rep": "410a56bf1305ab2126d04cda7172063626821c05c06068526eb8a6a1e7653010", "run.log": "18613095233643e04d65582b8b88201de74ddd18feaf25a84a6935330bee97b3", "identity.json": "76b84fab5b38082216d52035bd5e5322fcf99a943efacd72330a2f93bebe8969", "replay_operator.py": "cbea82d5e280ab2be0b1ce8dfebce6c1e80888c1d9fe727638e97a2652578d9b", "manifest.json": "62950a49284199a77ab2b8acae894b07fb007ddf1c060c21d2185aa43956538f" } }, { "stage": "vae", "status": "passed", "scheduler_note": [], "output": "/data/zhoutaichang/feature/Lingbot_world_realtime/runs/analysis_report_20260908/raw/ncu_missing704_9a2a65226b52659e_v1", "elapsed_seconds": 14.520900011062622, "allocated": "0", "input_sha256": "bccd4b59806daff52b45f9c87ba32579441de0ed0adb3dd99356b7ebbcc1f4c7", "correctness": { "op": "aten.conv3d.default", "stage": "vae", "shape": [ 1, 384, 2, 176, 80 ], "exact": true, "max_abs": 0.0, "rmse": 0.0, "finite": true, "math_policy": { "matmul_precision": "highest", "matmul_allow_tf32": false, "cudnn_allow_tf32": true, "cudnn_benchmark": false, "cudnn_deterministic": false }, "scope": "Original ATen op on saved real tensors; isolated diagnostic" }, "kernel_count": 3, "kernels": [ { "id": "0", "name": "void at::elementwise_kernel<128, 2, void at::gpu_kernel_impl_nocast(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 16.736, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 55.700949, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 31.109782, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 58.851907, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 80.743628, "unit": "%" }, "dram__bytes_read.sum": { "value": 21.641984, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 3.384576, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 1.495373, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 18.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 1.024, "unit": "Kbyte/block" } } }, { "id": "1", "name": "void cutlass::Kernel2(Params)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 33.568, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 38.541084, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 25.477149, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 56.133692, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 11.155685, "unit": "%" }, "dram__bytes_read.sum": { "value": 22.0352, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 19.05152, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 1.223985, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 214.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 82.944, "unit": "Kbyte/block" } } }, { "id": "2", "name": "void at::elementwise_kernel<128, 2, void at::gpu_kernel_impl_nocast>(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 30.656, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 58.936192, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 42.051514, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 60.877906, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 81.170483, "unit": "%" }, "dram__bytes_read.sum": { "value": 43.271168, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 18.736384, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 2.022689, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 18.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 1.024, "unit": "Kbyte/block" } } } ], "sha256": { "guard_own_pid_history_v2.json": "6c187c6c047b532c31541f61e950e2eeaa5257e16eab94ee2e4c9bced9a576cf", "replay_comparison.json": "60930f2572471940d75b458e424510051d51cf7489567d81905a53cc60aeb4f5", "metrics.csv": "d1057d2c0073e4116247d85f5b4bb8fffccac12d8e5f9ac20970696121831070", "replay_comparison.output.pt": "e198d784eac3cc59b72d907bd517d206dbaf6626284f5259233959a164e64b5c", "operator.ncu-rep": "86441e4cfbf48668fc965537243da2a82bc8a6894a5028b59b1408068f1a1208", "run.log": "c5feb4e9e7becf7963ce870601f08adc3482c57ba9403dd9bd04f7c8befe7710", "identity.json": "5337a027627655812ff32f768232bb4901d3a6cc61170383e4d9328004189e19", "replay_operator.py": "cbea82d5e280ab2be0b1ce8dfebce6c1e80888c1d9fe727638e97a2652578d9b", "manifest.json": "5ec1098093fce1fa1ea0195786dfbc6134479efc647518a2025700068b3f8aab" } }, { "stage": "vae", "status": "passed", "scheduler_note": [], "output": "/data/zhoutaichang/feature/Lingbot_world_realtime/runs/analysis_report_20260908/raw/ncu_missing704_740d67537eaca4e1_v1", "elapsed_seconds": 20.815129041671753, "allocated": "0", "input_sha256": "d56deba2cf2a7667416ca96682c1bcff892969b687097e1b3ea3921ac1a1d6ac", "correctness": { "op": "aten.conv3d.default", "stage": "vae", "shape": [ 1, 384, 1, 176, 80 ], "exact": true, "max_abs": 0.0, "rmse": 0.0, "finite": true, "math_policy": { "matmul_precision": "highest", "matmul_allow_tf32": false, "cudnn_allow_tf32": true, "cudnn_benchmark": false, "cudnn_deterministic": false }, "scope": "Original ATen op on saved real tensors; isolated diagnostic" }, "kernel_count": 6, "kernels": [ { "id": "0", "name": "void cudnn::nchwToNhwcKernel(cudnn::nchw2nhwc_params_t, const T1 *, T2 *)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 18.976, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 40.26307, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 54.591576, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 79.430109, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 87.141999, "unit": "%" }, "dram__bytes_read.sum": { "value": 33.639168, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 16.110336, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 2.621707, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 32.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 5.248, "unit": "Kbyte/block" } } }, { "id": "1", "name": "void cudnn::nchwToNhwcKernel(cudnn::nchw2nhwc_params_t, const T1 *, T2 *)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 6.816, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 47.003145, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 24.437219, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 53.921645, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 82.359237, "unit": "%" }, "dram__bytes_read.sum": { "value": 7.971584, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 0.000512, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 1.169615, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 32.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 5.248, "unit": "Kbyte/block" } } }, { "id": "2", "name": "void cask_plugin__5x_cudnn::init_device_workspace_kernel>(T1, bool)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 4.96, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 0.042188, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 0.020587, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 0.226394, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 1.57003, "unit": "%" }, "dram__bytes_read.sum": { "value": 0.004864, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 0.0, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 0.000981, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 16.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 1.024, "unit": "Kbyte/block" } } }, { "id": "3", "name": "sm90_xmma_fprop_implicit_gemm_f32f32_tf32f32_f32_nhwckrsc_nhwc_tilesize128x128x32_warpgroupsize1x1x1_g1_execute_segment_k_on_kernel__5x_cudnn", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 247.264, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 62.617369, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 7.530364, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 56.794153, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 16.474858, "unit": "%" }, "dram__bytes_read.sum": { "value": 66.658816, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 22.921472, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 0.362286, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 168.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 232.448, "unit": "Kbyte/block" } } }, { "id": "4", "name": "void cudnn::nhwcToNchwKernel(cudnn::nhwc2nchw_params_t, const T1 *, T2 *)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 12.224, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 36.079744, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 44.352836, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 72.918726, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 87.156686, "unit": "%" }, "dram__bytes_read.sum": { "value": 21.643776, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 4.352256, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 2.126639, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 32.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 5.248, "unit": "Kbyte/block" } } }, { "id": "5", "name": "void at::elementwise_kernel<128, 2, void at::gpu_kernel_impl_nocast>(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 16.16, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 56.505241, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 28.769562, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 59.815566, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 76.979545, "unit": "%" }, "dram__bytes_read.sum": { "value": 21.642752, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 0.695808, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 1.382337, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 18.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 1.024, "unit": "Kbyte/block" } } } ], "sha256": { "guard_own_pid_history_v2.json": "caab7c5e23c0494c22e7ec2ac73bf2324dbfba63c3c94850e879263904fad210", "replay_comparison.json": "34f9cea92af76b4a871c2cf5043318de8bd6eecf99da49cbfe66c4035fbdffe8", "metrics.csv": "085a6ac91102daeaa91c746478b6dfdedd9a64785f2f13970191fb734e2884a6", "replay_comparison.output.pt": "0a9396ac12a9b164ee57c119475e2d91d6555d8b6a21730f717fb649e0d94007", "operator.ncu-rep": "8e238c838100dc4b4ceed17fcb4ad32ec12564306e631932e7ad894292695ecc", "run.log": "da02985143de3565f642c98a54c8ef93cdd1cb787baef493dbe18306077a21ff", "identity.json": "1459268d16fb401c2dd6914ef29d9801726befbed69b4ca51c5b489ca60354bb", "replay_operator.py": "cbea82d5e280ab2be0b1ce8dfebce6c1e80888c1d9fe727638e97a2652578d9b", "manifest.json": "8dcc3635b1460fe0aa810ab735392b38c6542ff5ff543e3c7e12d4baa42704b4" } }, { "stage": "vae", "status": "passed", "scheduler_note": [], "output": "/data/zhoutaichang/feature/Lingbot_world_realtime/runs/analysis_report_20260908/raw/ncu_missing704_9e24c4a75ab83dea_v1", "elapsed_seconds": 19.67279624938965, "allocated": "0", "input_sha256": "f8fb1466dc2a1a6e997b5eff656e0d892480f377234df948eb6b06009d0b88b2", "correctness": { "op": "aten.conv3d.default", "stage": "vae", "shape": [ 1, 384, 1, 88, 40 ], "exact": true, "max_abs": 0.0, "rmse": 0.0, "finite": true, "math_policy": { "matmul_precision": "highest", "matmul_allow_tf32": false, "cudnn_allow_tf32": true, "cudnn_benchmark": false, "cudnn_deterministic": false }, "scope": "Original ATen op on saved real tensors; isolated diagnostic" }, "kernel_count": 4, "kernels": [ { "id": "0", "name": "void cudnn::nchwToNhwcKernel(cudnn::nchw2nhwc_params_t, const T1 *, T2 *)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 3.328, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 11.366096, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 4.60672, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 12.750725, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 31.156783, "unit": "%" }, "dram__bytes_read.sum": { "value": 0.734976, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 0.0, "unit": "byte" }, "dram__bytes.sum.per_second": { "value": 220.846154, "unit": "Gbyte/s" }, "launch__registers_per_thread": { "value": 32.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 5.248, "unit": "Kbyte/block" } } }, { "id": "1", "name": "void cudnn::nchwToNhwcKernel(cudnn::nchw2nhwc_params_t, const T1 *, T2 *)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 3.2, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 17.64436, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 4.405579, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 11.130917, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 33.868282, "unit": "%" }, "dram__bytes_read.sum": { "value": 0.673024, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 0.0, "unit": "byte" }, "dram__bytes.sum.per_second": { "value": 210.32, "unit": "Gbyte/s" }, "launch__registers_per_thread": { "value": 32.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 5.248, "unit": "Kbyte/block" } } }, { "id": "2", "name": "sm80_xmma_fprop_implicit_gemm_tf32f32_tf32f32_f32_nhwckrsc_nchw_tilesize128x64x32_stage5_warpsize2x2x1_g1_tensor16x8x8_execute_kernel__5x_cudnn", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 29.408, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 24.811341, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 1.026599, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 19.082483, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 6.25421, "unit": "%" }, "dram__bytes_read.sum": { "value": 1.451776, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 0.0, "unit": "byte" }, "dram__bytes.sum.per_second": { "value": 49.366703, "unit": "Gbyte/s" }, "launch__registers_per_thread": { "value": 168.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 123.904, "unit": "Kbyte/block" } } }, { "id": "3", "name": "void at::elementwise_kernel<128, 2, void at::gpu_kernel_impl_nocast>(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 6.432, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 35.853635, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 17.604721, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 38.787897, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 72.671736, "unit": "%" }, "dram__bytes_read.sum": { "value": 5.422592, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 0.0, "unit": "byte" }, "dram__bytes.sum.per_second": { "value": 843.064677, "unit": "Gbyte/s" }, "launch__registers_per_thread": { "value": 18.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 1.024, "unit": "Kbyte/block" } } } ], "sha256": { "guard_own_pid_history_v2.json": "09367574feeec741617bf895b6c4252253485aeeea86bce83d2ae9230dc97093", "replay_comparison.json": "385b975e001198fe437d8490d91614dd8c52aec195eeeab5d94028edff5c5e7e", "metrics.csv": "d2acd52f6d045bdddb1869c7da2d5c76c226e8b79b9dc45ced407b606751b1f6", "replay_comparison.output.pt": "d19b580463b13f89e14fb782188cabc959942d3909aff288b96ec0df10fb155b", "operator.ncu-rep": "f0ff950053c11a1c3b1347925a26ac1c1330f1070d1dff6698db4f3a9ebdf0f7", "run.log": "a401d141a2fb32d874e5d903db4e08f1e3389732751c06a43d4f1586a114a566", "identity.json": "2dc16d120e911717271a54d5f25ae8d3abf403f792cc226b7e3eaedeead6832e", "replay_operator.py": "cbea82d5e280ab2be0b1ce8dfebce6c1e80888c1d9fe727638e97a2652578d9b", "manifest.json": "f48f00a163f0416888bac16349ef99b0ba147c7236b35eb0d00fca4d7dd81f2d" } }, { "stage": "vae", "status": "passed", "scheduler_note": [], "output": "/data/zhoutaichang/feature/Lingbot_world_realtime/runs/analysis_report_20260908/raw/ncu_missing704_aa00674f1a72afcd_v1", "elapsed_seconds": 22.631744146347046, "allocated": "0", "input_sha256": "a2673addbb1775eb79e779d7635c4fbcf9dd34b0d4953bdf81e740bec14aa1ef", "correctness": { "op": "aten.conv3d.default", "stage": "vae", "shape": [ 1, 3, 1, 704, 320 ], "exact": true, "max_abs": 0.0, "rmse": 0.0, "finite": true, "math_policy": { "matmul_precision": "highest", "matmul_allow_tf32": false, "cudnn_allow_tf32": true, "cudnn_benchmark": false, "cudnn_deterministic": false }, "scope": "Original ATen op on saved real tensors; isolated diagnostic" }, "kernel_count": 5, "kernels": [ { "id": "0", "name": "void cudnn::nchwToNhwcKernel(cudnn::nchw2nhwc_params_t, const T1 *, T2 *)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 158.016, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 37.570242, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 66.475095, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 82.215211, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 94.813912, "unit": "%" }, "dram__bytes_read.sum": { "value": 261.910016, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 243.464192, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 3.198247, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 32.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 5.248, "unit": "Kbyte/block" } } }, { "id": "1", "name": "void cudnn::nchwToNhwcKernel(cudnn::nchw2nhwc_params_t, const T1 *, T2 *)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 3.296, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 0.355851, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 0.257799, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 0.578387, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 14.966176, "unit": "%" }, "dram__bytes_read.sum": { "value": 0.040448, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 0.0, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 0.012272, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 32.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 5.248, "unit": "Kbyte/block" } } }, { "id": "2", "name": "sm80_xmma_fprop_implicit_gemm_indexed_wo_smem_tf32f32_tf32f32_f32_nhwckrsc_nhwc_tilesize128x16x64_stage1_warpsize4x1x1_g1_tensor16x8x8_alignc4_execute_kernel__5x_cudnn", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 324.32, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 45.730038, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 18.153743, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 60.857539, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 28.397117, "unit": "%" }, "dram__bytes_read.sum": { "value": 278.2208, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 5.054208, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 0.873443, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 94.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 5.12, "unit": "Kbyte/block" } } }, { "id": "3", "name": "void cudnn::nhwcToNchwKernel(cudnn::nhwc2nchw_params_t, const T1 *, T2 *)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 12.224, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 64.279271, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 4.637879, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 12.789653, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 84.5493, "unit": "%" }, "dram__bytes_read.sum": { "value": 2.71872, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 0.0, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 0.222408, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 32.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 5.248, "unit": "Kbyte/block" } } }, { "id": "4", "name": "void at::elementwise_kernel<128, 2, void at::gpu_kernel_impl_nocast>(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 4.704, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 24.949374, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 12.083042, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 26.574567, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 71.125525, "unit": "%" }, "dram__bytes_read.sum": { "value": 2.717696, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 0.0, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 0.577741, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 18.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 1.024, "unit": "Kbyte/block" } } } ], "sha256": { "guard_own_pid_history_v2.json": "b529dfef7c37ac311ea1affb1adf93a483fb77ddfcbac4b8f5a3480c8bee93c9", "replay_comparison.json": "13c81350e952689875b00a09fece7f95786a377a4d1811eead5fad7c71d8a3e3", "metrics.csv": "eae2467302dd677db282998fdded6638d33796da99c1db10c99e5d176d649245", "replay_comparison.output.pt": "c511ad5d7b46ec79b391fce6c8720949a4175cd280cab2891539fde0211da5bc", "operator.ncu-rep": "9b622b0d65cc76c31096c70c3b9600d9d71aa31e3b4b3ad32191e45070141a7b", "run.log": "3500c055229de2207795131416af9e0fa81c6806a13f42eb7573c3a73ca9250d", "identity.json": "9e3e16ce4563dcbb960cb0bcb1710d7d6b0c6c65b1537e8f3b3d8943c65d83c3", "replay_operator.py": "cbea82d5e280ab2be0b1ce8dfebce6c1e80888c1d9fe727638e97a2652578d9b", "manifest.json": "dd0a3822a377a2ca0610b0201ddce37af28b2cf110ccde11dc5ce18ba8744fe7" } }, { "stage": "vae", "status": "passed", "scheduler_note": [], "output": "/data/zhoutaichang/feature/Lingbot_world_realtime/runs/analysis_report_20260908/raw/ncu_missing704_2ea2cf3e2f86821c_v1", "elapsed_seconds": 38.670536518096924, "allocated": "0", "input_sha256": "85c1d12d0656fbd621625fe39cf33eef4823fe7d3338e672b380cbbe7e08c3bc", "correctness": { "op": "aten.conv3d.default", "stage": "vae", "shape": [ 1, 384, 1, 176, 80 ], "exact": true, "max_abs": 0.0, "rmse": 0.0, "finite": true, "math_policy": { "matmul_precision": "highest", "matmul_allow_tf32": false, "cudnn_allow_tf32": true, "cudnn_benchmark": false, "cudnn_deterministic": false }, "scope": "Original ATen op on saved real tensors; isolated diagnostic" }, "kernel_count": 2, "kernels": [ { "id": "0", "name": "void cutlass::Kernel2(Params)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 20.48, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 39.212265, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 14.119686, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 46.797248, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 16.420292, "unit": "%" }, "dram__bytes_read.sum": { "value": 11.135744, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 2.754304, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 0.678225, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 136.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 74.752, "unit": "Kbyte/block" } } }, { "id": "1", "name": "void at::elementwise_kernel<128, 2, void at::gpu_kernel_impl_nocast>(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 16.224, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 56.09709, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 29.31918, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 59.138082, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 77.342627, "unit": "%" }, "dram__bytes_read.sum": { "value": 21.643264, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 1.197312, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 1.407826, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 18.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 1.024, "unit": "Kbyte/block" } } } ], "sha256": { "guard_own_pid_history_v2.json": "9cb4eac7ce0f37fa3bdbc64c5dd04e7821f2402acf93f52e946227dab579c9d1", "replay_comparison.json": "34f9cea92af76b4a871c2cf5043318de8bd6eecf99da49cbfe66c4035fbdffe8", "metrics.csv": "1126a181566f3b0fb35d34a8a7bb43541344e7684ca954fc2d3d98fb2925609f", "replay_comparison.output.pt": "63498dd67b92beba3fb2c59eac04cdca62cf629df7ab4d40f01f1ae8e146e23b", "operator.ncu-rep": "62c8f77975a4e26a968b38e12f5b52b531bbbb3e09c566b47df5d173d9fb674b", "run.log": "dad13bc875f95b25b7b408d745ae94274a14b97ce10ea5f057a6963e214538db", "identity.json": "51cad8a6d33ba3d38aea595dd75160d2d0657d829c69efae6aca1e83e666696e", "replay_operator.py": "cbea82d5e280ab2be0b1ce8dfebce6c1e80888c1d9fe727638e97a2652578d9b", "manifest.json": "79e80d439beadd6b6dc8928ecf038cd7598161ae04129728378c19e3609e7c41" } }, { "stage": "vae", "status": "passed", "scheduler_note": [], "output": "/data/zhoutaichang/feature/Lingbot_world_realtime/runs/analysis_report_20260908/raw/ncu_missing704_b611cd0d7be4ea16_v1", "elapsed_seconds": 40.59581995010376, "allocated": "0", "input_sha256": "28dd946d4a53d1263bf5d552d2aacaa22bc7162bea85950320f6d0e02d5750e7", "correctness": { "op": "aten.conv3d.default", "stage": "vae", "shape": [ 1, 16, 3, 88, 160 ], "exact": true, "max_abs": 0.0, "rmse": 0.0, "finite": true, "math_policy": { "matmul_precision": "highest", "matmul_allow_tf32": false, "cudnn_allow_tf32": true, "cudnn_benchmark": false, "cudnn_deterministic": false }, "scope": "Original ATen op on saved real tensors; isolated diagnostic" }, "kernel_count": 2, "kernels": [ { "id": "0", "name": "void cutlass::Kernel2(Params)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 5.824, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 23.516289, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 9.813669, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 24.316526, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 15.687418, "unit": "%" }, "dram__bytes_read.sum": { "value": 2.730752, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 0.0, "unit": "byte" }, "dram__bytes.sum.per_second": { "value": 468.879121, "unit": "Gbyte/s" }, "launch__registers_per_thread": { "value": 136.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 74.752, "unit": "Kbyte/block" } } }, { "id": "1", "name": "void at::elementwise_kernel<128, 2, void at::gpu_kernel_impl_nocast>(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 4.672, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 25.277386, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 12.240192, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 28.005679, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 71.326351, "unit": "%" }, "dram__bytes_read.sum": { "value": 2.717696, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 0.0, "unit": "byte" }, "dram__bytes.sum.per_second": { "value": 581.69863, "unit": "Gbyte/s" }, "launch__registers_per_thread": { "value": 18.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 1.024, "unit": "Kbyte/block" } } } ], "sha256": { "guard_own_pid_history_v2.json": "fc30e40c4f5d935dd4ca82b66d28ccbc8d7e39e77c8c9d007d46bc7a16dfa7fb", "replay_comparison.json": "86611151722f50ea9f8960fb6b0d6f58cef0a9441afbe5f4a67a0823b39bd720", "metrics.csv": "eca989236210ce6700fea8fb13100fac5b0c0c8624f6fea4d9452260ffc2bd23", "replay_comparison.output.pt": "7723c8f254a8bd6e152ec400e944738ace508da7b60fbb216aa35ff5eedd1053", "operator.ncu-rep": "d2214937ed13cb70857a6e61904676f8677d5a9a087fa9e8eca28515b80bd1ed", "run.log": "2652502ef3e4ddd55fb953b569792b3c62fcac37d0e81f81473a16c69757ee3d", "identity.json": "f99714acfc796d9c6f74cb19025c401a017de1d8335eb203a8de44827afd277e", "replay_operator.py": "cbea82d5e280ab2be0b1ce8dfebce6c1e80888c1d9fe727638e97a2652578d9b", "manifest.json": "7bcb420a7f488c60bef258e9e0cab2671f347fd370f7bcc231cf5956bea62e2d" } }, { "stage": "output", "status": "passed", "scheduler_note": [], "output": "/data/zhoutaichang/feature/Lingbot_world_realtime/runs/analysis_report_20260908/raw/ncu_missing704_59ef4c5b540ee17f_v1", "elapsed_seconds": 13.520835399627686, "allocated": "0", "input_sha256": "80d26723a59d26f80a9dfde0de9bf8e28bb650baf110fa77f33200b5bb9e4198", "correctness": { "op": "aten.to.dtype", "stage": "output", "shape": [ 1, 3, 9, 704, 1280 ], "exact": true, "max_abs": 0.0, "rmse": 0.0, "finite": true, "math_policy": { "matmul_precision": "highest", "matmul_allow_tf32": false, "cudnn_allow_tf32": true, "cudnn_benchmark": false, "cudnn_deterministic": false }, "scope": "Original ATen op on saved real tensors; isolated diagnostic" }, "kernel_count": 1, "kernels": [ { "id": "0", "name": "void unrolled_elementwise_kernel, 4, TrivialOffsetCalculator<1, unsigned int>, TrivialOffsetCalculator<1, unsigned int>, LoadWithCast<1>, StoreWithCast<1>>(int, T1, T2, T4, T5, T6, T7)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 64.64, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 57.807509, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 37.370313, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 47.510617, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 90.044417, "unit": "%" }, "dram__bytes_read.sum": { "value": 97.357056, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 18.819328, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 1.797283, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 32.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 1.024, "unit": "Kbyte/block" } } } ], "sha256": { "guard_own_pid_history_v2.json": "e40c75a2424731585478e9b13914691e4418e886538e591678e7467a2e255549", "replay_comparison.json": "85888667c568a7c1c4ccbd9f3df5764b9457b1044444bd61c720aed0e3f0e70f", "metrics.csv": "90f390f74fcc3c51ff2b918a3a653cf5596f4df073c878673bc2b2c0b5bf80ac", "replay_comparison.output.pt": "f942f46fd6b9e534b2b2dccb6aff3a0ab0732bcd9e6fe8b43e389330626cb66b", "operator.ncu-rep": "39f3e627e6f63ddb76dbc4ed3ec786304d25ea0bf5f63ccdbc9d3dcb0dc853ee", "run.log": "6d34ce1c8e154ff51608fcebe990b54d37d1a45bb0e2cc7b0db8657c418ab1ca", "identity.json": "443bf629a431740f78af36dcc40c65a0fff20af4cb01d78c99392dbe57a2e24c", "replay_operator.py": "cbea82d5e280ab2be0b1ce8dfebce6c1e80888c1d9fe727638e97a2652578d9b", "manifest.json": "052017328ded7d04e39aabf68b265ff4105deafb089c0d0aa5d2a4aa24104a1f" } }, { "stage": "output", "status": "passed", "scheduler_note": [], "output": "/data/zhoutaichang/feature/Lingbot_world_realtime/runs/analysis_report_20260908/raw/ncu_missing704_99c188aeb0ac98c3_v1", "elapsed_seconds": 16.84561538696289, "allocated": "0", "input_sha256": "ff3ca616760a5d842212457937cd00ab117a7e2049cc892ea9847400317b91f6", "correctness": { "op": "aten.to.dtype", "stage": "output", "shape": [ 1, 3, 12, 704, 1280 ], "exact": true, "max_abs": 0.0, "rmse": 0.0, "finite": true, "math_policy": { "matmul_precision": "highest", "matmul_allow_tf32": false, "cudnn_allow_tf32": true, "cudnn_benchmark": false, "cudnn_deterministic": false }, "scope": "Original ATen op on saved real tensors; isolated diagnostic" }, "kernel_count": 1, "kernels": [ { "id": "0", "name": "void unrolled_elementwise_kernel, 4, TrivialOffsetCalculator<1, unsigned int>, TrivialOffsetCalculator<1, unsigned int>, LoadWithCast<1>, StoreWithCast<1>>(int, T1, T2, T4, T5, T6, T7)", "device": "0", "metrics": { "gpu__time_duration.sum": { "value": 84.224, "unit": "us" }, "sm__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 59.270813, "unit": "%" }, "gpu__dram_throughput.avg.pct_of_peak_sustained_elapsed": { "value": 38.833031, "unit": "%" }, "lts__throughput.avg.pct_of_peak_sustained_elapsed": { "value": 49.213573, "unit": "%" }, "sm__warps_active.avg.pct_of_peak_sustained_active": { "value": 89.787204, "unit": "%" }, "dram__bytes_read.sum": { "value": 129.7984, "unit": "Mbyte" }, "dram__bytes_write.sum": { "value": 27.542784, "unit": "Mbyte" }, "dram__bytes.sum.per_second": { "value": 1.868128, "unit": "Tbyte/s" }, "launch__registers_per_thread": { "value": 32.0, "unit": "register/thread" }, "launch__shared_mem_per_block": { "value": 1.024, "unit": "Kbyte/block" } } } ], "sha256": { "guard_own_pid_history_v2.json": "a9b38bb1c4eabc62288f6d39f2f09c8705211dadf8db7c021636327be0b3599a", "replay_comparison.json": "55ddcce61f2733a969538cd15bbeef29346a668dbccf68918db89384ee61cc7e", "metrics.csv": "dfb3824f64d854f49c2cfa530488add9e055291731235921024a3b93d22bed08", "replay_comparison.output.pt": "24c20ec98cea09e7808c5fdbda9d4185c626a8cef6284515a9201f6a16580a4e", "operator.ncu-rep": "2f828fb0698db65e226c0985d2d789e6b22c643fda3a50733e7d18cc621c27e6", "run.log": "cce5701769102fd1356108e56f7ad6fddb6f67024ffe63988c09c230346af6ea", "identity.json": "c6f076c002849835b2a98cf31a61d1bd962b4182ed7a30bc3b0a4d32167b3845", "replay_operator.py": "cbea82d5e280ab2be0b1ce8dfebce6c1e80888c1d9fe727638e97a2652578d9b", "manifest.json": "0bf2724ce940cfef93ec85d3650a079aac6ab64b644c451e47af4286c32cdc9d" } } ], "scope": "Twenty-eight additional isolated real-input operator replays, single device per job and distinct allocated devices across jobs. Exact reference output passed. NCU replay instrumentation and clocks/cache effects prevent treating these timings as clean integrated performance. No whole-model MFU or optimization acceptance. Guard history is a point-in-time check of each own PID. PID2019660 has one unknown/foreign teardown-adjacent record, no warning/signal, resolved process exited; cause unproven. This is not a clean scheduler audit." }