{ "generated_at": "2026-09-10T00:28:12.704072+00:00", "source_sqlite": "/data/zhoutaichang/feature/Lingbot_world_realtime/runs/analysis_report_20260908/raw/original_nsys_worker_flush_v2/pipeline.sqlite", "scope": "Longest observed kernel gap for each of four profiled delivery intervals and physical GPUs. GPU mapping uses process-local cudaId through TARGET_INFO_CUDA_DEVICE. Runtime calls from processes mapping to that GPU are contextual, not a causal or device-specific join; multi-GPU processes may overlap other work. Summed thread time is not wall time. Missing recorded GPU activity is not proof of idle.", "host_source_review": [ { "path": "/data/zhoutaichang/feature/Lingbot_world_realtime/runs/analysis_report_20260908/raw/original_nsys_worker_flush_v2/async_vae_spatial_worker.py", "sha256": "c0d863bdd20ae55596048b327f612cd70d33568e6e7077781b6de3d4bb4556e8" }, { "path": "/data/zhoutaichang/feature/Lingbot_world_realtime/runs/analysis_report_20260908/raw/original_nsys_worker_flush_v2/async_vae_spatial_lingbot.py", "sha256": "109cc774d565bfe8fae83a5242f093ac9a4184eb4dd208f5bef9a731d69c6b37" }, { "path": "/data/zhoutaichang/feature/Lingbot_world_realtime/runs/analysis_report_20260908/raw/original_nsys_worker_flush_v2/out/summary.json", "sha256": "b405634ad590f50f77c6bb74a946f3f5e9c07c340ca4fe619342f1a7fcf69c37" } ], "rows": [ { "chunk": 5, "physical_gpu": 0, "gap_start_ns": 260193474718, "gap_end_ns": 260234031861, "gap_ms": 40.557143, "memcpy_memset_union_ms": 0.004448, "without_recorded_kernel_copy_memset_ms": 40.552695, "device_activities": { "CUPTI_ACTIVITY_KIND_MEMCPY": [ { "event_id": 181501, "start": 260193494334, "end": 260193497982, "deviceId": 0, "contextId": 1, "greenContextId": 0, "streamId": 7, "correlationId": 815777, "globalPid": 287143243022336, "bytes": 1, "copyKind": 2, "deprecatedSrcId": null, "srcKind": 2, "dstKind": 1, "srcDeviceId": null, "srcContextId": null, "dstDeviceId": null, "dstContextId": null, "migrationCause": null, "graphNodeId": null, "virtualAddress": null, "copyCount": 1, "overlap_ms": 0.003648 }, { "event_id": 181503, "start": 260233930132, "end": 260233930932, "deviceId": 0, "contextId": 1, "greenContextId": 0, "streamId": 7, "correlationId": 1056629, "globalPid": 287210553212928, "bytes": 256, "copyKind": 1, "deprecatedSrcId": null, "srcKind": 0, "dstKind": 2, "srcDeviceId": null, "srcContextId": null, "dstDeviceId": null, "dstContextId": null, "migrationCause": null, "graphNodeId": null, "virtualAddress": null, "copyCount": 1, "overlap_ms": 0.0008 } ], "CUPTI_ACTIVITY_KIND_MEMSET": [] }, "participating_process_runtime_apis": { "cudaStreamSynchronize_v3020": { "count": 5, "summed_thread_overlap_ms": 0.594438 }, "cudaMemcpyAsync_v3020": { "count": 5, "summed_thread_overlap_ms": 0.146589 }, "cudaLaunchKernel_v7000": { "count": 3, "summed_thread_overlap_ms": 0.076812 }, "cudaStreamIsCapturing_v10000": { "count": 8, "summed_thread_overlap_ms": 0.032224 }, "cuKernelGetName": { "count": 3, "summed_thread_overlap_ms": 0.006771 }, "cudaStreamGetCaptureInfo_v12030": { "count": 4, "summed_thread_overlap_ms": 0.00445 } }, "host_markers": { "previous_cpu_ready_ns": 260193278338, "previous_result_received_ns": 260209564513, "next_dit_start_ns": 260212120448, "ready_to_received_ms": 16.286175, "received_to_dit_start_ms": 2.555935, "dit_start_to_first_kernel_after_gap_ms": 21.911413 }, "host_marker_scope": "Host intervals delimit recorded handoff, not individual hashing/IPC/scheduling costs; first kernel after gap need not be DiT." }, { "chunk": 5, "physical_gpu": 1, "gap_start_ns": 260193278338, "gap_end_ns": 260234047166, "gap_ms": 40.768828, "memcpy_memset_union_ms": 0.000768, "without_recorded_kernel_copy_memset_ms": 40.76806, "device_activities": { "CUPTI_ACTIVITY_KIND_MEMCPY": [ { "event_id": 181504, "start": 260233949662, "end": 260233950430, "deviceId": 1, "contextId": 1, "greenContextId": 0, "streamId": 7, "correlationId": 1091911, "globalPid": 287210569990144, "bytes": 256, "copyKind": 1, "deprecatedSrcId": null, "srcKind": 0, "dstKind": 2, "srcDeviceId": null, "srcContextId": null, "dstDeviceId": null, "dstContextId": null, "migrationCause": null, "graphNodeId": null, "virtualAddress": null, "copyCount": 1, "overlap_ms": 0.000768 } ], "CUPTI_ACTIVITY_KIND_MEMSET": [] }, "participating_process_runtime_apis": { "cudaStreamSynchronize_v3020": { "count": 4, "summed_thread_overlap_ms": 0.5858699999999999 }, "cudaMemcpyAsync_v3020": { "count": 4, "summed_thread_overlap_ms": 0.135227 }, "cudaLaunchKernel_v7000": { "count": 5, "summed_thread_overlap_ms": 0.107164 }, "cudaStreamIsCapturing_v10000": { "count": 8, "summed_thread_overlap_ms": 0.032224 }, "cuKernelGetName": { "count": 5, "summed_thread_overlap_ms": 0.007673 }, "cudaStreamGetCaptureInfo_v12030": { "count": 4, "summed_thread_overlap_ms": 0.00445 } }, "host_markers": { "previous_cpu_ready_ns": 260193278338, "previous_result_received_ns": 260209564513, "next_dit_start_ns": 260212120448, "ready_to_received_ms": 16.286175, "received_to_dit_start_ms": 2.555935, "dit_start_to_first_kernel_after_gap_ms": 21.926718 }, "host_marker_scope": "Host intervals delimit recorded handoff, not individual hashing/IPC/scheduling costs; first kernel after gap need not be DiT." }, { "chunk": 5, "physical_gpu": 2, "gap_start_ns": 260193278338, "gap_end_ns": 260234024825, "gap_ms": 40.746487, "memcpy_memset_union_ms": 0.000768, "without_recorded_kernel_copy_memset_ms": 40.745719, "device_activities": { "CUPTI_ACTIVITY_KIND_MEMCPY": [ { "event_id": 181502, "start": 260233920537, "end": 260233921305, "deviceId": 2, "contextId": 1, "greenContextId": 0, "streamId": 7, "correlationId": 1105664, "globalPid": 287210586767360, "bytes": 256, "copyKind": 1, "deprecatedSrcId": null, "srcKind": 0, "dstKind": 2, "srcDeviceId": null, "srcContextId": null, "dstDeviceId": null, "dstContextId": null, "migrationCause": null, "graphNodeId": null, "virtualAddress": null, "copyCount": 1, "overlap_ms": 0.000768 } ], "CUPTI_ACTIVITY_KIND_MEMSET": [] }, "participating_process_runtime_apis": { "cudaStreamSynchronize_v3020": { "count": 4, "summed_thread_overlap_ms": 0.5858699999999999 }, "cudaMemcpyAsync_v3020": { "count": 4, "summed_thread_overlap_ms": 0.135227 }, "cudaLaunchKernel_v7000": { "count": 3, "summed_thread_overlap_ms": 0.060464000000000004 }, "cudaStreamIsCapturing_v10000": { "count": 8, "summed_thread_overlap_ms": 0.032224 }, "cuKernelGetName": { "count": 3, "summed_thread_overlap_ms": 0.006771 }, "cudaStreamGetCaptureInfo_v12030": { "count": 4, "summed_thread_overlap_ms": 0.00445 } }, "host_markers": { "previous_cpu_ready_ns": 260193278338, "previous_result_received_ns": 260209564513, "next_dit_start_ns": 260212120448, "ready_to_received_ms": 16.286175, "received_to_dit_start_ms": 2.555935, "dit_start_to_first_kernel_after_gap_ms": 21.904377 }, "host_marker_scope": "Host intervals delimit recorded handoff, not individual hashing/IPC/scheduling costs; first kernel after gap need not be DiT." }, { "chunk": 5, "physical_gpu": 3, "gap_start_ns": 260193278338, "gap_end_ns": 260234084282, "gap_ms": 40.805944, "memcpy_memset_union_ms": 0.000736, "without_recorded_kernel_copy_memset_ms": 40.805208, "device_activities": { "CUPTI_ACTIVITY_KIND_MEMCPY": [ { "event_id": 181505, "start": 260233989818, "end": 260233990554, "deviceId": 3, "contextId": 1, "greenContextId": 0, "streamId": 7, "correlationId": 1105838, "globalPid": 287210603544576, "bytes": 256, "copyKind": 1, "deprecatedSrcId": null, "srcKind": 0, "dstKind": 2, "srcDeviceId": null, "srcContextId": null, "dstDeviceId": null, "dstContextId": null, "migrationCause": null, "graphNodeId": null, "virtualAddress": null, "copyCount": 1, "overlap_ms": 0.000736 } ], "CUPTI_ACTIVITY_KIND_MEMSET": [] }, "participating_process_runtime_apis": { "cudaStreamSynchronize_v3020": { "count": 4, "summed_thread_overlap_ms": 0.5858699999999999 }, "cudaMemcpyAsync_v3020": { "count": 6, "summed_thread_overlap_ms": 0.161147 }, "cudaLaunchKernel_v7000": { "count": 7, "summed_thread_overlap_ms": 0.147978 }, "cudaStreamIsCapturing_v10000": { "count": 8, "summed_thread_overlap_ms": 0.032224 }, "cuKernelGetName": { "count": 7, "summed_thread_overlap_ms": 0.009714 }, "cudaStreamGetCaptureInfo_v12030": { "count": 4, "summed_thread_overlap_ms": 0.00445 } }, "host_markers": { "previous_cpu_ready_ns": 260193278338, "previous_result_received_ns": 260209564513, "next_dit_start_ns": 260212120448, "ready_to_received_ms": 16.286175, "received_to_dit_start_ms": 2.555935, "dit_start_to_first_kernel_after_gap_ms": 21.963834 }, "host_marker_scope": "Host intervals delimit recorded handoff, not individual hashing/IPC/scheduling costs; first kernel after gap need not be DiT." }, { "chunk": 6, "physical_gpu": 0, "gap_start_ns": 261040954845, "gap_end_ns": 261089904755, "gap_ms": 48.94991, "memcpy_memset_union_ms": 0.004224, "without_recorded_kernel_copy_memset_ms": 48.945686, "device_activities": { "CUPTI_ACTIVITY_KIND_MEMCPY": [ { "event_id": 191672, "start": 261040992509, "end": 261040995933, "deviceId": 0, "contextId": 1, "greenContextId": 0, "streamId": 7, "correlationId": 853148, "globalPid": 287143243022336, "bytes": 1, "copyKind": 2, "deprecatedSrcId": null, "srcKind": 2, "dstKind": 1, "srcDeviceId": null, "srcContextId": null, "dstDeviceId": null, "dstContextId": null, "migrationCause": null, "graphNodeId": null, "virtualAddress": null, "copyCount": 1, "overlap_ms": 0.003424 }, { "event_id": 191675, "start": 261089810387, "end": 261089811187, "deviceId": 0, "contextId": 1, "greenContextId": 0, "streamId": 7, "correlationId": 1080101, "globalPid": 287210553212928, "bytes": 256, "copyKind": 1, "deprecatedSrcId": null, "srcKind": 0, "dstKind": 2, "srcDeviceId": null, "srcContextId": null, "dstDeviceId": null, "dstContextId": null, "migrationCause": null, "graphNodeId": null, "virtualAddress": null, "copyCount": 1, "overlap_ms": 0.0008 } ], "CUPTI_ACTIVITY_KIND_MEMSET": [] }, "participating_process_runtime_apis": { "cudaStreamSynchronize_v3020": { "count": 5, "summed_thread_overlap_ms": 0.5755049999999999 }, "cudaEventQuery_v3020": { "count": 123, "summed_thread_overlap_ms": 0.27792300000000003 }, "cudaMemcpyAsync_v3020": { "count": 5, "summed_thread_overlap_ms": 0.168414 }, "cudaLaunchKernel_v7000": { "count": 4, "summed_thread_overlap_ms": 0.086882 }, "cudaStreamIsCapturing_v10000": { "count": 8, "summed_thread_overlap_ms": 0.036824 }, "cudaThreadExchangeStreamCaptureMode_v10010": { "count": 82, "summed_thread_overlap_ms": 0.026813000000000007 }, "cuKernelGetName": { "count": 4, "summed_thread_overlap_ms": 0.007039 }, "cudaStreamGetCaptureInfo_v12030": { "count": 4, "summed_thread_overlap_ms": 0.0048070000000000005 } }, "host_markers": { "previous_cpu_ready_ns": 261040772109, "previous_result_received_ns": 261063278322, "next_dit_start_ns": 261065899476, "ready_to_received_ms": 22.506213, "received_to_dit_start_ms": 2.621154, "dit_start_to_first_kernel_after_gap_ms": 24.005279 }, "host_marker_scope": "Host intervals delimit recorded handoff, not individual hashing/IPC/scheduling costs; first kernel after gap need not be DiT." }, { "chunk": 6, "physical_gpu": 1, "gap_start_ns": 261040772109, "gap_end_ns": 261089898010, "gap_ms": 49.125901, "memcpy_memset_union_ms": 0.000736, "without_recorded_kernel_copy_memset_ms": 49.125165, "device_activities": { "CUPTI_ACTIVITY_KIND_MEMCPY": [ { "event_id": 191673, "start": 261089798970, "end": 261089799706, "deviceId": 1, "contextId": 1, "greenContextId": 0, "streamId": 7, "correlationId": 1115318, "globalPid": 287210569990144, "bytes": 256, "copyKind": 1, "deprecatedSrcId": null, "srcKind": 0, "dstKind": 2, "srcDeviceId": null, "srcContextId": null, "dstDeviceId": null, "dstContextId": null, "migrationCause": null, "graphNodeId": null, "virtualAddress": null, "copyCount": 1, "overlap_ms": 0.000736 } ], "CUPTI_ACTIVITY_KIND_MEMSET": [] }, "participating_process_runtime_apis": { "cudaStreamSynchronize_v3020": { "count": 4, "summed_thread_overlap_ms": 0.5669019999999999 }, "cudaMemcpyAsync_v3020": { "count": 4, "summed_thread_overlap_ms": 0.156045 }, "cudaLaunchKernel_v7000": { "count": 3, "summed_thread_overlap_ms": 0.065709 }, "cudaStreamIsCapturing_v10000": { "count": 8, "summed_thread_overlap_ms": 0.036824 }, "cuKernelGetName": { "count": 4, "summed_thread_overlap_ms": 0.006275 }, "cudaStreamGetCaptureInfo_v12030": { "count": 4, "summed_thread_overlap_ms": 0.0048070000000000005 } }, "host_markers": { "previous_cpu_ready_ns": 261040772109, "previous_result_received_ns": 261063278322, "next_dit_start_ns": 261065899476, "ready_to_received_ms": 22.506213, "received_to_dit_start_ms": 2.621154, "dit_start_to_first_kernel_after_gap_ms": 23.998534 }, "host_marker_scope": "Host intervals delimit recorded handoff, not individual hashing/IPC/scheduling costs; first kernel after gap need not be DiT." }, { "chunk": 6, "physical_gpu": 2, "gap_start_ns": 261040772109, "gap_end_ns": 261089905878, "gap_ms": 49.133769, "memcpy_memset_union_ms": 0.000768, "without_recorded_kernel_copy_memset_ms": 49.133001, "device_activities": { "CUPTI_ACTIVITY_KIND_MEMCPY": [ { "event_id": 191674, "start": 261089808918, "end": 261089809686, "deviceId": 2, "contextId": 1, "greenContextId": 0, "streamId": 7, "correlationId": 1129075, "globalPid": 287210586767360, "bytes": 256, "copyKind": 1, "deprecatedSrcId": null, "srcKind": 0, "dstKind": 2, "srcDeviceId": null, "srcContextId": null, "dstDeviceId": null, "dstContextId": null, "migrationCause": null, "graphNodeId": null, "virtualAddress": null, "copyCount": 1, "overlap_ms": 0.000768 } ], "CUPTI_ACTIVITY_KIND_MEMSET": [] }, "participating_process_runtime_apis": { "cudaStreamSynchronize_v3020": { "count": 4, "summed_thread_overlap_ms": 0.5669019999999999 }, "cudaEventQuery_v3020": { "count": 135, "summed_thread_overlap_ms": 0.3057799999999999 }, "cudaMemcpyAsync_v3020": { "count": 4, "summed_thread_overlap_ms": 0.156045 }, "cudaLaunchKernel_v7000": { "count": 4, "summed_thread_overlap_ms": 0.090251 }, "cudaStreamIsCapturing_v10000": { "count": 8, "summed_thread_overlap_ms": 0.036824 }, "cudaThreadExchangeStreamCaptureMode_v10010": { "count": 90, "summed_thread_overlap_ms": 0.029774999999999992 }, "cuKernelGetName": { "count": 4, "summed_thread_overlap_ms": 0.007039 }, "cudaStreamGetCaptureInfo_v12030": { "count": 4, "summed_thread_overlap_ms": 0.0048070000000000005 } }, "host_markers": { "previous_cpu_ready_ns": 261040772109, "previous_result_received_ns": 261063278322, "next_dit_start_ns": 261065899476, "ready_to_received_ms": 22.506213, "received_to_dit_start_ms": 2.621154, "dit_start_to_first_kernel_after_gap_ms": 24.006402 }, "host_marker_scope": "Host intervals delimit recorded handoff, not individual hashing/IPC/scheduling costs; first kernel after gap need not be DiT." }, { "chunk": 6, "physical_gpu": 3, "gap_start_ns": 261040772109, "gap_end_ns": 261089925442, "gap_ms": 49.153333, "memcpy_memset_union_ms": 0.000768, "without_recorded_kernel_copy_memset_ms": 49.152565, "device_activities": { "CUPTI_ACTIVITY_KIND_MEMCPY": [ { "event_id": 191676, "start": 261089832450, "end": 261089833218, "deviceId": 3, "contextId": 1, "greenContextId": 0, "streamId": 7, "correlationId": 1129249, "globalPid": 287210603544576, "bytes": 256, "copyKind": 1, "deprecatedSrcId": null, "srcKind": 0, "dstKind": 2, "srcDeviceId": null, "srcContextId": null, "dstDeviceId": null, "dstContextId": null, "migrationCause": null, "graphNodeId": null, "virtualAddress": null, "copyCount": 1, "overlap_ms": 0.000768 } ], "CUPTI_ACTIVITY_KIND_MEMSET": [] }, "participating_process_runtime_apis": { "cudaStreamSynchronize_v3020": { "count": 4, "summed_thread_overlap_ms": 0.5669019999999999 }, "cudaEventQuery_v3020": { "count": 222, "summed_thread_overlap_ms": 0.436899 }, "cudaMemcpyAsync_v3020": { "count": 4, "summed_thread_overlap_ms": 0.156045 }, "cudaLaunchKernel_v7000": { "count": 7, "summed_thread_overlap_ms": 0.137739 }, "cudaThreadExchangeStreamCaptureMode_v10010": { "count": 148, "summed_thread_overlap_ms": 0.04465800000000001 }, "cudaStreamIsCapturing_v10000": { "count": 8, "summed_thread_overlap_ms": 0.036824 }, "cuKernelGetName": { "count": 7, "summed_thread_overlap_ms": 0.008390000000000002 }, "cudaStreamGetCaptureInfo_v12030": { "count": 4, "summed_thread_overlap_ms": 0.0048070000000000005 } }, "host_markers": { "previous_cpu_ready_ns": 261040772109, "previous_result_received_ns": 261063278322, "next_dit_start_ns": 261065899476, "ready_to_received_ms": 22.506213, "received_to_dit_start_ms": 2.621154, "dit_start_to_first_kernel_after_gap_ms": 24.025966 }, "host_marker_scope": "Host intervals delimit recorded handoff, not individual hashing/IPC/scheduling costs; first kernel after gap need not be DiT." }, { "chunk": 7, "physical_gpu": 0, "gap_start_ns": 261894132314, "gap_end_ns": 261941808300, "gap_ms": 47.675986, "memcpy_memset_union_ms": 0.004224, "without_recorded_kernel_copy_memset_ms": 47.671762, "device_activities": { "CUPTI_ACTIVITY_KIND_MEMCPY": [ { "event_id": 201843, "start": 261894153082, "end": 261894156506, "deviceId": 0, "contextId": 1, "greenContextId": 0, "streamId": 7, "correlationId": 890466, "globalPid": 287143243022336, "bytes": 1, "copyKind": 2, "deprecatedSrcId": null, "srcKind": 2, "dstKind": 1, "srcDeviceId": null, "srcContextId": null, "dstDeviceId": null, "dstContextId": null, "migrationCause": null, "graphNodeId": null, "virtualAddress": null, "copyCount": 1, "overlap_ms": 0.003424 }, { "event_id": 201856, "start": 261941640395, "end": 261941641195, "deviceId": 0, "contextId": 1, "greenContextId": 0, "streamId": 7, "correlationId": 1103563, "globalPid": 287210553212928, "bytes": 256, "copyKind": 1, "deprecatedSrcId": null, "srcKind": 0, "dstKind": 2, "srcDeviceId": null, "srcContextId": null, "dstDeviceId": null, "dstContextId": null, "migrationCause": null, "graphNodeId": null, "virtualAddress": null, "copyCount": 1, "overlap_ms": 0.0008 } ], "CUPTI_ACTIVITY_KIND_MEMSET": [] }, "participating_process_runtime_apis": { "cudaStreamSynchronize_v3020": { "count": 5, "summed_thread_overlap_ms": 0.612806 }, "cudaLaunchKernel_v7000": { "count": 45, "summed_thread_overlap_ms": 0.4570870000000001 }, "cudaMemcpyAsync_v3020": { "count": 14, "summed_thread_overlap_ms": 0.304432 }, "cudaStreamIsCapturing_v10000": { "count": 8, "summed_thread_overlap_ms": 0.049057 }, "cudaLaunchKernelExC_v11060": { "count": 5, "summed_thread_overlap_ms": 0.045617000000000005 }, "cuKernelGetName": { "count": 45, "summed_thread_overlap_ms": 0.023079999999999996 }, "cudaStreamGetCaptureInfo_v12030": { "count": 4, "summed_thread_overlap_ms": 0.0047810000000000005 } }, "host_markers": { "previous_cpu_ready_ns": 261893917185, "previous_result_received_ns": 261916254372, "next_dit_start_ns": 261918795880, "ready_to_received_ms": 22.337187, "received_to_dit_start_ms": 2.541508, "dit_start_to_first_kernel_after_gap_ms": 23.01242 }, "host_marker_scope": "Host intervals delimit recorded handoff, not individual hashing/IPC/scheduling costs; first kernel after gap need not be DiT." }, { "chunk": 7, "physical_gpu": 1, "gap_start_ns": 261893917185, "gap_end_ns": 261941107540, "gap_ms": 47.190355, "memcpy_memset_union_ms": 0.000736, "without_recorded_kernel_copy_memset_ms": 47.189619, "device_activities": { "CUPTI_ACTIVITY_KIND_MEMCPY": [ { "event_id": 201846, "start": 261941014612, "end": 261941015348, "deviceId": 1, "contextId": 1, "greenContextId": 0, "streamId": 7, "correlationId": 1138726, "globalPid": 287210569990144, "bytes": 256, "copyKind": 1, "deprecatedSrcId": null, "srcKind": 0, "dstKind": 2, "srcDeviceId": null, "srcContextId": null, "dstDeviceId": null, "dstContextId": null, "migrationCause": null, "graphNodeId": null, "virtualAddress": null, "copyCount": 1, "overlap_ms": 0.000736 } ], "CUPTI_ACTIVITY_KIND_MEMSET": [] }, "participating_process_runtime_apis": { "cudaStreamSynchronize_v3020": { "count": 3, "summed_thread_overlap_ms": 0.445627 }, "cudaEventQuery_v3020": { "count": 123, "summed_thread_overlap_ms": 0.2612689999999999 }, "cudaMemcpyAsync_v3020": { "count": 3, "summed_thread_overlap_ms": 0.094697 }, "cudaLaunchKernel_v7000": { "count": 3, "summed_thread_overlap_ms": 0.075332 }, "cudaStreamIsCapturing_v10000": { "count": 7, "summed_thread_overlap_ms": 0.045726 }, "cudaThreadExchangeStreamCaptureMode_v10010": { "count": 82, "summed_thread_overlap_ms": 0.026675000000000004 }, "cuKernelGetName": { "count": 3, "summed_thread_overlap_ms": 0.005085 }, "cudaStreamGetCaptureInfo_v12030": { "count": 4, "summed_thread_overlap_ms": 0.0047810000000000005 } }, "host_markers": { "previous_cpu_ready_ns": 261893917185, "previous_result_received_ns": 261916254372, "next_dit_start_ns": 261918795880, "ready_to_received_ms": 22.337187, "received_to_dit_start_ms": 2.541508, "dit_start_to_first_kernel_after_gap_ms": 22.31166 }, "host_marker_scope": "Host intervals delimit recorded handoff, not individual hashing/IPC/scheduling costs; first kernel after gap need not be DiT." }, { "chunk": 7, "physical_gpu": 2, "gap_start_ns": 261893917185, "gap_end_ns": 261941107839, "gap_ms": 47.190654, "memcpy_memset_union_ms": 0.000768, "without_recorded_kernel_copy_memset_ms": 47.189886, "device_activities": { "CUPTI_ACTIVITY_KIND_MEMCPY": [ { "event_id": 201845, "start": 261941013119, "end": 261941013887, "deviceId": 2, "contextId": 1, "greenContextId": 0, "streamId": 7, "correlationId": 1152474, "globalPid": 287210586767360, "bytes": 256, "copyKind": 1, "deprecatedSrcId": null, "srcKind": 0, "dstKind": 2, "srcDeviceId": null, "srcContextId": null, "dstDeviceId": null, "dstContextId": null, "migrationCause": null, "graphNodeId": null, "virtualAddress": null, "copyCount": 1, "overlap_ms": 0.000768 } ], "CUPTI_ACTIVITY_KIND_MEMSET": [] }, "participating_process_runtime_apis": { "cudaStreamSynchronize_v3020": { "count": 3, "summed_thread_overlap_ms": 0.445627 }, "cudaMemcpyAsync_v3020": { "count": 3, "summed_thread_overlap_ms": 0.094697 }, "cudaLaunchKernel_v7000": { "count": 3, "summed_thread_overlap_ms": 0.076229 }, "cudaStreamIsCapturing_v10000": { "count": 7, "summed_thread_overlap_ms": 0.045726 }, "cuKernelGetName": { "count": 3, "summed_thread_overlap_ms": 0.005085 }, "cudaStreamGetCaptureInfo_v12030": { "count": 4, "summed_thread_overlap_ms": 0.0047810000000000005 } }, "host_markers": { "previous_cpu_ready_ns": 261893917185, "previous_result_received_ns": 261916254372, "next_dit_start_ns": 261918795880, "ready_to_received_ms": 22.337187, "received_to_dit_start_ms": 2.541508, "dit_start_to_first_kernel_after_gap_ms": 22.311959 }, "host_marker_scope": "Host intervals delimit recorded handoff, not individual hashing/IPC/scheduling costs; first kernel after gap need not be DiT." }, { "chunk": 7, "physical_gpu": 3, "gap_start_ns": 261893917185, "gap_end_ns": 261941108253, "gap_ms": 47.191068, "memcpy_memset_union_ms": 0.001088, "without_recorded_kernel_copy_memset_ms": 47.18998, "device_activities": { "CUPTI_ACTIVITY_KIND_MEMCPY": [ { "event_id": 201844, "start": 261941009789, "end": 261941010877, "deviceId": 3, "contextId": 1, "greenContextId": 0, "streamId": 7, "correlationId": 1152657, "globalPid": 287210603544576, "bytes": 256, "copyKind": 1, "deprecatedSrcId": null, "srcKind": 0, "dstKind": 2, "srcDeviceId": null, "srcContextId": null, "dstDeviceId": null, "dstContextId": null, "migrationCause": null, "graphNodeId": null, "virtualAddress": null, "copyCount": 1, "overlap_ms": 0.001088 } ], "CUPTI_ACTIVITY_KIND_MEMSET": [] }, "participating_process_runtime_apis": { "cudaStreamSynchronize_v3020": { "count": 3, "summed_thread_overlap_ms": 0.445627 }, "cudaMemcpyAsync_v3020": { "count": 3, "summed_thread_overlap_ms": 0.094697 }, "cudaLaunchKernel_v7000": { "count": 3, "summed_thread_overlap_ms": 0.07747100000000001 }, "cudaStreamIsCapturing_v10000": { "count": 7, "summed_thread_overlap_ms": 0.045726 }, "cuKernelGetName": { "count": 3, "summed_thread_overlap_ms": 0.005085 }, "cudaStreamGetCaptureInfo_v12030": { "count": 4, "summed_thread_overlap_ms": 0.0047810000000000005 } }, "host_markers": { "previous_cpu_ready_ns": 261893917185, "previous_result_received_ns": 261916254372, "next_dit_start_ns": 261918795880, "ready_to_received_ms": 22.337187, "received_to_dit_start_ms": 2.541508, "dit_start_to_first_kernel_after_gap_ms": 22.312373 }, "host_marker_scope": "Host intervals delimit recorded handoff, not individual hashing/IPC/scheduling costs; first kernel after gap need not be DiT." }, { "chunk": 8, "physical_gpu": 0, "gap_start_ns": 262747885112, "gap_end_ns": 262797615760, "gap_ms": 49.730648, "memcpy_memset_union_ms": 0.004256, "without_recorded_kernel_copy_memset_ms": 49.726392, "device_activities": { "CUPTI_ACTIVITY_KIND_MEMCPY": [ { "event_id": 212014, "start": 262747913272, "end": 262747916728, "deviceId": 0, "contextId": 1, "greenContextId": 0, "streamId": 7, "correlationId": 927421, "globalPid": 287143243022336, "bytes": 1, "copyKind": 2, "deprecatedSrcId": null, "srcKind": 2, "dstKind": 1, "srcDeviceId": null, "srcContextId": null, "dstDeviceId": null, "dstContextId": null, "migrationCause": null, "graphNodeId": null, "virtualAddress": null, "copyCount": 1, "overlap_ms": 0.003456 }, { "event_id": 212016, "start": 262797509040, "end": 262797509840, "deviceId": 0, "contextId": 1, "greenContextId": 0, "streamId": 7, "correlationId": 1127025, "globalPid": 287210553212928, "bytes": 256, "copyKind": 1, "deprecatedSrcId": null, "srcKind": 0, "dstKind": 2, "srcDeviceId": null, "srcContextId": null, "dstDeviceId": null, "dstContextId": null, "migrationCause": null, "graphNodeId": null, "virtualAddress": null, "copyCount": 1, "overlap_ms": 0.0008 } ], "CUPTI_ACTIVITY_KIND_MEMSET": [] }, "participating_process_runtime_apis": { "cudaEventQuery_v3020": { "count": 123, "summed_thread_overlap_ms": 0.3321100000000001 }, "cudaStreamSynchronize_v3020": { "count": 3, "summed_thread_overlap_ms": 0.278651 }, "cudaMemcpyAsync_v3020": { "count": 3, "summed_thread_overlap_ms": 0.11646000000000001 }, "cudaLaunchKernel_v7000": { "count": 2, "summed_thread_overlap_ms": 0.060229000000000005 }, "cudaThreadExchangeStreamCaptureMode_v10010": { "count": 82, "summed_thread_overlap_ms": 0.04934700000000001 }, "cudaStreamIsCapturing_v10000": { "count": 6, "summed_thread_overlap_ms": 0.029148999999999998 }, "cudaStreamGetCaptureInfo_v12030": { "count": 4, "summed_thread_overlap_ms": 0.005041 }, "cuKernelGetName": { "count": 2, "summed_thread_overlap_ms": 0.00389 } }, "host_markers": { "previous_cpu_ready_ns": 262747644781, "previous_result_received_ns": 262771966351, "next_dit_start_ns": 262775947165, "ready_to_received_ms": 24.32157, "received_to_dit_start_ms": 3.980814, "dit_start_to_first_kernel_after_gap_ms": 21.668595 }, "host_marker_scope": "Host intervals delimit recorded handoff, not individual hashing/IPC/scheduling costs; first kernel after gap need not be DiT." }, { "chunk": 8, "physical_gpu": 1, "gap_start_ns": 262747644781, "gap_end_ns": 262798056635, "gap_ms": 50.411854, "memcpy_memset_union_ms": 0.000736, "without_recorded_kernel_copy_memset_ms": 50.411118, "device_activities": { "CUPTI_ACTIVITY_KIND_MEMCPY": [ { "event_id": 212020, "start": 262797961595, "end": 262797962331, "deviceId": 1, "contextId": 1, "greenContextId": 0, "streamId": 7, "correlationId": 1162126, "globalPid": 287210569990144, "bytes": 256, "copyKind": 1, "deprecatedSrcId": null, "srcKind": 0, "dstKind": 2, "srcDeviceId": null, "srcContextId": null, "dstDeviceId": null, "dstContextId": null, "migrationCause": null, "graphNodeId": null, "virtualAddress": null, "copyCount": 1, "overlap_ms": 0.000736 } ], "CUPTI_ACTIVITY_KIND_MEMSET": [] }, "participating_process_runtime_apis": { "cudaStreamSynchronize_v3020": { "count": 4, "summed_thread_overlap_ms": 0.564089 }, "cudaMemcpyAsync_v3020": { "count": 7, "summed_thread_overlap_ms": 0.22814 }, "cudaLaunchKernel_v7000": { "count": 15, "summed_thread_overlap_ms": 0.216484 }, "cudaStreamIsCapturing_v10000": { "count": 8, "summed_thread_overlap_ms": 0.035494 }, "cuKernelGetName": { "count": 15, "summed_thread_overlap_ms": 0.012302 }, "cudaLaunchKernelExC_v11060": { "count": 1, "summed_thread_overlap_ms": 0.010516 }, "cudaStreamGetCaptureInfo_v12030": { "count": 4, "summed_thread_overlap_ms": 0.005041 } }, "host_markers": { "previous_cpu_ready_ns": 262747644781, "previous_result_received_ns": 262771966351, "next_dit_start_ns": 262775947165, "ready_to_received_ms": 24.32157, "received_to_dit_start_ms": 3.980814, "dit_start_to_first_kernel_after_gap_ms": 22.10947 }, "host_marker_scope": "Host intervals delimit recorded handoff, not individual hashing/IPC/scheduling costs; first kernel after gap need not be DiT." }, { "chunk": 8, "physical_gpu": 2, "gap_start_ns": 262747644781, "gap_end_ns": 262797614605, "gap_ms": 49.969824, "memcpy_memset_union_ms": 0.000768, "without_recorded_kernel_copy_memset_ms": 49.969056, "device_activities": { "CUPTI_ACTIVITY_KIND_MEMCPY": [ { "event_id": 212015, "start": 262797507853, "end": 262797508621, "deviceId": 2, "contextId": 1, "greenContextId": 0, "streamId": 7, "correlationId": 1175882, "globalPid": 287210586767360, "bytes": 256, "copyKind": 1, "deprecatedSrcId": null, "srcKind": 0, "dstKind": 2, "srcDeviceId": null, "srcContextId": null, "dstDeviceId": null, "dstContextId": null, "migrationCause": null, "graphNodeId": null, "virtualAddress": null, "copyCount": 1, "overlap_ms": 0.000768 } ], "CUPTI_ACTIVITY_KIND_MEMSET": [] }, "participating_process_runtime_apis": { "cudaEventQuery_v3020": { "count": 135, "summed_thread_overlap_ms": 0.28537500000000005 }, "cudaStreamSynchronize_v3020": { "count": 2, "summed_thread_overlap_ms": 0.270427 }, "cudaMemcpyAsync_v3020": { "count": 2, "summed_thread_overlap_ms": 0.10114600000000001 }, "cudaLaunchKernel_v7000": { "count": 2, "summed_thread_overlap_ms": 0.057919 }, "cudaStreamIsCapturing_v10000": { "count": 6, "summed_thread_overlap_ms": 0.029148999999999998 }, "cudaThreadExchangeStreamCaptureMode_v10010": { "count": 90, "summed_thread_overlap_ms": 0.027911000000000002 }, "cudaStreamGetCaptureInfo_v12030": { "count": 4, "summed_thread_overlap_ms": 0.005041 }, "cuKernelGetName": { "count": 2, "summed_thread_overlap_ms": 0.00389 } }, "host_markers": { "previous_cpu_ready_ns": 262747644781, "previous_result_received_ns": 262771966351, "next_dit_start_ns": 262775947165, "ready_to_received_ms": 24.32157, "received_to_dit_start_ms": 3.980814, "dit_start_to_first_kernel_after_gap_ms": 21.66744 }, "host_marker_scope": "Host intervals delimit recorded handoff, not individual hashing/IPC/scheduling costs; first kernel after gap need not be DiT." }, { "chunk": 8, "physical_gpu": 3, "gap_start_ns": 262747644781, "gap_end_ns": 262798032450, "gap_ms": 50.387669, "memcpy_memset_union_ms": 0.000768, "without_recorded_kernel_copy_memset_ms": 50.386901, "device_activities": { "CUPTI_ACTIVITY_KIND_MEMCPY": [ { "event_id": 212019, "start": 262797934530, "end": 262797935298, "deviceId": 3, "contextId": 1, "greenContextId": 0, "streamId": 7, "correlationId": 1176063, "globalPid": 287210603544576, "bytes": 256, "copyKind": 1, "deprecatedSrcId": null, "srcKind": 0, "dstKind": 2, "srcDeviceId": null, "srcContextId": null, "dstDeviceId": null, "dstContextId": null, "migrationCause": null, "graphNodeId": null, "virtualAddress": null, "copyCount": 1, "overlap_ms": 0.000768 } ], "CUPTI_ACTIVITY_KIND_MEMSET": [] }, "participating_process_runtime_apis": { "cudaStreamSynchronize_v3020": { "count": 4, "summed_thread_overlap_ms": 0.564089 }, "cudaEventQuery_v3020": { "count": 222, "summed_thread_overlap_ms": 0.43817099999999987 }, "cudaMemcpyAsync_v3020": { "count": 6, "summed_thread_overlap_ms": 0.21724600000000002 }, "cudaLaunchKernel_v7000": { "count": 14, "summed_thread_overlap_ms": 0.182479 }, "cudaThreadExchangeStreamCaptureMode_v10010": { "count": 148, "summed_thread_overlap_ms": 0.056965000000000016 }, "cudaStreamIsCapturing_v10000": { "count": 8, "summed_thread_overlap_ms": 0.035494 }, "cuKernelGetName": { "count": 14, "summed_thread_overlap_ms": 0.011642000000000001 }, "cudaLaunchKernelExC_v11060": { "count": 1, "summed_thread_overlap_ms": 0.010516 }, "cudaStreamGetCaptureInfo_v12030": { "count": 4, "summed_thread_overlap_ms": 0.005041 } }, "host_markers": { "previous_cpu_ready_ns": 262747644781, "previous_result_received_ns": 262771966351, "next_dit_start_ns": 262775947165, "ready_to_received_ms": 24.32157, "received_to_dit_start_ms": 3.980814, "dit_start_to_first_kernel_after_gap_ms": 22.085285 }, "host_marker_scope": "Host intervals delimit recorded handoff, not individual hashing/IPC/scheduling costs; first kernel after gap need not be DiT." } ] }