Cupti HES not compatible with `cuGraphUpload`

When HES and cuGraphUpload are used together, cupti kernel activities always return correlationId = 0. It’s verified using cupti 13.3 and cuda driver 13.3.

  Both options default to enabled:

  # HES on, upload on
  /mnt/keren/tmp/cupti_hes_graph_activity

  # HES on, upload off
  /mnt/keren/tmp/cupti_hes_graph_activity --no-upload

  # HES off, upload on
  /mnt/keren/tmp/cupti_hes_graph_activity --no-hes

  # HES off, upload off
  /mnt/keren/tmp/cupti_hes_graph_activity --no-hes --no-upload

  Results with 10 nodes × 3 replays:

   HES    Upload    Zero correlation records
  ━━━━━  ━━━━━━━━  ━━━━━━━━━━━━━━━━━━━━━━━━━━
   On         On                       30/30
  ─────  ────────  ──────────────────────────
   On        Off                        0/30
  ─────  ────────  ──────────────────────────
   Off        On                        0/30
  ─────  ────────  ──────────────────────────
   Off       Off                        0/30
#include <cuda.h>
#include <cuda_runtime.h>
#include <cupti.h>

#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <map>
#include <mutex>
#include <utility>
#include <vector>

#define CHECK(call)                                                            \
  do {                                                                         \
    auto status = (call);                                                      \
    if (status != 0) {                                                         \
      std::fprintf(stderr, "%s failed: %d\n", #call, int(status));             \
      return 1;                                                                \
    }                                                                          \
  } while (0)

__global__ void kernel(float *output, int iterations) {
  float value = threadIdx.x;
  for (int i = 0; i < iterations; ++i)
    value = value * 1.0000001192092896f + 0.0000001192092896f;
  output[threadIdx.x] = value;
}

static std::mutex completedMutex;
static std::vector<std::pair<uint8_t *, size_t>> completedBuffers;

static void CUPTIAPI apiCallback(void *, CUpti_CallbackDomain domain,
                                 CUpti_CallbackId callbackId,
                                 const void *callbackData) {
  if (domain != CUPTI_CB_DOMAIN_DRIVER_API ||
      callbackId != CUPTI_DRIVER_TRACE_CBID_cuGraphLaunch)
    return;
  const auto *data = static_cast<const CUpti_CallbackData *>(callbackData);
  std::printf("graph launch site=%u corr=%u\n", unsigned(data->callbackSite),
              data->correlationId);
}

static void CUPTIAPI requestBuffer(uint8_t **buffer, size_t *size,
                                   size_t *maxRecords) {
  *size = 1024 * 1024;
  *buffer = static_cast<uint8_t *>(std::malloc(*size));
  *maxRecords = 0;
}

static void CUPTIAPI completeBuffer(CUcontext, uint32_t, uint8_t *buffer,
                                    size_t, size_t validSize) {
  std::lock_guard<std::mutex> lock(completedMutex);
  completedBuffers.emplace_back(buffer, validSize);
}

int main(int argc, char **argv) {
  bool enableHes = true;
  bool uploadGraph = true;
  for (int i = 1; i < argc; ++i) {
    if (std::strcmp(argv[i], "--hes") == 0) {
      enableHes = true;
    } else if (std::strcmp(argv[i], "--no-hes") == 0) {
      enableHes = false;
    } else if (std::strcmp(argv[i], "--upload") == 0) {
      uploadGraph = true;
    } else if (std::strcmp(argv[i], "--no-upload") == 0) {
      uploadGraph = false;
    } else if (std::strcmp(argv[i], "--help") == 0) {
      std::printf("usage: %s [--hes|--no-hes] "
                  "[--upload|--no-upload]\n",
                  argv[0]);
      return 0;
    } else {
      std::fprintf(stderr, "unknown option: %s\n", argv[i]);
      return 2;
    }
  }

  constexpr int kernelNodes = 10;
  constexpr int replays = 3;
  std::printf("configuration: hes=%s upload=%s nodes=%d replays=%d\n",
              enableHes ? "on" : "off", uploadGraph ? "on" : "off",
              kernelNodes, replays);

  CHECK(cuInit(0));
  if (enableHes)
    CHECK(cuptiActivityEnableHWTrace(1));
  CUpti_SubscriberHandle subscriber;
  CHECK(cuptiSubscribe(&subscriber, apiCallback, nullptr));
  CHECK(cuptiEnableCallback(1, subscriber, CUPTI_CB_DOMAIN_DRIVER_API,
                            CUPTI_DRIVER_TRACE_CBID_cuGraphLaunch));
  CHECK(cuptiActivityRegisterCallbacks(requestBuffer, completeBuffer));
  CHECK(cuptiActivityEnable(CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL));

  float *output;
  CHECK(cudaMalloc(&output, 256 * sizeof(*output)));

  cudaStream_t stream;
  cudaGraph_t graph;
  cudaGraphExec_t executable;
  CHECK(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking));
  CHECK(cudaGraphCreate(&graph, 0));

  int iterations = 100000;
  void *arguments[] = {&output, &iterations};
  cudaGraphNode_t dependency = nullptr;
  for (int i = 0; i < kernelNodes; ++i) {
    cudaKernelNodeParams parameters{};
    parameters.func = reinterpret_cast<void *>(kernel);
    parameters.gridDim = dim3(1);
    parameters.blockDim = dim3(256);
    parameters.kernelParams = arguments;
    cudaGraphNode_t node;
    CHECK(cudaGraphAddKernelNode(&node, graph,
                                 dependency ? &dependency : nullptr,
                                 dependency ? 1 : 0, &parameters));
    dependency = node;
  }
  CHECK(cudaGraphInstantiate(&executable, graph));
  if (uploadGraph)
    CHECK(cudaGraphUpload(executable, stream));
  for (int replay = 0; replay < replays; ++replay) {
    CHECK(cudaGraphLaunch(executable, stream));
    CHECK(cudaDeviceSynchronize());
  }
  CHECK(cuptiActivityFlushAll(CUPTI_ACTIVITY_FLAG_FLUSH_FORCED));

  std::map<int, int> counts;
  for (auto [buffer, validSize] : completedBuffers) {
    CUpti_Activity *activity = nullptr;
    while (cuptiActivityGetNextRecord(buffer, validSize, &activity) ==
           CUPTI_SUCCESS) {
      ++counts[activity->kind];
      if (activity->kind == CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL) {
        auto *record = reinterpret_cast<CUpti_ActivityKernel12 *>(activity);
        std::printf("kernel name=%s corr=%u graph=%u node=%llu grid=%lld "
                    "start=%llu end=%llu\n",
                    record->name ? record->name : "<null>",
                    record->correlationId, record->graphId,
                    static_cast<unsigned long long>(record->graphNodeId),
                    static_cast<long long>(record->gridId),
                    static_cast<unsigned long long>(record->start),
                    static_cast<unsigned long long>(record->end));
      } else if (activity->kind == CUPTI_ACTIVITY_KIND_GRAPH_TRACE) {
        auto *record = reinterpret_cast<CUpti_ActivityGraphTrace2 *>(activity);
        std::printf("graph corr=%u graph=%u start=%llu end=%llu\n",
                    record->correlationId, record->graphId,
                    static_cast<unsigned long long>(record->start),
                    static_cast<unsigned long long>(record->end));
      }
    }
    std::free(buffer);
  }
  for (auto [kind, count] : counts)
    std::printf("kind=%d count=%d\n", kind, count);

  CHECK(cuptiActivityDisable(CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL));
  if (enableHes)
    CHECK(cuptiActivityEnableHWTrace(0));
  return counts[CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL] ==
                 kernelNodes * replays
             ? 0
             : 6;
}
Hi Keren, thanks for the detailed reproducer.
We tested the provided code on Linux x86_64 with an RTX 5070 (compute capability 12.0), driver 610.43.02 / CUDA UMD 13.3, and CUDA Toolkit 13.3 (nvcc 13.3.73) and CUPTI 13.3.
So far, we have not reproduced the issue. With HES and cudaGraphUpload enabled, the shipped library produced nonzero correlation IDs for all 390 kernel records across 13 runs. The other HES/upload combinations also behaved as expected.
Could you please share:
The exact GPU model and compute capability.
The complete driver version from nvidia-smi.
OS/platform details, including whether MPS, MIG, vGPU, or WSL is involved.
Your compilation command and the exact libcupti and libnvperf_host files loaded at runtime.
If possible, please also include the complete output from one failing run. This should help us identify what differs between our environments.

CUDA UMD Version: 13.3
NVIDIA-SMI 610.43.02
NVIDIA GB200
It’s a ubuntu system configured on Azure

Let me know if there’s any addition information you would like to request

Hi Keren ,

Thanks for providing us this info. We are now able to repro the bug, and are working on a fix.