When HES and cuGraphUpload are used together, cupti kernel activities always return correlationId = 0. It’s verified using cupti 13.3 and cuda driver 13.3.
Both options default to enabled:
# HES on, upload on
/mnt/keren/tmp/cupti_hes_graph_activity
# HES on, upload off
/mnt/keren/tmp/cupti_hes_graph_activity --no-upload
# HES off, upload on
/mnt/keren/tmp/cupti_hes_graph_activity --no-hes
# HES off, upload off
/mnt/keren/tmp/cupti_hes_graph_activity --no-hes --no-upload
Results with 10 nodes × 3 replays:
HES Upload Zero correlation records
━━━━━ ━━━━━━━━ ━━━━━━━━━━━━━━━━━━━━━━━━━━
On On 30/30
───── ──────── ──────────────────────────
On Off 0/30
───── ──────── ──────────────────────────
Off On 0/30
───── ──────── ──────────────────────────
Off Off 0/30
#include <cuda.h>
#include <cuda_runtime.h>
#include <cupti.h>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <map>
#include <mutex>
#include <utility>
#include <vector>
#define CHECK(call) \
do { \
auto status = (call); \
if (status != 0) { \
std::fprintf(stderr, "%s failed: %d\n", #call, int(status)); \
return 1; \
} \
} while (0)
__global__ void kernel(float *output, int iterations) {
float value = threadIdx.x;
for (int i = 0; i < iterations; ++i)
value = value * 1.0000001192092896f + 0.0000001192092896f;
output[threadIdx.x] = value;
}
static std::mutex completedMutex;
static std::vector<std::pair<uint8_t *, size_t>> completedBuffers;
static void CUPTIAPI apiCallback(void *, CUpti_CallbackDomain domain,
CUpti_CallbackId callbackId,
const void *callbackData) {
if (domain != CUPTI_CB_DOMAIN_DRIVER_API ||
callbackId != CUPTI_DRIVER_TRACE_CBID_cuGraphLaunch)
return;
const auto *data = static_cast<const CUpti_CallbackData *>(callbackData);
std::printf("graph launch site=%u corr=%u\n", unsigned(data->callbackSite),
data->correlationId);
}
static void CUPTIAPI requestBuffer(uint8_t **buffer, size_t *size,
size_t *maxRecords) {
*size = 1024 * 1024;
*buffer = static_cast<uint8_t *>(std::malloc(*size));
*maxRecords = 0;
}
static void CUPTIAPI completeBuffer(CUcontext, uint32_t, uint8_t *buffer,
size_t, size_t validSize) {
std::lock_guard<std::mutex> lock(completedMutex);
completedBuffers.emplace_back(buffer, validSize);
}
int main(int argc, char **argv) {
bool enableHes = true;
bool uploadGraph = true;
for (int i = 1; i < argc; ++i) {
if (std::strcmp(argv[i], "--hes") == 0) {
enableHes = true;
} else if (std::strcmp(argv[i], "--no-hes") == 0) {
enableHes = false;
} else if (std::strcmp(argv[i], "--upload") == 0) {
uploadGraph = true;
} else if (std::strcmp(argv[i], "--no-upload") == 0) {
uploadGraph = false;
} else if (std::strcmp(argv[i], "--help") == 0) {
std::printf("usage: %s [--hes|--no-hes] "
"[--upload|--no-upload]\n",
argv[0]);
return 0;
} else {
std::fprintf(stderr, "unknown option: %s\n", argv[i]);
return 2;
}
}
constexpr int kernelNodes = 10;
constexpr int replays = 3;
std::printf("configuration: hes=%s upload=%s nodes=%d replays=%d\n",
enableHes ? "on" : "off", uploadGraph ? "on" : "off",
kernelNodes, replays);
CHECK(cuInit(0));
if (enableHes)
CHECK(cuptiActivityEnableHWTrace(1));
CUpti_SubscriberHandle subscriber;
CHECK(cuptiSubscribe(&subscriber, apiCallback, nullptr));
CHECK(cuptiEnableCallback(1, subscriber, CUPTI_CB_DOMAIN_DRIVER_API,
CUPTI_DRIVER_TRACE_CBID_cuGraphLaunch));
CHECK(cuptiActivityRegisterCallbacks(requestBuffer, completeBuffer));
CHECK(cuptiActivityEnable(CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL));
float *output;
CHECK(cudaMalloc(&output, 256 * sizeof(*output)));
cudaStream_t stream;
cudaGraph_t graph;
cudaGraphExec_t executable;
CHECK(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking));
CHECK(cudaGraphCreate(&graph, 0));
int iterations = 100000;
void *arguments[] = {&output, &iterations};
cudaGraphNode_t dependency = nullptr;
for (int i = 0; i < kernelNodes; ++i) {
cudaKernelNodeParams parameters{};
parameters.func = reinterpret_cast<void *>(kernel);
parameters.gridDim = dim3(1);
parameters.blockDim = dim3(256);
parameters.kernelParams = arguments;
cudaGraphNode_t node;
CHECK(cudaGraphAddKernelNode(&node, graph,
dependency ? &dependency : nullptr,
dependency ? 1 : 0, ¶meters));
dependency = node;
}
CHECK(cudaGraphInstantiate(&executable, graph));
if (uploadGraph)
CHECK(cudaGraphUpload(executable, stream));
for (int replay = 0; replay < replays; ++replay) {
CHECK(cudaGraphLaunch(executable, stream));
CHECK(cudaDeviceSynchronize());
}
CHECK(cuptiActivityFlushAll(CUPTI_ACTIVITY_FLAG_FLUSH_FORCED));
std::map<int, int> counts;
for (auto [buffer, validSize] : completedBuffers) {
CUpti_Activity *activity = nullptr;
while (cuptiActivityGetNextRecord(buffer, validSize, &activity) ==
CUPTI_SUCCESS) {
++counts[activity->kind];
if (activity->kind == CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL) {
auto *record = reinterpret_cast<CUpti_ActivityKernel12 *>(activity);
std::printf("kernel name=%s corr=%u graph=%u node=%llu grid=%lld "
"start=%llu end=%llu\n",
record->name ? record->name : "<null>",
record->correlationId, record->graphId,
static_cast<unsigned long long>(record->graphNodeId),
static_cast<long long>(record->gridId),
static_cast<unsigned long long>(record->start),
static_cast<unsigned long long>(record->end));
} else if (activity->kind == CUPTI_ACTIVITY_KIND_GRAPH_TRACE) {
auto *record = reinterpret_cast<CUpti_ActivityGraphTrace2 *>(activity);
std::printf("graph corr=%u graph=%u start=%llu end=%llu\n",
record->correlationId, record->graphId,
static_cast<unsigned long long>(record->start),
static_cast<unsigned long long>(record->end));
}
}
std::free(buffer);
}
for (auto [kind, count] : counts)
std::printf("kind=%d count=%d\n", kind, count);
CHECK(cuptiActivityDisable(CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL));
if (enableHes)
CHECK(cuptiActivityEnableHWTrace(0));
return counts[CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL] ==
kernelNodes * replays
? 0
: 6;
}