cuptiProfilerHostGetHostSubMetrics Returning CUPTI_ERROR_INVALID_METRIC_NAME with Cuda Toolkits 12.8.0 and 12.9.0

Developer Tools CUPTI – CUDA Profiler Tools Interface
1

Hello,

When using Cuda Toolkits 12.8.0 and 12.9.0 on an NVIDIA GA100, I encounter the error CUPTI_ERROR_INVALID_METRIC_NAMEfor cuptiProfilerHostGetSubMetrics. However, in Cuda Toolkit 13.0.0 cuptiProfilerHostGetSubMetricswill succeed.

Workflow that can reproduce the error:

#include <iostream>
#include <vector>

#include <cupti_profiler_host.h>
#include <cupti_profiler_target.h>
#include <cuda_runtime_api.h>
#include <cupti_target.h>

int main()
{
    CUresult cu_res = cuInit(0);
    if (cu_res != CUDA_SUCCESS) {
        std::cout << "Call to cuptiProfilerInitialize failed with error code: " << cu_res << std::endl;
        exit(EXIT_FAILURE);
    }

    CUpti_Profiler_Initialize_Params profilerInitializeParams {};
    profilerInitializeParams.structSize = CUpti_Profiler_Initialize_Params_STRUCT_SIZE;
    CUptiResult cupti_res = cuptiProfilerInitialize(&profilerInitializeParams);
    if (cupti_res != CUPTI_SUCCESS) {
        std::cout << "Call to cuptiProfilerInitialize failed with error code: " << cupti_res << std::endl;
        exit(EXIT_FAILURE);
    }

    CUcontext context;
    unsigned int flags = 0;
    int device_index = 0;
    #if CUDART_VERSION >= 13000
    cu_res = cuCtxCreate(&context, (CUctxCreateParams*)0, flags, device_index);
    #else
    cu_res = cuCtxCreate(&context, flags, device_index);
    #endif
    if (cu_res != CUDA_SUCCESS) {
        std::cout << "Call to cuCtxCreate failed with error code: " << cu_res << std::endl;
        exit(EXIT_FAILURE);
    }

    std::vector<uint8_t> counter_availability_image;
    CUpti_Profiler_GetCounterAvailability_Params getCounterAvailabilityParams {};
    getCounterAvailabilityParams.structSize = CUpti_Profiler_GetCounterAvailability_Params_STRUCT_SIZE;
    getCounterAvailabilityParams.ctx = context;
    #if CUDART_VERSION >= 13010
    getCounterAvailabilityParams.bAllowDeviceLevelCounters = 1;
    #endif
    cupti_res = cuptiProfilerGetCounterAvailability(&getCounterAvailabilityParams);
    if (cupti_res != CUPTI_SUCCESS) {
        std::cout << "Call to cuptiProfilerGetCounterAvailability failed with error code: " << cupti_res << std::endl;
        exit(EXIT_FAILURE);
    }

    counter_availability_image.resize(getCounterAvailabilityParams.counterAvailabilityImageSize);
    getCounterAvailabilityParams.pCounterAvailabilityImage = counter_availability_image.data();
    getCounterAvailabilityParams.counterAvailabilityImageSize = counter_availability_image.size();
    cupti_res = cuptiProfilerGetCounterAvailability(&getCounterAvailabilityParams);
    if (cupti_res != CUPTI_SUCCESS) {
        std::cout << "Second call to cuptiProfilerGetCounterAvailability failed with error code: " << cupti_res << std::endl;
        exit(EXIT_FAILURE);
    }

    CUpti_Device_GetChipName_Params getChipNameParams {};
    getChipNameParams.structSize = CUpti_Device_GetChipName_Params_STRUCT_SIZE;
    getChipNameParams.deviceIndex = device_index;
    cupti_res = cuptiDeviceGetChipName(&getChipNameParams);
    if (cupti_res != CUPTI_SUCCESS) {
        std::cout << "Call to cuptiDeviceGetChipName failed with error code: " << cupti_res << std::endl;
        exit(EXIT_FAILURE);
    }
    const char *chip_name = getChipNameParams.pChipName;

    CUpti_Profiler_Host_Initialize_Params initializeParams {};
    initializeParams.structSize = CUpti_Profiler_Host_Initialize_Params_STRUCT_SIZE;
    initializeParams.profilerType = CUPTI_PROFILER_TYPE_RANGE_PROFILER;
    initializeParams.pChipName = chip_name;
    initializeParams.pCounterAvailabilityImage = counter_availability_image.data();
    #if CUDART_VERSION >= 13020
    // Only valid for PM sampling and can be set to NULL for CUPTI Range Profiler.
    initializeParams.pSinglePassMetricSetName = NULL;
    #endif
    cupti_res = cuptiProfilerHostInitialize(&initializeParams);
    if (cupti_res != CUPTI_SUCCESS) {
        std::cout << "Call to cuptiProfilerHostInitialize failed with error code: " << cupti_res << std::endl;
        exit(EXIT_FAILURE);
    }
    CUpti_Profiler_Host_Object *host_object= initializeParams.pHostObject;

    for (size_t metric_type = 0; metric_type < CUPTI_METRIC_TYPE__COUNT; metric_type++) {
        CUpti_Profiler_Host_GetBaseMetrics_Params getBaseMetricsParams {};
        getBaseMetricsParams.structSize = CUpti_Profiler_Host_GetBaseMetrics_Params_STRUCT_SIZE;
        getBaseMetricsParams.pHostObject = host_object;
        getBaseMetricsParams.metricType = (CUpti_MetricType) metric_type;
        cupti_res = cuptiProfilerHostGetBaseMetrics(&getBaseMetricsParams);
        if (cupti_res != CUPTI_SUCCESS) {
            std::cout << "Call to cuptiProfilerHostGetBaseMetrics failed with error code: " << cupti_res << std::endl;
            exit(EXIT_FAILURE);
        }
        const char **base_metric_names = getBaseMetricsParams.ppMetricNames;
        size_t number_of_base_metrics = getBaseMetricsParams.numMetrics;

        for (size_t base_metric_idx = 0; base_metric_idx < number_of_base_metrics; base_metric_idx++) {
            const char *base_metric_name = base_metric_names[base_metric_idx];
            CUpti_Profiler_Host_GetSubMetrics_Params getSubMetricsParams {};
            getSubMetricsParams.structSize = CUpti_Profiler_Host_GetSubMetrics_Params_STRUCT_SIZE;
            getSubMetricsParams.pHostObject = host_object;
            getSubMetricsParams.metricType = (CUpti_MetricType) metric_type;
            getSubMetricsParams.pMetricName = base_metric_name;
            cupti_res = cuptiProfilerHostGetSubMetrics(&getSubMetricsParams);
            if (cupti_res != CUPTI_SUCCESS) {
                std::cout << "Call to cuptiProfilerHostGetSubMetrics failed with error code: " << cupti_res << std::endl;
                exit(EXIT_FAILURE);
            }
        }
    }

    CUpti_Profiler_Host_Deinitialize_Params hostDeinitializeParams {};
    hostDeinitializeParams.structSize = CUpti_Profiler_Host_Deinitialize_Params_STRUCT_SIZE;
    hostDeinitializeParams.pHostObject = host_object;
    cupti_res = cuptiProfilerHostDeinitialize(&hostDeinitializeParams);
    if (cupti_res != CUPTI_SUCCESS) {
        std::cout << "Call to cuptiProfilerHostDeinitialize failed with error code: " << cupti_res << std::endl;
        exit(EXIT_FAILURE);
    }
    host_object = nullptr;

    cu_res = cuCtxDestroy(context);
    if (cu_res != CUDA_SUCCESS) {
        std::cout << "Call to cuCtxDestroy failed with error code: " << cu_res << std::endl;
        exit(EXIT_FAILURE);
    }

    return 0;
}

To compile the above workflow:

export CUDA_HOME=/path/to/ctk/root/dir
nvcc -I$CUDA_HOME/extras/CUPTI/include -I$CUDA_HOME/include your_filename.cu -o your_filename -L$CUDA_HOME/extras/CUPTI/lib64 -lcupti -lcuda

From further testing with Cuda Toolkit 12.9.0, I have found adding a rollup metric when applicable to the base metric name will resolve the error. Do you need more than just the base metric name in Cuda Toolkits < 13.0.0 to get the sub metrics?

2

Starting in CUDA 13.0, CUPTI simplifies CUpti_Profiler_Host_GetSubMetrics_Params by appending rollups and submetrics automatically. You only need to pass the base metric name to retrieve the supported submetrics. If you explicitly pass a rollup or submetric, CUPTI will still accept it, provided it is valid for the specified metric type.

For Pre-CUDA 13.0 versions:

You must explicitly provide the metric name along with its appropriate rollup or submetric, as well as the metric type in the input struct. You can consult the CUPTI Documentation on Enumerating Metrics to find the correct rollup/submetric combination for your metric type.