I am working on interfacing two libraries, PETSc and libCEED for computational fluid dynamics running for HPC. The problem is that libCEED dynaimcally creates the cuda back end and uses nvrtc to compile it at run time. The errors I am getting are:
default_program(372): error: identifier “PetscInt” is undefined
PetscInt dof;
^
default_program(376): error: identifier “PetscInt” is undefined
void (*riemann)(PetscInt, PetscInt, const PetscReal, const PetscReal, const PetscScalar, const PetscScalar, PetscInt, const PetscScalar, PetscScalar, void *);
^
default_program(376): error: identifier “PetscInt” is undefined
void (*riemann)(PetscInt, PetscInt, const PetscReal, const PetscReal, const PetscScalar, const PetscScalar, PetscInt, const PetscScalar, PetscScalar, void *);
^
default_program(376): error: identifier “PetscReal” is undefined
void (*riemann)(PetscInt, PetscInt, const PetscReal, const PetscReal, const PetscScalar, const PetscScalar, PetscInt, const PetscScalar, PetscScalar, void *);
^
default_program(376): error: identifier “PetscReal” is undefined
void (*riemann)(PetscInt, PetscInt, const PetscReal, const PetscReal, const PetscScalar, const PetscScalar, PetscInt, const PetscScalar, PetscScalar, void *);
^
default_program(376): error: identifier “PetscScalar” is undefined
void (*riemann)(PetscInt, PetscInt, const PetscReal, const PetscReal, const PetscScalar, const PetscScalar, PetscInt, const PetscScalar, PetscScalar, void *);
^
default_program(376): error: identifier “PetscScalar” is undefined
void (*riemann)(PetscInt, PetscInt, const PetscReal, const PetscReal, const PetscScalar, const PetscScalar, PetscInt, const PetscScalar, PetscScalar, void *);
^
default_program(376): error: identifier “PetscInt” is undefined
void (*riemann)(PetscInt, PetscInt, const PetscReal, const PetscReal, const PetscScalar, const PetscScalar, PetscInt, const PetscScalar, PetscScalar, void *);
^
default_program(376): error: identifier “PetscScalar” is undefined
void (*riemann)(PetscInt, PetscInt, const PetscReal, const PetscReal, const PetscScalar, const PetscScalar, PetscInt, const PetscScalar, PetscScalar, void *);
^
default_program(376): error: identifier “PetscScalar” is undefined
void (*riemann)(PetscInt, PetscInt, const PetscReal, const PetscReal, const PetscScalar, const PetscScalar, PetscInt, const PetscScalar, PetscScalar, void *);
^
default_program(377): error: identifier “PetscInt” is undefined
PetscInt dof; /* number of degrees of freedom per cell */
^
default_program(378): error: identifier “PetscReal” is undefined
PetscReal maxspeed; /* kludge to pick initial time step, need to add monitoring and step control */
^
default_program(380): error: identifier “PetscInt” is undefined
PetscInt nfields;
^
default_program(385): error: identifier “PetscReal” is undefined
PetscReal gravity;
That is just part of the error output and it overall it stops at 100.
The relevant code in ceed-cuda-compile.cpp:
// Copyright (c) 2017-2022, Lawrence Livermore National Security, LLC and other CEED contributors.
// All Rights Reserved. See the top-level LICENSE and NOTICE files for details.
//
// SPDX-License-Identifier: BSD-2-Clause
//
// This file is part of CEED: CEED · GitHub
include “ceed-cuda-compile.h”
include <ceed.h>
include <ceed/backend.h>
include <ceed/jit-tools.h>
include <cuda.h>
include <cuda_runtime.h>
include <nvrtc.h>
include <stdarg.h>
include <string.h>
include “ceed-cuda-common.h”
define CeedChk_Nvrtc(ceed, x)
do {
nvrtcResult result = static_cast(x);
if (result != NVRTC_SUCCESS) return CeedError((ceed), CEED_ERROR_BACKEND, nvrtcGetErrorString(result));
} while (0)
define CeedCallNvrtc(ceed, …)
do {
int ierr_q_ = VA_ARGS;
CeedChk_Nvrtc(ceed, ierr_q_);
} while (0)
//------------------------------------------------------------------------------
// Compile CUDA kernel
//------------------------------------------------------------------------------
int CeedCompile_Cuda(Ceed ceed, const char *source, CUmodule *module, const CeedInt num_defines, …) {
cudaFree(0); // Make sure a Context exists for nvrtc
nvrtcProgram prog;
std::ostringstream code;
// Get kernel specific options, such as kernel constants
if (num_defines > 0) {
va_list args;
va_start(args, num_defines);
char *name;
int val;
for (int i = 0; i < num_defines; i++) {
name = va_arg(args, char *);
val = va_arg(args, int);
code << "define " << name << " " << val << “\n”;
}
va_end(args);
}
// Standard libCEED definitions for CUDA backends
char *jit_defs_path, *jit_defs_source;
CeedCallBackend(CeedGetJitAbsolutePath(ceed, “ceed/jit-source/cuda/cuda-jit.h”, &jit_defs_path));
CeedCallBackend(CeedLoadSourceToBuffer(ceed, jit_defs_path, &jit_defs_source));
code << jit_defs_source;
code << “\n\n”;
CeedCallBackend(CeedFree(&jit_defs_path));
CeedCallBackend(CeedFree(&jit_defs_source));
// Non-macro options
const int num_opts = 9;
const char *opts[num_opts];
opts[0] = “-default-device”;
struct cudaDeviceProp prop;
Ceed_Cuda *ceed_data;
CeedCallBackend(CeedGetData(ceed, &ceed_data));
CeedCallCuda(ceed, cudaGetDeviceProperties(&prop, ceed_data->device_id));
std::string arch_arg = “-arch=compute_” + std::to_string(prop.major) + std::to_string(prop.minor);
opts[1] = arch_arg.c_str();
opts[2] = “-Dint32_t=int”;
opts[3] = “–include-path=/home/arcowie/petsc/include”;
opts[4] = “–include-path=/home/arcowie/petsc/include/petsc”;
opts[5] = “–include-path=/home/arcowie/petsc/include/petsc/finclude”;
opts[6] = “–include-path=/home/arcowie/petsc/arch-linux-rdycore-debug/include”;
opts[7] = “–include-path=/home/arcowie/petsc/arch-linux-rdycore-debug/include/eigen3”;
opts[8] = “–include-path=/usr/local/cuda-12.1/include”;
// Add string source argument provided in call
code << source;
// Create Program
CeedCallNvrtc(ceed, nvrtcCreateProgram(&prog, code.str().c_str(), NULL, 0, NULL, NULL));
// Compile kernel
nvrtcResult result = nvrtcCompileProgram(prog, num_opts, opts);
if (result != NVRTC_SUCCESS) {
size_t log_size;
CeedCallNvrtc(ceed, nvrtcGetProgramLogSize(prog, &log_size));
char *log;
CeedCallBackend(CeedMalloc(log_size, &log));
CeedCallNvrtc(ceed, nvrtcGetProgramLog(prog, log));
return CeedError(ceed, CEED_ERROR_BACKEND, “%s\n%s”, nvrtcGetErrorString(result), log);
}
size_t ptx_size;
CeedCallNvrtc(ceed, nvrtcGetPTXSize(prog, &ptx_size));
char *ptx;
CeedCallBackend(CeedMalloc(ptx_size, &ptx));
CeedCallNvrtc(ceed, nvrtcGetPTX(prog, ptx));
CeedCallNvrtc(ceed, nvrtcDestroyProgram(&prog));
and some of the source code:
include <petscdm.h>
include <petscdmceed.h>
define DIM 2 /* Geometric dimension */
/* Represents continuum physical equations. */
typedef struct _n_Physics *Physics;
/* Physical model includes boundary conditions, initial conditions, and functionals of interest. It is
- discretization-independent, but its members depend on the scenario being solved. */
typedef struct _n_Model *Model;
struct FieldDescription {
const char *name;
PetscInt dof;
};
struct _n_Physics {
void (*riemann)(PetscInt, PetscInt, const PetscReal, const PetscReal, const PetscScalar, const PetscScalar, PetscInt, const PetscScalar, PetscScalar, void );
PetscInt dof; / number of degrees of freedom per cell /
PetscReal maxspeed; / kludge to pick initial time step, need to add monitoring and step control */
void *data;
PetscInt nfields;
const struct FieldDescription *field_desc;
};
typedef struct {
PetscReal gravity;
struct {
PetscInt Height;
PetscInt Speed;
PetscInt Energy;
} functional;
} Physics_SW;
typedef struct {
PetscReal h;
PetscReal uh[DIM];
} SWNode;
typedef union
{
SWNode swnode;
PetscReal vals[DIM + 1];
} SWNodeUnion;
typedef enum {
EULER_IV_SHOCK,
EULER_SS_SHOCK,
EULER_SHOCK_TUBE,
EULER_LINEAR_WAVE
} EulerType;
typedef struct {
PetscReal gamma;
PetscReal rhoR;
PetscReal amach;
PetscReal itana;
EulerType type;
struct {
PetscInt Density;
PetscInt Momentum;
PetscInt Energy;
PetscInt Pressure;
PetscInt Speed;
} monitor;
} Physics_Euler;
typedef struct {
PetscReal r;
PetscReal ru[DIM];
PetscReal E;
} EulerNode;
typedef union
{
EulerNode eulernode;
PetscReal vals[DIM + 2];
} EulerNodeUnion;
static inline PetscReal Dot2Real(const PetscReal *x, const PetscReal *y)
{
return x[0] * y[0] + x[1] * y[1];
}
static inline PetscReal Norm2Real(const PetscReal *x)
{
return PetscSqrtReal(PetscAbsReal(Dot2Real(x, x)));
}
static inline void Normalize2Real(PetscReal *x)
{
PetscReal a = 1. / Norm2Real(x);
x[0] *= a;
x[1] *= a;
}
static inline void Scale2Real(PetscReal a, const PetscReal *x, PetscReal *y)
{
y[0] = a * x[0];
y[1] = a * x[1];
}
static inline PetscReal DotDIMReal(const PetscReal *x, const PetscReal *y)
{
PetscInt i;
PetscReal prod = 0.0;
for (i = 0; i < DIM; i++) prod += x[i] * y[i];
return prod;
}
static inline PetscReal NormDIM(const PetscReal *x)
{
return PetscSqrtReal(PetscAbsReal(DotDIMReal(x, x)));
}
static inline void Waxpy2Real(PetscReal a, const PetscReal *x, const PetscReal *y, PetscReal *w)
{
w[0] = a * x[0] + y[0];
w[1] = a * x[1] + y[1];
}
NVIDIA info:
Nnvcc: NVIDIA (R) Cuda compiler driver
Copyright (c) 2005-2023 NVIDIA Corporation
Built on Mon_Apr__3_17:16:06_PDT_2023
Cuda compilation tools, release 12.1.105
Graphics Card:
NVIDIA GeForce RTX 3070 Laptop GPU
OS:
PRETTY_NAME=“Ubuntu 22.04.3 LTS”
NAME=“Ubuntu”
VERSION_ID=“22.04”
VERSION=“22.04.3 LTS (Jammy Jellyfish)”
Thanks,
Al