High Performance Linpack HPL with NVHPC 24.1 compilers for ppi64le in Almalinux 9.2

francesco.iannone · June 20, 2024, 7:04am

Hi
I have a problem of “undefined reference” to BLAS functions when built the HPL (netlib) in a compute node with Power9 cpu, Almalinux 9.2 and NVHPC 24.1.
It looks like the linker doesn’t resolve BLAS symbols , I tried different BLAS implementations: NVHPC, Almalinux RPM, OpenBLAS RPM, Atlas RPM. The error is as below . I used also compiler/linker options like “-fortranlibs” and others without successful.
HPL is built without problem using GNU/openMPI compilers.

mpif90 -Mnomain -DAdd_ -DF77_INTEGER=int /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/lib/PPI64/libhpl.a -L /afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none//Linux_ppc64le/24.1/comm_libs/12.3/openmpi4/latest/lib -lmpi -L/afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none/Linux_ppc64le/24.1/compilers/lib/ -lblas -L/usr/lib64 -lpthread -I/afs/enea.it/fra/user/iannone/private/src/hpl-2.3/include -I/afs/enea.it/fra/user/iannone/private/src/hpl-2.3/include/PPI64 -I /afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none/Linux_ppc64le/24.1//math_libs/include/ -I /afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none/Linux_ppc64le/24.1//cuda/include -I /afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none//Linux_ppc64le/24.1/comm_libs/12.3/openmpi4/latest/lib/include /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/lib/PPI64/libhpl.a -L /afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none//Linux_ppc64le/24.1/comm_libs/12.3/openmpi4/latest/lib -lmpi -L/afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none/Linux_ppc64le/24.1/compilers/lib/ -lblas -L/usr/lib64 -lpthread -g -mp -fPIC -fast -o /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/bin/PPI64/xhpl HPL_pddriver.o HPL_pdinfo.o HPL_pdtest.o /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/lib/PPI64/libhpl.a -L /afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none//Linux_ppc64le/24.1/comm_libs/12.3/openmpi4/latest/lib -lmpi -L/afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none/Linux_ppc64le/24.1/compilers/lib/ -lblas -L/usr/lib64 -lpthread
/usr/bin/ld: /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/lib/PPI64/libhpl.a(HPL_idamax.o): in function HPL_idamax(int, double const*, int)': /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/src/blas/PPI64/../HPL_idamax.c:159: undefined reference to idamax_(int const*, double const*, int const*)’
/usr/bin/ld: /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/lib/PPI64/libhpl.a(HPL_dgemv.o): in function HPL_dgemv(HPL_ORDER, HPL_TRANS, int, int, double, double const*, int, double const*, int, double, double*, int)': /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/src/blas/PPI64/../HPL_dgemv.c:(.text+0xb4): undefined reference to dgemv_(char*, int const*, int const*, double const*, double const*, int const*, double const*, int const*, double const*, double*, int const*, int)’
/usr/bin/ld: /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/lib/PPI64/libhpl.a(HPL_dcopy.o): in function HPL_dcopy(int, double const*, int, double*, int)': /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/src/blas/PPI64/../HPL_dcopy.c:161: undefined reference to dcopy_(int const*, double const*, int const*, double*, int const*)’
/usr/bin/ld: /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/lib/PPI64/libhpl.a(HPL_daxpy.o): in function HPL_daxpy(int, double, double const*, int, double*, int)': /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/src/blas/PPI64/../HPL_daxpy.c:168: undefined reference to daxpy_(int const*, double const*, double const*, int const*, double*, int const*)’
/usr/bin/ld: /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/lib/PPI64/libhpl.a(HPL_dscal.o): in function HPL_dscal(int, double, double*, int)': /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/src/blas/PPI64/../HPL_dscal.c:172: undefined reference to dscal_(int const*, double const*, double*, int const*)’
/usr/bin/ld: /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/lib/PPI64/libhpl.a(HPL_dtrsv.o): in function HPL_dtrsv(HPL_ORDER, HPL_UPLO, HPL_TRANS, HPL_DIAG, int, double const*, int, double*, int)': /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/src/blas/PPI64/../HPL_dtrsv.c:495: undefined reference to dtrsv_(char*, char*, char*, int const*, double const*, int const*, double*, int const*, int, int, int)’
/usr/bin/ld: /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/lib/PPI64/libhpl.a(HPL_dger.o): in function HPL_dger(HPL_ORDER, int, int, double, double const*, int, double*, int, double*, int)': /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/src/blas/PPI64/../HPL_dger.c:(.text+0x98): undefined reference to dger_(int const*, int const*, double const*, double const*, int const*, double const*, int const*, double*, int const*)’
/usr/bin/ld: /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/lib/PPI64/libhpl.a(HPL_dgemm.o): in function HPL_dgemm(HPL_ORDER, HPL_TRANS, HPL_TRANS, int, int, int, double, double const*, int, double const*, int, double, double*, int)': /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/src/blas/PPI64/../HPL_dgemm.c:(.text+0x118): undefined reference to dgemm_(char*, char*, int const*, int const*, int const*, double const*, double const*, int const*, double const*, int const*, double const*, double*, int const*, int, int)’
/usr/bin/ld: /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/lib/PPI64/libhpl.a(HPL_dtrsm.o): in function HPL_dtrsm(HPL_ORDER, HPL_SIDE, HPL_UPLO, HPL_TRANS, HPL_DIAG, int, int, double, double const*, int, double*, int)': /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/src/blas/PPI64/../HPL_dtrsm.c:(.text+0x114): undefined reference to dtrsm_(char*, char*, char*, char*, int const*, int const*, double const*, double const*, int const*, double*, int const*, int, int, int, int)’
pgacclnk: child process exit status 1: /usr/bin/ld
make[2]: *** [Makefile:76: dexe.grd] Error 2

MatColgrove · June 20, 2024, 5:20pm

It might be a link order issue. Try moving “-lblas” before “libhpl.a”.

Note it looks like you included all the libraries twice which might give multiply defined reference error depending on how the libraries were built. After the undefined reference errors, if you then start seeing these errors, remove the duplicates.

francesco.iannone · June 20, 2024, 5:51pm

Hi Mat

Unfortunately I have had the some problems:
mpicc -o HPL_pddriver.o -c -DAdd_ -DF77_INTEGER=int -I/afs/enea.it/fra/user/iannone/private/src/hpl-2.3/include -I/afs/enea.it/fra/user/iannone/private/src/hpl-2.3/include/PPI64 -I /afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none/Linux_ppc64le/24.1//math_libs/include/ -I /afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none/Linux_ppc64le/24.1//cuda/include -I /afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none//Linux_ppc64le/24.1/comm_libs/12.3/openmpi4/latest/lib/include -g -mp -fPIC -fast …/HPL_pddriver.c
mpicc -o HPL_pdinfo.o -c -DAdd_ -DF77_INTEGER=int -I/afs/enea.it/fra/user/iannone/private/src/hpl-2.3/include -I/afs/enea.it/fra/user/iannone/private/src/hpl-2.3/include/PPI64 -I /afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none/Linux_ppc64le/24.1//math_libs/include/ -I /afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none/Linux_ppc64le/24.1//cuda/include -I /afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none//Linux_ppc64le/24.1/comm_libs/12.3/openmpi4/latest/lib/include -g -mp -fPIC -fast …/HPL_pdinfo.c
mpicc -o HPL_pdtest.o -c -DAdd_ -DF77_INTEGER=int -I/afs/enea.it/fra/user/iannone/private/src/hpl-2.3/include -I/afs/enea.it/fra/user/iannone/private/src/hpl-2.3/include/PPI64 -I /afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none/Linux_ppc64le/24.1//math_libs/include/ -I /afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none/Linux_ppc64le/24.1//cuda/include -I /afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none//Linux_ppc64le/24.1/comm_libs/12.3/openmpi4/latest/lib/include -g -mp -fPIC -fast …/HPL_pdtest.c
mpif90 -Mnomain -o /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/bin/PPI64/xhpl HPL_pddriver.o HPL_pdinfo.o HPL_pdtest.o -L/afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none/Linux_ppc64le/24.1/compilers/lib/ -lblas -L/usr/lib64 -lpthread -L /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/lib/PPI64 -lhpl -L /afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none//Linux_ppc64le/24.1/comm_libs/12.3/openmpi4/latest/lib -lmpi
/usr/bin/ld: /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/lib/PPI64/libhpl.a(HPL_idamax.o): in function HPL_idamax(int, double const*, int)': /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/src/blas/PPI64/../HPL_idamax.c:159: undefined reference to idamax_(int const*, double const*, int const*)’
/usr/bin/ld: /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/lib/PPI64/libhpl.a(HPL_dgemv.o): in function HPL_dgemv(HPL_ORDER, HPL_TRANS, int, int, double, double const*, int, double const*, int, double, double*, int)': /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/src/blas/PPI64/../HPL_dgemv.c:(.text+0xb4): undefined reference to dgemv_(char*, int const*, int const*, double const*, double const*, int const*, double const*, int const*, double const*, double*, int const*, int)’
/usr/bin/ld: /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/lib/PPI64/libhpl.a(HPL_dcopy.o): in function HPL_dcopy(int, double const*, int, double*, int)': /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/src/blas/PPI64/../HPL_dcopy.c:161: undefined reference to dcopy_(int const*, double const*, int const*, double*, int const*)’
/usr/bin/ld: /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/lib/PPI64/libhpl.a(HPL_daxpy.o): in function HPL_daxpy(int, double, double const*, int, double*, int)': /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/src/blas/PPI64/../HPL_daxpy.c:168: undefined reference to daxpy_(int const*, double const*, double const*, int const*, double*, int const*)’
/usr/bin/ld: /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/lib/PPI64/libhpl.a(HPL_dscal.o): in function HPL_dscal(int, double, double*, int)': /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/src/blas/PPI64/../HPL_dscal.c:172: undefined reference to dscal_(int const*, double const*, double*, int const*)’
/usr/bin/ld: /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/lib/PPI64/libhpl.a(HPL_dtrsv.o): in function HPL_dtrsv(HPL_ORDER, HPL_UPLO, HPL_TRANS, HPL_DIAG, int, double const*, int, double*, int)': /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/src/blas/PPI64/../HPL_dtrsv.c:495: undefined reference to dtrsv_(char*, char*, char*, int const*, double const*, int const*, double*, int const*, int, int, int)’
/usr/bin/ld: /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/lib/PPI64/libhpl.a(HPL_dger.o): in function HPL_dger(HPL_ORDER, int, int, double, double const*, int, double*, int, double*, int)': /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/src/blas/PPI64/../HPL_dger.c:(.text+0x98): undefined reference to dger_(int const*, int const*, double const*, double const*, int const*, double const*, int const*, double*, int const*)’
/usr/bin/ld: /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/lib/PPI64/libhpl.a(HPL_dgemm.o): in function HPL_dgemm(HPL_ORDER, HPL_TRANS, HPL_TRANS, int, int, int, double, double const*, int, double const*, int, double, double*, int)': /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/src/blas/PPI64/../HPL_dgemm.c:(.text+0x118): undefined reference to dgemm_(char*, char*, int const*, int const*, int const*, double const*, double const*, int const*, double const*, int const*, double const*, double*, int const*, int, int)’
/usr/bin/ld: /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/lib/PPI64/libhpl.a(HPL_dtrsm.o): in function HPL_dtrsm(HPL_ORDER, HPL_SIDE, HPL_UPLO, HPL_TRANS, HPL_DIAG, int, int, double, double const*, int, double*, int)': /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/src/blas/PPI64/../HPL_dtrsm.c:(.text+0x114): undefined reference to dtrsm_(char*, char*, char*, char*, int const*, int const*, double const*, double const*, int const*, double*, int const*, int, int, int, int)’
make[2]: *** [Makefile:76: dexe.grd] Error 2
make[2]: Leaving directory ‘/afs/enea.it/fra/user/iannone/private/src/hpl-2.3/testing/ptest/PPI64’
make[1]: *** [Make.top:68: build_tst] Error 2
make[1]: Leaving directory ‘/afs/enea.it/fra/user/iannone/private/src/hpl-2.3’
make: *** [Makefile:73: build] Error 2

MatColgrove · June 21, 2024, 5:00pm

Hmm, the symbols are in the blas library (see below) and why I suspected a link order issue. Though this is just a guess. Since we dropped support for Power systems, I unfortunately do have access to a system which I can test myself.

Let’s assume my hunch is correct, but it was the multiple uses of the libraries and using “libhpl.a” vs “-lhpl” that was causing the problem. In this case, try adding “-lblas” after “-lhpl”.

If that doesn’t work, try adding “-v” (verbose) to the link like and post the command line from the link (i.e. the “ld” command line). This might give us some additional clues. Maybe the MPI driver is moving the order?

FYI, here you can see that the “idamax_” is defined in the BLAS library:

Linux_ppc64le/24.1/compilers/lib% nm libblas.a | grep idamax
idamax_k.o:
0000000000000008 T idamax_k
cblas_idamax.o:
0000000000000008 T cblas_idamax
                 U idamax_k
idamax.o:
0000000000000008 T idamax_
                 U idamax_k
                 U idamax_k

francesco.iannone · June 21, 2024, 5:45pm

Thanks Mat
NVIDIA dropped support for ppc64le is not good news.
However I did two link cases:

THE FIRST ONE:

<iannone@xcrescox001 ~/private/src/hpl-2.3/testing/ptest/PPI64> mpif90 -Mnomain -L/afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none/Linux_ppc64le/24.1/compilers/lib/ -lblas -v -o /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/bin/PPI64/xhpl HPL_pddriver.o HPL_pdinfo.o HPL_pdtest.o -L/afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none/Linux_ppc64le/24.1/compilers/lib/ -lblas -L/usr/lib64 -lpthread -L /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/lib/PPI64 -lhpl -L /afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none//Linux_ppc64le/24.1/comm_libs/12.3/openmpi4/latest/lib -lmpi

Export NVCOMPILER=/afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none/Linux_ppc64le/24.1

Export PGI=/afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none

/usr/bin/ld /usr/lib64/crt1.o /usr/lib64/crti.o /usr/lib/gcc/ppc64le-redhat-linux/11//crtbegin.o --eh-frame-hdr -dynamic-linker /lib64/ld64.so.2 -T /afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none/Linux_ppc64le/24.1/compilers/lib/nvhpc.ld --no-tls-optimize -L/afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none/Linux_ppc64le/24.1/compilers/lib/ -L/afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none/Linux_ppc64le/24.1/compilers/lib/ -L/usr/lib64 -L/afs/enea.it/fra/user/iannone/private/src/hpl-2.3/lib/PPI64 -L/afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none//Linux_ppc64le/24.1/comm_libs/12.3/openmpi4/latest/lib -L/afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none//Linux_ppc64le/24.1/comm_libs/12.3/openmpi4/latest/lib -L/afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none//Linux_ppc64le/24.1/comm_libs/12.3/openmpi4/latest/lib -L/afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none//Linux_ppc64le/24.1/cuda/lib64 -L/afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none/Linux_ppc64le/24.1/compilers/lib -L/usr/lib64 -L/usr/lib/gcc/ppc64le-redhat-linux/11/ -lblas HPL_pddriver.o HPL_pdinfo.o HPL_pdtest.o -lblas -lpthread -lhpl -lmpi -rpath /afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none//Linux_ppc64le/24.1/comm_libs/12.3/openmpi4/latest/lib -lmpi_usempif08 -lmpi_usempi_ignore_tkr -lmpi_mpifh -lmpi -rpath /afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none/Linux_ppc64le/24.1/compilers/lib -o /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/bin/PPI64/xhpl -L/usr/lib/gcc/ppc64le-redhat-linux/11//…/…/…/…/lib64 -lnvf -lnvomp -ldl --as-needed -latomic -lnvhpcatm --no-as-needed -ldl -lpthread -lnvcpumath -lnvc -lrt -lpthread -lgcc --as-needed -lgcc_s --no-as-needed -lc -lgcc --as-needed -lgcc_s --no-as-needed -lm /usr/lib/gcc/ppc64le-redhat-linux/11//crtend.o /usr/lib64/crtn.o

/usr/bin/ld: /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/lib/PPI64/libhpl.a(HPL_idamax.o): in function `HPL_idamax(int, double const*, int)':

/afs/enea.it/fra/user/iannone/private/src/hpl-2.3/src/blas/PPI64/…/HPL_idamax.c:159: undefined reference to `idamax_(int const*, double const*, int const*)’

/usr/bin/ld: /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/lib/PPI64/libhpl.a(HPL_dgemv.o): in function `HPL_dgemv(HPL_ORDER, HPL_TRANS, int, int, double, double const*, int, double const*, int, double, double*, int)':

/afs/enea.it/fra/user/iannone/private/src/hpl-2.3/src/blas/PPI64/…/HPL_dgemv.c:(.text+0xb4): undefined reference to `dgemv_(char*, int const*, int const*, double const*, double const*, int const*, double const*, int const*, double const*, double*, int const*, int)’

/usr/bin/ld: /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/lib/PPI64/libhpl.a(HPL_dcopy.o): in function `HPL_dcopy(int, double const*, int, double*, int)':

/afs/enea.it/fra/user/iannone/private/src/hpl-2.3/src/blas/PPI64/…/HPL_dcopy.c:161: undefined reference to `dcopy_(int const*, double const*, int const*, double*, int const*)’

/usr/bin/ld: /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/lib/PPI64/libhpl.a(HPL_daxpy.o): in function `HPL_daxpy(int, double, double const*, int, double*, int)':

/afs/enea.it/fra/user/iannone/private/src/hpl-2.3/src/blas/PPI64/…/HPL_daxpy.c:168: undefined reference to `daxpy_(int const*, double const*, double const*, int const*, double*, int const*)’

/usr/bin/ld: /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/lib/PPI64/libhpl.a(HPL_dscal.o): in function `HPL_dscal(int, double, double*, int)':

/afs/enea.it/fra/user/iannone/private/src/hpl-2.3/src/blas/PPI64/…/HPL_dscal.c:172: undefined reference to `dscal_(int const*, double const*, double*, int const*)’

/usr/bin/ld: /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/lib/PPI64/libhpl.a(HPL_dtrsv.o): in function `HPL_dtrsv(HPL_ORDER, HPL_UPLO, HPL_TRANS, HPL_DIAG, int, double const*, int, double*, int)':

/afs/enea.it/fra/user/iannone/private/src/hpl-2.3/src/blas/PPI64/…/HPL_dtrsv.c:495: undefined reference to `dtrsv_(char*, char*, char*, int const*, double const*, int const*, double*, int const*, int, int, int)’

/usr/bin/ld: /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/lib/PPI64/libhpl.a(HPL_dger.o): in function `HPL_dger(HPL_ORDER, int, int, double, double const*, int, double*, int, double*, int)':

/afs/enea.it/fra/user/iannone/private/src/hpl-2.3/src/blas/PPI64/…/HPL_dger.c:(.text+0x98): undefined reference to `dger_(int const*, int const*, double const*, double const*, int const*, double const*, int const*, double*, int const*)’

/usr/bin/ld: /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/lib/PPI64/libhpl.a(HPL_dgemm.o): in function `HPL_dgemm(HPL_ORDER, HPL_TRANS, HPL_TRANS, int, int, int, double, double const*, int, double const*, int, double, double*, int)':

/afs/enea.it/fra/user/iannone/private/src/hpl-2.3/src/blas/PPI64/…/HPL_dgemm.c:(.text+0x118): undefined reference to `dgemm_(char*, char*, int const*, int const*, int const*, double const*, double const*, int const*, double const*, int const*, double const*, double*, int const*, int, int)’

/usr/bin/ld: /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/lib/PPI64/libhpl.a(HPL_dtrsm.o): in function `HPL_dtrsm(HPL_ORDER, HPL_SIDE, HPL_UPLO, HPL_TRANS, HPL_DIAG, int, int, double, double const*, int, double*, int)':

/afs/enea.it/fra/user/iannone/private/src/hpl-2.3/src/blas/PPI64/…/HPL_dtrsm.c:(.text+0x114): undefined reference to `dtrsm_(char*, char*, char*, char*, int const*, int const*, double const*, double const*, int const*, double*, int const*, int, int, int, int)’

nvfortran-Fatal-linker completed with exit code 1

THE SECOND ONE:

<iannone@xcrescox001 ~/private/src/hpl-2.3/testing/ptest/PPI64> mpif90 -Mnomain -L/afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none/Linux_ppc64le/24.1/compilers/lib/ -lblas -v -o /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/bin/PPI64/xhpl HPL_pddriver.o HPL_pdinfo.o HPL_pdtest.o -L/afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none/Linux_ppc64le/24.1/compilers/lib/ -lblas -L/usr/lib64 -lpthread -L /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/lib/PPI64 -lhpl -L /afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none//Linux_ppc64le/24.1/comm_libs/12.3/openmpi4/latest/lib -lmpi -L/afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none/Linux_ppc64le/24.1/compilers/lib/ -lblas

Export NVCOMPILER=/afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none/Linux_ppc64le/24.1

Export PGI=/afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none

/usr/bin/ld /usr/lib64/crt1.o /usr/lib64/crti.o /usr/lib/gcc/ppc64le-redhat-linux/11//crtbegin.o --eh-frame-hdr -dynamic-linker /lib64/ld64.so.2 -T /afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none/Linux_ppc64le/24.1/compilers/lib/nvhpc.ld --no-tls-optimize -L/afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none/Linux_ppc64le/24.1/compilers/lib/ -L/afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none/Linux_ppc64le/24.1/compilers/lib/ -L/usr/lib64 -L/afs/enea.it/fra/user/iannone/private/src/hpl-2.3/lib/PPI64 -L/afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none//Linux_ppc64le/24.1/comm_libs/12.3/openmpi4/latest/lib -L/afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none/Linux_ppc64le/24.1/compilers/lib/ -L/afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none//Linux_ppc64le/24.1/comm_libs/12.3/openmpi4/latest/lib -L/afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none//Linux_ppc64le/24.1/comm_libs/12.3/openmpi4/latest/lib -L/afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none//Linux_ppc64le/24.1/cuda/lib64 -L/afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none/Linux_ppc64le/24.1/compilers/lib -L/usr/lib64 -L/usr/lib/gcc/ppc64le-redhat-linux/11/ -lblas HPL_pddriver.o HPL_pdinfo.o HPL_pdtest.o -lblas -lpthread -lhpl -lmpi -lblas -rpath /afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none//Linux_ppc64le/24.1/comm_libs/12.3/openmpi4/latest/lib -lmpi_usempif08 -lmpi_usempi_ignore_tkr -lmpi_mpifh -lmpi -rpath /afs/.enea.it/software/xcresco/cineca/prod/opt/compilers/cuda/12.3/none/Linux_ppc64le/24.1/compilers/lib -o /afs/enea.it/fra/user/iannone/private/src/hpl-2.3/bin/PPI64/xhpl -L/usr/lib/gcc/ppc64le-redhat-linux/11//…/…/…/…/lib64 -lnvf -lnvomp -ldl --as-needed -latomic -lnvhpcatm --no-as-needed -ldl -lpthread -lnvcpumath -lnvc -lrt -lpthread -lgcc --as-needed -lgcc_s --no-as-needed -lc -lgcc --as-needed -lgcc_s --no-as-needed -lm /usr/lib/gcc/ppc64le-redhat-linux/11//crtend.o /usr/lib64/crtn.o