I’m trying to build an image using osu benchmark for my application with the code below.
##############################################################
# This Dockerfile contains the NVidia HPC SDK (nvc, cuda, OpenMPI) for Devito
##############################################################
ARG ver="nvhpc"
ARG pyversion=python:3.9
ARG arch="nvc"
########################################################################
# Build base image with apt setup and common env
########################################################################
FROM nvcr.io/nvidia/cuda:12.2.0-devel-ubuntu22.04 as sdk-base
# FROM docker.io/ubuntu:22.04 as sdk-base
# NVIDIA HPC SDK
RUN apt-get update -y && \
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
ca-certificates \
gnupg \
wget && \
rm -rf /var/lib/apt/lists/*
RUN echo "deb [trusted=yes] https://developer.download.nvidia.com/hpc-sdk/ubuntu/amd64 /" >> /etc/apt/sources.list.d/hpccm.list && \
apt-get update -y && \
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
nvhpc && \
rm -rf /var/lib/apt/lists/*
# MPI_VER options 3,4,HPCX
ARG MPI_VER=4
ENV MPIVER=${MPI_VER}
# nvidia-container-runtime
ENV NVIDIA_VISIBLE_DEVICES all
ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
# MPI ROOT USER DEFAULTS
ENV OMPI_ALLOW_RUN_AS_ROOT=1
ENV OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
ENV OMPI_MCA_rmaps_base_oversubscribe=1
ENV OMPI_MCA_btl_base_warn_component_unused=0
ENV OMPI_MCA_hwloc_base_binding_policy=""
ENV UCX_MEMTYPE_CACHE=no
ENV UCX_NET_DEVICES=all
ENV UCX_SHM_DEVICES=all
ENV UCX_ACC_DEVICES=all
ENV UCX_RNDV_THRESH=0
ENV UCX_RNDV_SCHEME=get_zcopy
ENV NCCL_UCX_RNDV_THRESH=0
ENV NCCL_UCX_RNDV_SCHEME=get_zcopy
ENV NCCL_PLUGIN_P2P=ucx
ENV MELLANOX_MOUNT_DRIVER=1
ENV UCX_TLS=cuda,cuda_copy,cuda_ipc,sm,shm,self
# For Baremetal, these flags are also available
#ENV UCX_TLS=cuda,cuda_copy,cuda_ipc,sm,shm,self,rc_x,gdr_copy
# Make simlink for path setup since ENV doesn't accept shell commands.
RUN export NVARCH=$(ls -1 /opt/nvidia/hpc_sdk/Linux_x86_64/ | grep '\.' | head -n 1) && \
export CUDA_V=$(ls /opt/nvidia/hpc_sdk/Linux_x86_64/${NVARCH}/cuda/ | grep '\.') && \
ln -sf /opt/nvidia/hpc_sdk/Linux_x86_64/${NVARCH} /opt/nvhpc && \
ln -sf /opt/nvidia/hpc_sdk/Linux_x86_64/${NVARCH}/cuda/${CUDA_V}/extras/CUPTI /opt/CUPTI && \
ln -sf /opt/nvidia/hpc_sdk/Linux_x86_64/comm_libs/${CUDA_V}/nvshmem /opt/nvhpc/comm_libs/nvshmem && \
ln -sf /opt/nvidia/hpc_sdk/Linux_x86_64/comm_libs/${CUDA_V}/nccl /opt/nvhpc/comm_libs/nccl
# Starting nvhpc 23.5 and cuda 12.1, hpcx and openmpi are inside the cuda version folder, only the bin is in the comm_libs path
RUN export CUDA_V=$(/opt/nvhpc/cuda/bin/nvcc --version | sed -n 's/^.*release \([0-9]\+\.[0-9]\+\).*$/\1/p') && \
ls /opt/nvhpc/comm_libs/${CUDA_V}/hpcx/ &&\
if [ -d /opt/nvhpc/comm_libs/${CUDA_V}/hpcx ]; then \
rm -rf /opt/nvhpc/comm_libs/hpcx && rm -rf /opt/nvhpc/comm_libs/openmpi4 && \
ln -sf /opt/nvhpc/comm_libs/${CUDA_V}/hpcx /opt/nvhpc/comm_libs/hpcx && \
ln -sf /opt/nvhpc/comm_libs/${CUDA_V}/openmpi4 /opt/nvhpc/comm_libs/openmpi4;\
fi;
# Set base path based on version
ENV HPCSDK_HOME=/opt/nvhpc
ENV HPCSDK_CUPTI=/opt/CUPTI
# required for nvidia-docker v1
RUN echo "$HPCSDK_HOME/cuda/lib" >> /etc/ld.so.conf.d/nvidia.conf && \
echo "$HPCSDK_HOME/cuda/lib64" >> /etc/ld.so.conf.d/nvidia.conf && \
echo "$HPCSDK_HOME/compilers/lib" >> /etc/ld.so.conf.d/nvidia.conf && \
echo "$HPCSDK_HOME/comm_libs/mpi/lib" >> /etc/ld.so.conf.d/nvidia.conf && \
echo "$HPCSDK_CUPTI/lib64" >> /etc/ld.so.conf.d/nvidia.conf && \
echo "$HPCSDK_HOME/math_libs/lib64" >> /etc/ld.so.conf.d/nvidia.conf
# Compiler, CUDA, and Library paths
# CUDA_HOME has been deprecated but keep for now because of other dependencies (@mloubout).
ENV CUDA_HOME $HPCSDK_HOME/cuda
ENV NVHPC_CUDA_HOME $HPCSDK_HOME/cuda
ENV CUDA_ROOT $HPCSDK_HOME/cuda/bin
ENV PATH $HPCSDK_HOME/compilers/bin:$HPCSDK_HOME/cuda/bin:$HPCSDK_HOME/comm_libs/mpi/bin:${PATH}
ENV LD_LIBRARY_PATH $HPCSDK_HOME/cuda/lib:$HPCSDK_HOME/cuda/lib64:$HPCSDK_HOME/compilers/lib:$HPCSDK_HOME/math_libs/lib64:$HPCSDK_HOME/comm_libs/mpi/lib:$HPCSDK_CUPTI/lib64:bitcomp_DIR:${LD_LIBRARY_PATH}
ENV CPATH $HPCSDK_HOME/comm_libs/mpi/include:$HPCSDK_HOME/comm_libs/nvshmem/include:$HPCSDK_HOME/comm_libs/nccl/include:${CPATH}
# MPI
RUN if [ "x$MPI_VER" = "x4" ]; then \
rm -f $HPCSDK_HOME/comm_libs/mpi && \
ln -sf $HPCSDK_HOME/comm_libs/openmpi4/latest \
$HPCSDK_HOME/comm_libs/mpi ; \
elif [ "$MPI_VER" = "HPCX" ]; then \
rm -f $HPCSDK_HOME/comm_libs/mpi && \
ln -sf $HPCSDK_HOME/comm_libs/hpcx/latest/ompi \
$HPCSDK_HOME/comm_libs/mpi ; \
fi;
RUN apt-get clean && apt-get autoclean && apt-get autoremove && \
rm -rf /var/lib/apt/lists/*
RUN ln -s /opt/nvhpc/cuda /usr/local/cuda
RUN apt-get update -y && \
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
git \
make \
wget && \
rm -rf /var/lib/apt/lists/*
# https://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-7.2.tar.gz
RUN mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp https://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-7.2.tar.gz && \
mkdir -p /var/tmp && tar -x -f /var/tmp/osu-micro-benchmarks-7.2.tar.gz -C /var/tmp -z && \
cd /var/tmp/osu-micro-benchmarks-7.2 && CC=mpicc CXX=mpicxx ./configure --prefix=/usr/local/osu --enable-cuda --with-cuda-include=/opt/nvidia/hpc_sdk/Linux_x86_64/23.7/cuda/include --with-cuda-libpath=/opt/nvidia/hpc_sdk/Linux_x86_64/23.7/cuda/lib64 && \
make -j$(nproc) && \
make -j$(nproc) install && \
rm -rf /var/tmp/osu-micro-benchmarks-7.2 /var/tmp/osu-micro-benchmarks-7.2.tar.gz
ENV PATH=/usr/local/osu/libexec/osu-micro-benchmarks:/usr/local/osu/libexec/osu-micro-benchmarks/mpi/collective:/usr/local/osu/libexec/osu-micro-benchmarks/mpi/one-sided:/usr/local/osu/libexec/osu-micro-benchmarks/mpi/pt2pt:/usr/local/osu/libexec/osu-micro-benchmarks/mpi/startup:$PATH
EXPOSE 8888
CMD ["/bin/bash"]
Using the Nvidia/CUDA image as a starting point i can build it without any problem, the thing is when i use an ubuntu:22.04 instead, something goes wrong and the Dockerfile can’t build the image, showing the error below.
=> ERROR [11/11] RUN mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp https://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-be 14.1s
------
> [11/11] RUN mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp https://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-7.2.tar.gz && mkdir -p /var/tmp && tar -x -f /var/tmp/osu-micro-benchmarks-7.2.tar.gz -C /var/tmp -z && cd /var/tmp/osu-micro-benchmarks-7.2 && CC=mpicc CXX=mpicxx ./configure --prefix=/usr/local/osu --enable-cuda --with-cuda-include=/opt/nvidia/hpc_sdk/Linux_x86_64/23.7/cuda/include --with-cuda-libpath=/opt/nvidia/hpc_sdk/Linux_x86_64/23.7/cuda/lib64 && make -j$(nproc) && make -j$(nproc) install && rm -rf /var/tmp/osu-micro-benchmarks-7.2 /var/tmp/osu-micro-benchmarks-7.2.tar.gz:
5.374 checking for a BSD-compatible install... /usr/bin/install -c
5.384 checking whether build environment is sane... yes
5.392 checking for a thread-safe mkdir -p... /usr/bin/mkdir -p
5.393 checking for gawk... no
5.394 checking for mawk... mawk
5.394 checking whether make sets $(MAKE)... yes
5.405 checking whether make supports nested variables... yes
5.418 checking build system type... x86_64-pc-linux-gnu
5.640 checking host system type... x86_64-pc-linux-gnu
5.640 checking how to print strings... printf
5.642 checking whether make supports the include directive... yes (GNU style)
5.650 checking for gcc... mpicc
5.811 checking whether the C compiler works... yes
6.319 checking for C compiler default output file name... a.out
6.320 checking for suffix of executables...
6.416 checking whether we are cross compiling... no
6.550 checking for suffix of object files... o
6.607 checking whether we are using the GNU C compiler... yes
6.660 checking whether mpicc accepts -g... yes
6.714 checking for mpicc option to accept ISO C89... none needed
6.993 checking whether mpicc understands -c and -o together... yes
7.110 checking dependency style of mpicc... tcc
7.442 checking for a sed that does not truncate output... /usr/bin/sed
7.446 checking for grep that handles long lines and -e... /usr/bin/grep
7.448 checking for egrep... /usr/bin/grep -E
7.449 checking for fgrep... /usr/bin/grep -F
7.451 checking for ld used by mpicc... /bin/ld
7.489 checking if the linker (/bin/ld) is GNU ld... yes
7.491 checking for BSD- or MS-compatible name lister (nm)... /usr/bin/nm -B
7.494 checking the name lister (/usr/bin/nm -B) interface... BSD nm
7.565 checking whether ln -s works... yes
7.565 checking the maximum length of command line arguments... 1572864
7.573 checking how to convert x86_64-pc-linux-gnu file names to x86_64-pc-linux-gnu format... func_convert_file_noop
7.573 checking how to convert x86_64-pc-linux-gnu file names to toolchain format... func_convert_file_noop
7.573 checking for /bin/ld option to reload object files... -r
7.573 checking for objdump... objdump
7.573 checking how to recognize dependent libraries... pass_all
7.574 checking for dlltool... no
7.574 checking how to associate runtime and link libraries... printf %s\n
7.574 checking for ar... ar
7.574 checking for archiver @FILE support... @
7.641 checking for strip... strip
7.641 checking for ranlib... ranlib
7.642 checking command to parse /usr/bin/nm -B output from mpicc object... ok
7.813 checking for sysroot... no
7.813 checking for a working dd... /usr/bin/dd
7.818 checking how to truncate binary pipes... /usr/bin/dd bs=4096 count=1
7.881 ./configure: line 6708: /usr/bin/file: No such file or directory
7.882 checking for mt... no
7.882 checking if : is a manifest tool... no
7.886 checking how to run the C preprocessor... mpicc -E
8.077 checking for ANSI C header files... yes
8.400 checking for sys/types.h... yes
8.481 checking for sys/stat.h... yes
8.560 checking for stdlib.h... yes
8.639 checking for string.h... yes
8.722 checking for memory.h... yes
8.803 checking for strings.h... yes
8.884 checking for inttypes.h... yes
8.966 checking for stdint.h... yes
9.049 checking for unistd.h... yes
9.137 checking for dlfcn.h... yes
9.220 checking for objdir... .libs
9.384 checking if mpicc supports -fno-rtti -fno-exceptions... no
9.428 checking for mpicc option to produce PIC... -fPIC -DPIC
9.428 checking if mpicc PIC flag -fPIC -DPIC works... yes
9.495 checking if mpicc static flag -static works... no
9.573 checking if mpicc supports -c -o file.o... yes
9.643 checking if mpicc supports -c -o file.o... (cached) yes
9.643 checking whether the mpicc linker (/bin/ld) supports shared libraries... yes
9.686 checking whether -lc should be explicitly linked in... no
9.811 checking dynamic linker characteristics... nvc-Error-Unknown switch: -print-search-dirs
9.996 GNU/Linux ld.so
9.996 checking how to hardcode library paths into programs... immediate
9.996 checking whether stripping libraries is possible... yes
9.998 checking if libtool supports shared libraries... yes
9.998 checking whether to build shared libraries... yes
9.998 checking whether to build static libraries... yes
9.999 checking for MPI_Session_init... no
10.27 checking whether we are using the GNU C compiler... (cached) yes
10.27 checking whether mpicc accepts -g... (cached) yes
10.27 checking for mpicc option to accept ISO C89... (cached) none needed
10.27 checking whether mpicc understands -c and -o together... (cached) yes
10.27 checking dependency style of mpicc... (cached) tcc
10.44 checking whether we are using the GNU C++ compiler... yes
10.50 checking whether mpicxx accepts -g... yes
10.56 checking dependency style of mpicxx... tcc
10.92 checking how to run the C++ preprocessor... mpicxx -E
11.32 checking for ld used by mpicxx... /bin/ld
11.36 checking if the linker (/bin/ld) is GNU ld... yes
11.41 nvc++-Error-Unknown switch: -print-prog-name=ld
11.41 checking whether the mpicxx linker (/bin/ld) supports shared libraries... yes
11.64 checking for mpicxx option to produce PIC... -fPIC -DPIC
11.64 checking if mpicxx PIC flag -fPIC -DPIC works... yes
11.71 checking if mpicxx static flag -static works... no
11.78 checking if mpicxx supports -c -o file.o... yes
11.85 checking if mpicxx supports -c -o file.o... (cached) yes
11.85 checking whether the mpicxx linker (/bin/ld) supports shared libraries... yes
11.85 checking dynamic linker characteristics... (cached) GNU/Linux ld.so
11.85 checking how to hardcode library paths into programs... immediate
11.86 checking for library containing sqrt... none required
11.96 checking for library containing pthread_join... none required
12.07 checking for library containing clock_gettime... none required
12.18 checking for stdlib.h... (cached) yes
12.19 checking for string.h... (cached) yes
12.19 checking sys/time.h usability... yes
12.28 checking sys/time.h presence... yes
12.33 checking for sys/time.h... yes
12.34 checking for unistd.h... (cached) yes
12.34 checking math.h usability... yes
12.44 checking math.h presence... yes
12.50 checking for math.h... yes
12.50 checking for inline... inline
12.57 checking for getpagesize... yes
12.69 checking for gettimeofday... yes
12.80 checking for memset... yes
12.92 checking for sqrt... yes
13.04 checking for MPI_Init... yes
13.15 checking for MPI_Accumulate... yes
13.26 checking for MPI_Get_accumulate... yes
13.37 checking for shmem_barrier_all... no
13.48 checking for upc_memput... no
13.61 checking whether upcxx_alltoall is declared... no
13.66 checking for shmem_finalize... no
13.79 checking for library containing cuPointerGetAttribute... no
13.99 configure: error: cannot link with -lcuda
------
Dockerfile.gpu:118
--------------------
117 | # https://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-7.2.tar.gz
118 | >>> RUN mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp https://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-7.2.tar.gz && \
119 | >>> mkdir -p /var/tmp && tar -x -f /var/tmp/osu-micro-benchmarks-7.2.tar.gz -C /var/tmp -z && \
120 | >>> cd /var/tmp/osu-micro-benchmarks-7.2 && CC=mpicc CXX=mpicxx ./configure --prefix=/usr/local/osu --enable-cuda --with-cuda-include=/opt/nvidia/hpc_sdk/Linux_x86_64/23.7/cuda/include --with-cuda-libpath=/opt/nvidia/hpc_sdk/Linux_x86_64/23.7/cuda/lib64 && \
121 | >>> make -j$(nproc) && \
122 | >>> make -j$(nproc) install && \
123 | >>> rm -rf /var/tmp/osu-micro-benchmarks-7.2 /var/tmp/osu-micro-benchmarks-7.2.tar.gz
124 |
--------------------
ERROR: failed to solve: process "/bin/sh -c mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp https://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-7.2.tar.gz && mkdir -p /var/tmp && tar -x -f /var/tmp/osu-micro-benchmarks-7.2.tar.gz -C /var/tmp -z && cd /var/tmp/osu-micro-benchmarks-7.2 && CC=mpicc CXX=mpicxx ./configure --prefix=/usr/local/osu --enable-cuda --with-cuda-include=/opt/nvidia/hpc_sdk/Linux_x86_64/23.7/cuda/include --with-cuda-libpath=/opt/nvidia/hpc_sdk/Linux_x86_64/23.7/cuda/lib64 && make -j$(nproc) && make -j$(nproc) install && rm -rf /var/tmp/osu-micro-benchmarks-7.2 /var/tmp/osu-micro-benchmarks-7.2.tar.gz" did not complete successfully: exit code: 1
I wonder why does this happen since the NVHPC installation has the CUDA library. Isn’t it enough to make it all work?
Can anyone shine a light on this issue for me? Has anyone ever faced this kind of problem before?