I’m trying to view overlapping behavior in kernel launches and memcpy for streams on my L40S gpu server running Ubuntu 22.04. For some reason, I fail to see any overlap in any of the memcpy/kernels:
Both the kernels and the kernel launches end up being sequentially ordered. My memcpy is also rather small (4Kb).
Some details of my device properties
can map host memory to cuda addresses : Yes
max threads per block : 1024
concurrent kernels supported : Yes
Overlap allowed in kernel and memcopy : Yes
Compute mode : 0
Async engines : 2
The code is compiled with --default-stream per-thread for sm_89
#include <cuda.h>
#include <cuda_runtime.h>
#include <cassert>
#include <cstdio>
__global__ void computeSquares(int *deviceArray, int *resultArray);
__global__ void computeCubes(int *deviceArray, int *resultArray);
int main() {
// Allocate page locked memory on the host
const int ArraySize = 1024;
const int BlockSize = 128;
const int GridSize = ArraySize / BlockSize;
const int ArraySizeInBytes = ArraySize * sizeof(int);
int *hostArray;
cudaMallocHost(&hostArray, ArraySizeInBytes);
// Allocate memory for the results
int *squares, *cubes;
cudaMallocHost(&squares, ArraySizeInBytes);
cudaMallocHost(&cubes, ArraySizeInBytes);
// Initialize array elements to 10
for (int i = 0; i < ArraySize; i++) {
hostArray[i] = 10;
}
// Create a stream for each operation
cudaStream_t squareStream, cubeStream;
cudaStreamCreateWithFlags(&squareStream, cudaStreamNonBlocking);
cudaStreamCreateWithFlags(&cubeStream, cudaStreamNonBlocking);
// Copy the array to the device
int *inputArrSquares;
cudaMallocAsync(&inputArrSquares, ArraySizeInBytes, squareStream);
cudaMemcpyAsync(inputArrSquares, hostArray, ArraySizeInBytes,
cudaMemcpyHostToDevice, squareStream);
int *inputArrCubes;
cudaMallocAsync(&inputArrCubes, ArraySizeInBytes, cubeStream);
cudaMemcpyAsync(inputArrCubes, hostArray, ArraySizeInBytes,
cudaMemcpyHostToDevice, cubeStream);
// Allocate memory on the device to store the results
int *devSquaredArray;
cudaMallocAsync(&devSquaredArray, ArraySizeInBytes, squareStream);
int *devCubedArray;
cudaMallocAsync(&devCubedArray, ArraySizeInBytes, cubeStream);
// Launch kernel
computeSquares<<<GridSize, BlockSize, 0, squareStream>>>(inputArrSquares,
devSquaredArray);
computeCubes<<<GridSize, BlockSize, 0, cubeStream>>>(inputArrCubes,
devCubedArray);
// Copy the result back to the host
cudaMemcpyAsync(squares, devSquaredArray, ArraySizeInBytes,
cudaMemcpyDeviceToHost, squareStream);
cudaMemcpyAsync(cubes, devCubedArray, ArraySizeInBytes,
cudaMemcpyDeviceToHost, cubeStream);
// Synchronize ALL work on the device
cudaDeviceSynchronize();
// Sanity check
for (int i = 0; i < ArraySize; i++) {
assert(squares[i] == 100);
assert(cubes[i] == 1000);
}
}
__global__ void computeSquares(int *deviceArray, int *resultArray) {
int index = threadIdx.x + blockIdx.x * blockDim.x;
resultArray[index] = deviceArray[index] * deviceArray[index];
}
__global__ void computeCubes(int *deviceArray, int *resultArray) {
int index = threadIdx.x + blockIdx.x * blockDim.x;
resultArray[index] =
deviceArray[index] * deviceArray[index] * deviceArray[index];
}
I was initially copying the host array to a location in memory that both kernels read from. I thought that since the memory was not marked read only, the scheduler ended up serializing the kernels. I modified my code to use separate device memory for each kernel (I don’t know how to mark device memory as read only). However, this made no difference to the observed behavior of the streams.
Any clues on why I don’t see any overlapping behavior?

