Hello,
While porting a large Fortran CFD code (~130k lines) to nvfortran and OpenACC, we hit several wrong-code bugs that involve internal procedures or MOVE_ALLOC. Each one is reduced below to a small standalone program.
Environment:
- NVIDIA HPC SDK 26.9 (nvfortran 26.9-0), x86-64.
- RHEL 8.10 on Xeon Sapphire Rapids. We first saw bugs 1 and 2 on Ubuntu with a Xeon Silver 4316, with the same results.
OMP_NUM_THREADS=2unless noted.- gfortran 13.3 (
-O3 -fopenmp -fcheck=all) gives the expected answer for every program.
Bugs 2, 4b and 5 give wrong answers silently at the optimization levels used in production. In our code, bug 5 made an MPI run assign data to the wrong rank, and the bug 4b pattern broke the results of OpenACC compute regions.
1. Array-valued internal function using host variables, called in a parallel region
! Array-valued internal function that uses a host variable,
! called inside an OpenMP parallel region, when the host has a second
! internal procedure that also uses a host variable: crashes at -O0.
module m
contains
subroutine s(res)
integer, intent(out) :: res(100)
real(8) :: h(3), x(3)
integer :: nb, i, ib(3)
h = 0.5d0; nb = 7
!$omp parallel do private(i,x,ib)
do i = 1, 100
x = dble(i)*0.01d0
ib = bin_of(x)
res(i) = sum(ib)
end do
!$omp end parallel do
contains
function bin_of(x) result(o)
real(8), intent(in) :: x(3)
integer :: o(3)
o = min( floor(x/h) + 1, nb ) ! uses host variables h and nb
end function
integer function other(l) ! not even called
integer, intent(in) :: l
other = nb + l
end function
end subroutine
end module
program p
use m
integer :: r(100)
call s(r)
print *, sum(r), ' (expected 456)'
end program
| Options | Result |
|---|---|
-mp -O0, 2 threads |
NVIDIA OpenMP Runtime Error: detected a stack overflow in a parallel region (the same with OMP_STACKSIZE=1G) |
-mp -O0, 1 thread |
segmentation fault |
-mp -O1, -O2, -O3; no -mp |
456 (correct) |
The crash needs the second internal procedure other, which is never called.
2. Non-contiguous section of a fixed-size PRIVATE array passed to an assumed-shape dummy
! Non-contiguous section of a fixed-size PRIVATE array passed to an
! assumed-shape dummy inside an OpenMP parallel region.
module m
contains
subroutine take2(a, r)
real(8), intent(in) :: a(:,:)
real(8), intent(out) :: r
r = sum(a)
end subroutine
subroutine take1(a, r)
real(8), intent(in) :: a(:)
real(8), intent(out) :: r
r = sum(a)
end subroutine
subroutine s(tot2, tot1)
real(8), intent(out) :: tot2, tot1
real(8) :: w(6,6), v(8), r2, r1, acc2(100), acc1(100)
integer :: c
!$omp parallel do private(c,w,v,r2,r1)
do c = 1, 100
w = 1.d0
v = 1.d0
call take2(w(1:5,:), r2) ! 5x6 section, non-contiguous
call take1(v(1:8:2), r1) ! strided section
acc2(c) = r2
acc1(c) = r1
end do
!$omp end parallel do
tot2 = sum(acc2); tot1 = sum(acc1)
end subroutine
end module
program p
use m
real(8) :: t2, t1
call s(t2, t1)
print *, t2, ' (expected 3000)'
print *, t1, ' (expected 400)'
end program
| Options | Result |
|---|---|
-mp -O0, -mp -O1 |
segmentation fault |
-mp -O2, -mp -O3 |
detected a stack overflow in a parallel region |
-mp -O0 -Mbounds |
Subscript out of range for array w (line 21) subscript=1, lower bound=0, upper bound=-1 (line 21 is w = 1.d0) |
no -mp |
3000 and 400 (correct) |
With only the strided 1-D call, the program does not crash. It silently prints 0 instead of 400.
Not affected:
- contiguous sections and the whole array;
- an explicit-shape dummy;
- the same section inside an intrinsic;
- an ALLOCATABLE private array.
3. Named-constant array passed from an internal procedure arrives with size 0
module g
contains
subroutine show(fr)
real(8),intent(in) :: fr(:)
print *, 'size', size(fr), ' sum', sum(fr)
end subroutine
end module
module h
use g
real(8),parameter :: Y(2) = (/ 0.75d0, 0.25d0 /)
contains
subroutine s1()
call show(Y)
end subroutine
subroutine s2()
call show(Y)
call inner()
contains
subroutine inner()
call show(Y)
end subroutine
end subroutine
end module
program p
use h
call s1(); call s2()
end program
Expected: three lines size 2 sum 1.0.
| Options | Third line (from inner) |
|---|---|
-O0, -mp -O0, -mp -O1 |
size 0 sum 0.0 |
-mp -O2, -mp -O3 (also 1 thread) |
segmentation fault |
-O3 without -mp |
correct |
4. MOVE_ALLOC from a local allocatable that an (uncalled) internal procedure references
module m
real(8),allocatable :: gv(:,:)
contains
subroutine from_local_to_module()
real(8),allocatable :: pt(:,:)
allocate( pt(3,4) ); pt = 1.d0
call move_alloc( pt, gv )
print *, 'A: gv', sum(gv), shape(gv), ' pt allocated', allocated(pt)
contains
subroutine unused()
pt = 0.d0
end subroutine
end subroutine
subroutine from_local_to_local()
real(8),allocatable :: pt(:,:), dst(:,:)
allocate( pt(3,4) ); pt = 1.d0
call move_alloc( pt, dst )
print *, 'B: dst', sum(dst), shape(dst), ' pt allocated', allocated(pt)
contains
subroutine unused()
pt = 0.d0
end subroutine
end subroutine
subroutine to_local_from_local()
real(8),allocatable :: pt(:,:), src(:,:)
allocate( src(3,4) ); src = 1.d0
call move_alloc( src, pt )
print *, 'C: pt', allocated(pt)
if( allocated(pt) ) print *, ' ', sum(pt), shape(pt)
contains
subroutine unused()
pt = 0.d0
end subroutine
end subroutine
end module
program p
use m
call from_local_to_module(); call from_local_to_local(); call to_local_from_local()
end program
Expected: sum 12, shape 3 4 in A, B and C.
| Options | Result |
|---|---|
-O0, -mp -O0 |
B: dst 0.0 1 1 (wrong shape and values); A and C correct |
-mp -O1 |
hangs |
-O3, -mp -O2, -mp -O3 |
segmentation fault |
4b. Host allocatable filled by MOVE_ALLOC, read by an internal procedure
! A host allocatable that receives its data by MOVE_ALLOC, read by an
! internal procedure.
module m
real(8),allocatable :: rg(:,:)
contains
subroutine s(from_module, r)
logical,intent(in) :: from_module
real(8),intent(out) :: r
real(8),allocatable :: pt(:,:)
if( from_module )then
call move_alloc( rg, pt )
else
allocate( pt(3,4) ); pt = 2.d0
end if
call inner()
contains
subroutine inner()
integer :: j
r = 0.d0
do j=1,size(pt,2)
r = r + sum( pt(:,j) )
end do
end subroutine
end subroutine
end module
program p
use m
real(8) :: r
allocate( rg(3,4) ); rg = 1.d0
call s(.true., r); print *, r, ' (expected 12)'
call s(.false., r); print *, r, ' (expected 24)'
end program
| Options | Result |
|---|---|
-O0, -mp -O0 … -mp -O3 |
3 instead of 12 (only the first column is seen); the ALLOCATE branch is correct |
-O3 without -mp |
segmentation fault |
5. Host scalar updated by a sibling internal procedure is read stale (no OpenMP directives)
module m
contains
subroutine s(nadd, r)
integer, intent(in) :: nadd
integer, intent(out) :: r
integer :: n, k, last(100), seen(100), tag(100)
integer, allocatable :: atag(:)
n = 0; tag = -1; last = 0; seen = 0
allocate( atag(100) ); atag = -1
call q()
r = count( last(1:nadd) == [(k, k=1,nadd)] )
contains
subroutine q()
integer :: k
do k = 1, nadd
call add_one()
last(k) = n
end do
end subroutine
subroutine add_one()
n = n + 1
tag(n) = 5
end subroutine
subroutine add_one_a()
n = n + 1
atag(n) = 0
end subroutine
end subroutine
end module
program p
use m
integer :: r
call s(68, r)
print '(i5,a)', r, ' (expected 68)'
end program
| Options | Result |
|---|---|
-mp -O2, -mp -O3 (also with 1 thread) |
0 (expected 68) |
-mp -O0, -mp -O1; no -mp |
68 |
5b. The same with a host allocatable that the sibling reallocates
! Internal procedure p grows the host allocatable a (move_alloc in grow);
! its sibling q reads a(m) in a loop right after calling p.
module r2m
implicit none
contains
subroutine host( n, nbad )
integer,intent(in) :: n
integer,intent(out) :: nbad
integer,allocatable :: a(:)
integer :: num
allocate( a(4) ) ; num = 0 ; nbad = 0
call q
contains
subroutine q
integer :: k, m
do k = 1, n
call p( 1000 + k, m )
if( a(m) /= 1000 + k ) nbad = nbad + 1
end do
end subroutine q
subroutine p( v, m_out )
integer,intent(in) :: v
integer,intent(out) :: m_out
if( num >= size(a) ) call grow( a, 2*size(a) )
num = num + 1
m_out = num
a(num) = v
end subroutine p
end subroutine host
subroutine grow( a1, n_need )
integer,allocatable,intent(inout) :: a1(:)
integer,intent(in) :: n_need
integer,allocatable :: tmp(:)
allocate( tmp(n_need) ) ; tmp(:) = 0 ; tmp(1:size(a1)) = a1(:)
call move_alloc( tmp, a1 )
end subroutine grow
end module r2m
program r2
use r2m
implicit none
integer :: nbad
call host( 200, nbad )
print '(a,i0,a)', 'bad reads=', nbad, ' (0 is correct)'
end program r2
| Options | Result |
|---|---|
-mp -O2, -mp -O3 |
bad reads=196 |
-mp -O0, -mp -O1; no -mp |
bad reads=0 |
Our workarounds:
- avoid array-valued internal functions;
- make private arrays allocatable;
- copy parameter arrays to locals;
- do not reference MOVE_ALLOC sources or targets from internal procedures;
- pass values between sibling internal procedures as arguments.
Could you confirm these and let us know whether fixes are planned? Thank you.