Segmentation fault : cublasDgetrfBatched and cublasDgetriBatched

jchentrlc1 · July 9, 2019, 8:32pm

Hi,

I’m getting segmentation fault from calling “cublasDgetrfBatched” and “cublasDgetriBatched”. Then I wrote a small code to repeat this error. Would you please take a look?

Thanks.

– Jing

----------------------------------- code start from here ------------------------------

module LU
interface cuda_getrf
subroutine cuda_dgetrf(n, Aarray, lda, ipvt, info, batchCount) bind(C,name=‘cublasDgetrfBatched’)
use cudafor
use iso_c_binding
integer, value :: n
type(c_devptr), device :: Aarray()
integer, value :: lda
integer, device :: ipvt()
integer, device :: info()
integer, value :: batchCount
end subroutine cuda_dgetrf
end interface
interface cuda_getri
subroutine cuda_dgetri(n, Aarray, lda, ipvt, Carray, ldc, info, batchCount) bind(C,name=‘cublasDgetriBatched’)
use cudafor
use iso_c_binding
integer, value:: n
type(c_devptr), device :: Aarray()
integer, value :: lda
integer, device :: ipvt()
type(c_devptr), device :: Carray()
integer, value :: ldc
integer, device :: info(*)
integer, value :: batchCount
end subroutine cuda_dgetri
end interface
end module LU

PROGRAM solvelinear
use cudafor
use LU
implicit none
REAL(8), dimension(3,3) :: A,Ainv,M,LU
INTEGER,dimension(3) :: ipiv
INTEGER :: i

integer, parameter :: batchCount=1, ndim=3
integer, device :: ipiv_d(ndim)
integer, device :: info1_d(batchCount), info2_d(batchCount)
real(8), device :: t_d(ndim,ndim)
real(8), device :: ti_d(ndim,ndim)
type(c_devptr) :: devPtrA(batchCount), devPtrC(batchCount)
type(c_devptr), device :: devPtrA_d(batchCount), devPtrC_d(batchCount)
INTEGER istat, status

A = reshape((/1,-1,3,3,1,1,1,1,1,1/),(/3,3/))

t_d(1:ndim,1:ndim) = A
ti_d(1:ndim,1:ndim) = 0.

devPtrA(batchCount) = c_devloc(t_d(1,1))
devPtrC(batchCount) = c_devloc(ti_d(1,1))

devPtrA_d = devPtrA
devPtrC_d = devPtrC

call cuda_dgetrf(ndim,devPtrA_d,ndim,ipiv_d,info1_d,batchCount)
call cuda_dgetri(ndim,devPtrA_d,ndim,ipiv_d,devPtrC_d,ndim,info2_d,batchCount)
Ainv(1:ndim,1:ndim)=ti_d(1:ndim,1:ndim)

!-- computation of A^-1 * A to check the inverse
M = matmul(Ainv,A)
do i=1,3
print*,M(i,:)
end do
END PROGRAM solvelinear

!.. CORRECT OUTPUT
!.. 1. 0. 0.
!.. 0. 1. 0.
!.. 0. 0. 1.

------------------------------ code end here -------------------------

MatColgrove · July 9, 2019, 9:54pm

Hi Jing,

You are missing a cuBLAS handle which is probably the cause of the segv.

Is there a reason why you not using the cuBLAS interface modules we provide? Might be easier than trying to write your own interface.

-Mat

jchentrlc1 · July 10, 2019, 1:31pm

Hi Mat,

I wrote my own is because whenever I declare the handle

type(cublasHandle) :: handle

The compile complains about

mpif90 -c -r8 -w -O3 -fast -Mcuda=cuda10.0 mylinearsolve.CUF

PGF90-S-0155-Derived type has not been declared - cublashandle (mylinearsolve.CUF: 377)
0 inform, 0 warnings, 1 severes, 0 fatal for tmatrix
makefile:107: recipe for target ‘mylinearsolve.o’ failed

Do you know why?

Thanks,

– Jing

MatColgrove · July 10, 2019, 2:21pm

Hi Jing,

“type(cublasHandle)” is defined in the PGI cuBLAS interface module. Did you add “use cublas” which will include this module?

-Mat

jchentrlc1 · July 10, 2019, 2:50pm

Yes. I did. I’m re-post my code here which include “cublas” and “handle” and gives the complains:

----------------------------------- code start from here ------------------------------

module LU
interface cuda_getrf
subroutine cuda_dgetrf(n, Aarray, lda, ipvt, info, batchCount) bind(C,name=‘cublasDgetrfBatched’)
use cudafor
use iso_c_binding
integer, value :: n
type(c_devptr), device :: Aarray()
integer, value :: lda
integer, device :: ipvt()
integer, device :: info()
integer, value :: batchCount
end subroutine cuda_dgetrf
end interface
interface cuda_getri
subroutine cuda_dgetri(n, Aarray, lda, ipvt, Carray, ldc, info, batchCount) bind(C,name=‘cublasDgetriBatched’)
use cudafor
use iso_c_binding
integer, value:: n
type(c_devptr), device :: Aarray()
integer, value :: lda
integer, device :: ipvt()
type(c_devptr), device :: Carray()
integer, value :: ldc
integer, device :: info(*)
integer, value :: batchCount
end subroutine cuda_dgetri
end interface
end module LU

PROGRAM solvelinear
use cudafor
use cublas
use LU
implicit none
REAL(8), dimension(3,3) :: A,Ainv,M,LU
INTEGER,dimension(3) :: ipiv
INTEGER :: i

integer, parameter :: batchCount=1, ndim=3
integer, device :: ipiv_d(ndim)
integer, device :: info1_d(batchCount), info2_d(batchCount)
real(8), device :: t_d(ndim,ndim)
real(8), device :: ti_d(ndim,ndim)
type(c_devptr) :: devPtrA(batchCount), devPtrC(batchCount)
type(c_devptr), device :: devPtrA_d(batchCount), devPtrC_d(batchCount)
INTEGER istat, status
type(cublasHandle) :: h

A = reshape((/1,-1,3,3,1,1,1,1,1,1/),(/3,3/))

t_d(1:ndim,1:ndim) = A
ti_d(1:ndim,1:ndim) = 0.

devPtrA(batchCount) = c_devloc(t_d(1,1))
devPtrC(batchCount) = c_devloc(ti_d(1,1))

devPtrA_d = devPtrA
devPtrC_d = devPtrC

call cuda_dgetrf(ndim,devPtrA_d,ndim,ipiv_d,info1_d,batchCount)
call cuda_dgetri(ndim,devPtrA_d,ndim,ipiv_d,devPtrC_d,ndim,info2_d,batchCount)
Ainv(1:ndim,1:ndim)=ti_d(1:ndim,1:ndim)

!-- computation of A^-1 * A to check the inverse
M = matmul(Ainv,A)
do i=1,3
print*,M(i,:)
end do
END PROGRAM solvelinear

!.. CORRECT OUTPUT
!.. 1. 0. 0.
!.. 0. 1. 0.
!.. 0. 0. 1.

MatColgrove · July 10, 2019, 5:36pm

I wasn’t able to reproduce this error, but did go ahead and fix the segv by passing in a handle. I have not investigated why the results are incorrect.

% cat solve2.cuf
module LU
interface cuda_getrf
subroutine cuda_dgetrf(h,n, Aarray, lda, ipvt, info, batchCount) bind(C,name='cublasDgetrfBatched')
use cudafor
use cublas
use iso_c_binding
type(cublasHandle) :: h
integer, value :: n
type(c_devptr), device :: Aarray(*)
integer, value :: lda
integer, device :: ipvt(*)
integer, device :: info(*)
integer, value :: batchCount
end subroutine cuda_dgetrf
end interface
interface cuda_getri
subroutine cuda_dgetri(h,n, Aarray, lda, ipvt, Carray, ldc, info, batchCount) bind(C,name='cublasDgetriBatched')
use cudafor
use cublas
use iso_c_binding
integer, value:: n
type(cublasHandle) :: h
type(c_devptr), device :: Aarray(*)
integer, value :: lda
integer, device :: ipvt(*)
type(c_devptr), device :: Carray(*)
integer, value :: ldc
integer, device :: info(*)
integer, value :: batchCount
end subroutine cuda_dgetri
end interface
end module LU

PROGRAM solvelinear
use cudafor
use cublas
use LU
implicit none
REAL(8), dimension(3,3) :: A,Ainv,M,LU
INTEGER,dimension(3) :: ipiv
INTEGER :: i

integer, parameter :: batchCount=1, ndim=3
integer, device :: ipiv_d(ndim)
integer, device :: info1_d(batchCount), info2_d(batchCount)
real(8), device :: t_d(ndim,ndim)
real(8), device :: ti_d(ndim,ndim)
type(c_devptr) :: devPtrA(batchCount), devPtrC(batchCount)
type(c_devptr), device :: devPtrA_d(batchCount), devPtrC_d(batchCount)
INTEGER istat, status
type(cublasHandle) :: h

A = reshape((/1,-1,3,3,1,1,1,1,1,1/),(/3,3/))

t_d(1:ndim,1:ndim) = A
ti_d(1:ndim,1:ndim) = 0.

devPtrA(batchCount) = c_devloc(t_d(1,1))
devPtrC(batchCount) = c_devloc(ti_d(1,1))

devPtrA_d = devPtrA
devPtrC_d = devPtrC

istat = cublasGetHandle(h)

call cuda_dgetrf(h,ndim,devPtrA_d,ndim,ipiv_d,info1_d,batchCount)
call cuda_dgetri(h,ndim,devPtrA_d,ndim,ipiv_d,devPtrC_d,ndim,info2_d,batchCount)
Ainv(1:ndim,1:ndim)=ti_d(1:ndim,1:ndim)

!-- computation of A^-1 * A to check the inverse
M = matmul(Ainv,A)
do i=1,3
print*,M(i,:)
end do
END PROGRAM solvelinear

!... CORRECT OUTPUT
!... 1. 0. 0.
!... 0. 1. 0.
!... 0. 0. 1.
% pgfortran solve2.cuf -Mcudalib=cublas; a.out
    0.000000000000000         0.000000000000000         0.000000000000000
    0.000000000000000         0.000000000000000         0.000000000000000
    0.000000000000000         0.000000000000000         0.000000000000000

Here’s the same code using just the cuBLAS interface module we provide

% cat solvelinear.cuf
PROGRAM solvelinear
use cudafor
use cublas
implicit none
integer, parameter :: batchCount=1, ndim=3
INTEGER :: i
REAL(8), dimension(ndim,ndim) :: A,Ainv,M,LU
INTEGER,dimension(ndim) :: ipiv
integer, device :: ipiv_d(ndim)
integer, device :: info1_d(ndim*batchCount), info2_d(ndim*batchCount)
real(8), device :: t_d(ndim,ndim)
real(8), device :: ti_d(ndim,ndim)
type(c_devptr) :: devPtrA(batchCount), devPtrC(batchCount)
type(c_devptr), device :: devPtrA_d(batchCount), devPtrC_d(batchCount)
type(cublasHandle) :: h
INTEGER istat, status

istat = cublasGetHandle(h)

A = reshape((/1,-1,3,3,1,1,1,1,1,1/),(/3,3/))

t_d(1:ndim,1:ndim) = A
ti_d(1:ndim,1:ndim) = 0.

devPtrA = c_devloc(t_d(1,1))
devPtrC = c_devloc(ti_d(1,1))

devPtrA_d = devPtrA
devPtrC_d = devPtrC

istat = cublasDgetrfBatched(h,ndim,devPtrA_d,ndim,ipiv_d,info1_d,batchCount)
istat = cublasDgetriBatched(h,ndim,devPtrA_d,ndim,ipiv_d,devPtrC_d,ndim,info2_d,batchCount)
!Ainv(1:ndim,1:ndim)=ti_d(1:ndim,1:ndim)
Ainv=ti_d

!-- computation of A^-1 * A to check the inverse
M = matmul(Ainv,A)
do i=1,3
print*,M(i,:)
end do
END PROGRAM solvelinear

!... CORRECT OUTPUT
!... 1. 0. 0.
!... 0. 1. 0.
!... 0. 0. 1.
% pgf90 solvelinear.cuf -Mcudalib=cublas ; a.out
    1.000000000000000         0.000000000000000       -5.5511151231257827E-017
    0.000000000000000         1.000000000000000       -5.5511151231257827E-017
  -1.1102230246251565E-016  -1.1102230246251565E-016    1.000000000000000

Topic		Replies	Views
Segmentation fault using cublas<T>getrsBatched GPU-Accelerated Libraries cuda , cublas	3	1805	October 12, 2021
Calling CUBLAS' cublasDgetrfBatched proper procedure Legacy PGI Compilers	3	4942	January 30, 2017
cublas_cublasDgetrsBatched_problem GPU-Accelerated Libraries cublas , cusolver	5	1047	October 6, 2022
Cublas batched lu decomposition get segmentation fault GPU-Accelerated Libraries	3	1266	April 23, 2014
Trouble with cublas and matrix inverse in Fortran Legacy PGI Compilers	1	1973	April 9, 2018
New cuBlas function "getrfBatched" Legacy PGI Compilers	5	8649	January 21, 2014
Exception Error cublasSgetrsBatched while cublasSgetrfBatched has no issues (cuda12.8) GPU-Accelerated Libraries cublas	0	53	September 24, 2025
Problem with using CUBLAS getrfbatched() as it returns error: Illegal memory access was encountered GPU-Accelerated Libraries	3	4596	February 2, 2018
cublasDtrsmBatched giving error when to solve array linear systems GPU-Accelerated Libraries	0	515	October 25, 2017
PGF90-S-0155-Could not resolve generic procedure cublasdgemm Legacy PGI Compilers	4	4015	November 10, 2018

Segmentation fault : cublasDgetrfBatched and cublasDgetriBatched

Related topics