HYB format in cuSPARSE

Hello,

I am trying to use the Hybrid (HYB) format in cuSPARSE for the first time.

I first need to convert a matrix from the CSR format to the HYB format, and then multiply a dense vector with the sparse matrix in the HYB format.

For this I am using the following three functions.
cusparseCreateHybMat
cusparseDcsr2hyb
cusparseDhybmv

Unfortunately, unlike the cuBLAS manual which tells the location (Host or Device) for each argument of each function, the cuSPARSE manual does not tell the location of the arguments for the functions in the library.

To start with I assumed that all the matrices and vectors are in the device memory, which compiled fine, but then gave segmentation fault during execution.

I tried various combinations of the memory locations for the three functions. Eventually, when I assumed that matrices in cusparseCreateHybMat and cusparseDcsr2hyb are in Host, and those in cusparseDhybmv are in Device, the code compiled fine and ran.

But, during execution, I am getting cusparseDcsr2hyb error “6”, which is then (I believe) leading to cusparseDhybmv error “7”.

The code is attached below.

Does anyone has experience with cusparseDcsr2hyb, and identify the problem in the code?

Thanks in advance.

Mahesh


module cusparse
use iso_c_binding

! enums

enum, bind(C) ! cusparseStatus_t
enumerator :: CUSPARSE_STATUS_SUCCESS=0
enumerator :: CUSPARSE_STATUS_NOT_INITIALIZED=1
enumerator :: CUSPARSE_STATUS_ALLOC_FAILED=2
enumerator :: CUSPARSE_STATUS_INVALID_VALUE=3
enumerator :: CUSPARSE_STATUS_ARCH_MISMATCH=4
enumerator :: CUSPARSE_STATUS_MAPPING_ERROR=5
enumerator :: CUSPARSE_STATUS_EXECUTION_FAILED=6
enumerator :: CUSPARSE_STATUS_INTERNAL_ERROR=7
enumerator :: CUSPARSE_STATUS_MATRIX_TYPE_NOT_SUPPORTED=8
end enum

enum, bind(c) ! cusparsePointerMode_t
enumerator :: CUSPARSE_POINTER_MODE_HOST = 0
enumerator :: CUSPARSE_POINTER_MODE_DEVICE = 1
end enum

enum, bind(c) ! cusparseAction_t
enumerator :: CUSPARSE_ACTION_SYMBOLIC = 0
enumerator :: CUSPARSE_ACTION_NUMERIC = 1
end enum

enum, bind(C) ! cusparseMatrixType_t
enumerator :: CUSPARSE_MATRIX_TYPE_GENERAL = 0
enumerator :: CUSPARSE_MATRIX_TYPE_SYMMETRIC = 1
enumerator :: CUSPARSE_MATRIX_TYPE_HERMITIAN = 2
enumerator :: CUSPARSE_MATRIX_TYPE_TRIANGULAR = 3
end enum

enum, bind(C) ! cusparseFillMode_t
enumerator :: CUSPARSE_FILL_MODE_LOWER = 0
enumerator :: CUSPARSE_FILL_MODE_UPPER = 1
end enum

enum, bind(C) ! cusparseDiagType_t
enumerator :: CUSPARSE_DIAG_TYPE_NON_UNIT = 0
enumerator :: CUSPARSE_DIAG_TYPE_UNIT = 1
end enum

enum, bind(C) ! cusparseIndexBase_t
enumerator :: CUSPARSE_INDEX_BASE_ZERO = 0
enumerator :: CUSPARSE_INDEX_BASE_ONE = 1
end enum

enum, bind(C) ! cusparseOperation_t
enumerator :: CUSPARSE_OPERATION_NON_TRANSPOSE = 0
enumerator :: CUSPARSE_OPERATION_TRANSPOSE = 1
enumerator :: CUSPARSE_OPERATION_CONJUGATE_TRANSPOSE = 2
end enum

enum, bind(C) ! cusparseDirection_t
enumerator :: CUSPARSE_DIRECTION_ROW = 0
enumerator :: CUSPARSE_DIRECTION_COLUMN = 1
end enum

enum, bind(C) ! cusparseHybPartition_t
enumerator :: CUSPARSE_HYB_PARTITION_AUTO = 0
enumerator :: CUSPARSE_HYB_PARTITION_USER = 1
enumerator :: CUSPARSE_HYB_PARTITION_MAX = 2
end enum

! types

type cusparseHandle
type(c_ptr) :: handle
end type cusparseHandle

type :: cusparseMatDescr
type(c_ptr) :: descr
end type cusparseMatDescr

type cusparseSolveAnalysisInfo
type(c_ptr) :: info
end type cusparseSolveAnalysisInfo

type cusparseHybMat
type(c_ptr) :: mat
end type cusparseHybMat

! ----------------
! helper functions
! ----------------

interface
integer(c_int) function cusparseCreate(handle) bind(C,name=‘cusparseCreate’)
import cusparseHandle
type(cusparseHandle) :: handle
end function cusparseCreate
end interface

interface
integer(c_int) function cusparseDestroy(handle) bind(C,name=‘cusparseDestroy’)
import cusparseHandle
type(cusparseHandle), value :: handle
end function cusparseDestroy
end interface

interface
integer(c_int) function cusparseSetPointerMode(handle, mode) bind(C,name=‘cusparseSetPointerMode’)
use iso_c_binding
import cusparseHandle
type(cusparseHandle), value :: handle
integer(c_int), value :: mode
end function cusparseSetPointerMode
end interface

! matrix descriptor

interface
integer(c_int) function cusparseCreateMatDescr(descrA) bind(C,name=‘cusparseCreateMatDescr’)
import cusparseMatDescr
type(cusparseMatDescr) :: descrA
end function cusparseCreateMatDescr
end interface

interface
integer(c_int) function cusparseDestroyMatDescr(descrA) bind(C,name=‘cusparseDestroyMatDescr’)
import cusparseMatDescr
type(cusparseMatDescr), value :: descrA
end function cusparseDestroyMatDescr
end interface

interface
integer(c_int) function cusparseSetMatType(descrA, type) bind(C,name=‘cusparseSetMatType’)
use iso_c_binding
import cusparseMatDescr
type(cusparseMatDescr), value :: descrA
integer(c_int), value :: type
end function cusparseSetMatType
end interface

interface
integer(c_int) function cusparseSetMatIndexBase(descrA, base) bind(C,name=‘cusparseSetMatIndexBase’)
use iso_c_binding
import cusparseMatDescr
type(cusparseMatDescr), value :: descrA
integer(c_int), value :: base
end function cusparseSetMatIndexBase
end interface

interface cusparseCreateHybMat
integer(c_int) function cusparseCreateHybMat(hybmat) bind(C,name=‘cusparseCreateHybMat’)
import cusparseHybMat
type(cusparseHybMat) :: hybmat
end function cusparseCreateHybMat
end interface cusparseCreateHybMat

interface cusparseDhybmv
integer(c_int) function cusparseDhybmv(handle, trans, &
alpha, descr, hybA, x, beta, y) bind(C,name=‘cusparseDcsrmv’)
use iso_c_binding
import cusparseHandle, cusparseMatDescr, cusparseHybMat
type(cusparseHandle), value :: handle
integer(c_int), value :: trans
!pgi$ ignore_tkr (d) alpha, (d) beta
real(c_double) :: alpha
! type(cusparseMatDescr), value :: descr
integer(8), value :: descr
type(cusparseHybMat), device :: hybA
real(c_double), device :: x()
real(c_double) :: beta
real(c_double), device :: y(
)
end function cusparseDhybmv
end interface cusparseDhybmv


interface cusparseDcsr2hyb
integer(c_int) function cusparseDcsr2hyb(handle, m, n, &
descr, csrVal, csrRowPtr, csrColInd, hybmat, ellWidth, partType) bind(C,name=‘cusparseDcsr2hyb’)
use iso_c_binding
import cusparseHandle, cusparseMatDescr, cusparseHybMat
type(cusparseHandle), value :: handle
integer(c_int), value :: m, n
!type(cusparseMatDescr), value :: descr
integer(8), value :: descr
real(c_double) :: csrVal()
integer(c_int) :: csrRowPtr(
)
integer(c_int) :: csrColInd(*)
type(cusparseHybMat) :: hybmat
integer(c_int), value :: ellWidth
integer(c_int), value :: partType
end function cusparseDcsr2hyb
end interface cusparseDcsr2hyb

end module cusparse

program hybmv
use cudafor
use cusparse

implicit none

real(8), dimension (:), allocatable :: bmat,x,y
integer, dimension (:), allocatable :: irstrt,icol
real(8), device, dimension (:), allocatable :: bmat_d,x_d,y_d
integer, device, dimension (:), allocatable :: irstrt_d,icol_d
integer :: istat, version
integer(8) :: iworkaround

type(cusparseHandle) :: h
type(cusparseMatDescr) :: descrA
type(cusparseHybMat) :: hybBmat
type(cusparseHybMat), device :: hybBmat_d

allocate (bmat(5),irstrt(4),icol(5),x(3),y(3))
allocate (bmat_d(5),irstrt_d(4),icol_d(5),x_d(3),y_d(3))

bmat = (/1.d0,1.d0,2.d0,1.0d0,2.d0/)
irstrt = (/1,2,4,6/)
icol = (/1,1,2,2,3/)
x = 1.0d0
y = 0.0d0

bmat_d = bmat
irstrt_d = irstrt
icol_d = icol
x_d = x
y_d = y

print *,"bmat "
print *,bmat
print *,"irstrt ",irstrt
print *,"icol ",icol
print *,"x "
print *,x
print *,“y original”
print *,y

istat = cusparseCreate(h)
if (istat /= CUSPARSE_STATUS_SUCCESS) write(,) ‘cusparseCreate error:’, istat
istat = cusparseCreateMatDescr(descrA)
if (istat /= CUSPARSE_STATUS_SUCCESS) write(,) ‘cusparseCreateMatDescr error:’, istat
istat = cusparseSetMatIndexBase(descrA, CUSPARSE_INDEX_BASE_ONE)
if (istat /= CUSPARSE_STATUS_SUCCESS) write(,) ‘cusparseSetMatIndexBase error:’, istat
istat = cusparseSetMatType(descrA, CUSPARSE_MATRIX_TYPE_GENERAL)
if (istat /= CUSPARSE_STATUS_SUCCESS) write(,) ‘cusparseSetMatType error:’, istat

istat = cusparseCreateHybMat(hybBmat)
if (istat /= CUSPARSE_STATUS_SUCCESS) write(,) ‘cusparseCreateHybMat error:’, istat

iworkaround = transfer(descrA,iworkaround)
istat = cusparseDcsr2hyb(h,3,3,iworkaround,bmat,irstrt,icol,hybBmat,1,CUSPARSE_HYB_PARTITION_AUTO)
if (istat /= CUSPARSE_STATUS_SUCCESS) write(,) ‘cusparseDcsr2hyb error:’, istat
hybBmat_d = HybBmat

istat = cusparseDhybmv(h,CUSPARSE_OPERATION_NON_TRANSPOSE,1.0d0,iworkaround, &
hybBmat_d,x_d,1.0d0,y_d)
if (istat /= CUSPARSE_STATUS_SUCCESS) write(,) ‘cusparseDhybmv error:’, istat
y = y_d

print *,“y new”
print *,y

deallocate(bmat,irstrt,icol,x,y)
deallocate(bmat_d,irstrt_d,icol_d,x_d,y_d)

istat = cusparseDestroyMatDescr(descrA)
istat = cusparseDestroy(h)

end program hybmv