I was getting some weird problems in my code and I think I boiled it down to this: If I attempt to allocate memory on the card and it fails, all subsequent attempts will fail regardless.
Here is code:
#include <stdio.h>
#include <iostream>
#include <cuda.h>
#include <cublas.h>
#include <cuda_runtime.h>
using namespace std;
#define test( condition ) {if( (condition) != 0 ) { fprintf( stderr, "\n FAILURE in %s, line %d\n", __FILE__, __LINE__ );exit( 1 );}}
int main(int argc, const char** argv)
{
cublasStatus s;
s = cublasInit();
cout << "cublasInit() " << s << endl;
float *d_1 = NULL;
unsigned int size = 10*(1<<20); /*10 MB*/
unsigned int freeMem, total;
if ( cuMemGetInfo( &freeMem, &total ) == cudaSuccess)
cout << "Free: " << freeMem << " Total: " << total << endl;
{ /*This block should fail*/
s = cublasAlloc(freeMem, 1, (void**)&d_1);
cout << "cublasAlloc(freeMem): " ;
switch (s)
{
case CUBLAS_STATUS_NOT_INITIALIZED:
cout << "CUBLAS_STATUS_NOT_INITIALIZED\n";
break;
case CUBLAS_STATUS_INVALID_VALUE:
cout << "CUBLAS_STATUS_INVALID_VALUE\n";
break;
case CUBLAS_STATUS_ALLOC_FAILED:
cout << "CUBLAS_STATUS_ALLOC_FAILED\n";
break;
case CUBLAS_STATUS_SUCCESS:
cout << "CUBLAS_STATUS_SUCCESS\n";
test(cublasFree(d_1));
break;
}
}
/*reset error*/
cublasGetError();
/*
cublasShutdown();
s = cublasInit();
cout << "cublasInit() " << s << endl;
*/
{ /*This block should succeed*/
size = freeMem - size;
cout << "Allocation size in bytes " << size << endl;
s = cublasAlloc(size, 1, (void**)&d_1);
cout << "cublasAlloc(size): " ;
switch (s)
{
case CUBLAS_STATUS_NOT_INITIALIZED:
cout << "CUBLAS_STATUS_NOT_INITIALIZED\n";
break;
case CUBLAS_STATUS_INVALID_VALUE:
cout << "CUBLAS_STATUS_INVALID_VALUE\n";
break;
case CUBLAS_STATUS_ALLOC_FAILED:
cout << "CUBLAS_STATUS_ALLOC_FAILED\n";
break;
case CUBLAS_STATUS_SUCCESS:
cout << "CUBLAS_STATUS_SUCCESS\n";
test(cublasFree(d_1));
break;
}
}
cublasShutdown();
}
The first block fails as expected with CUBLAS_STATUS_ALLOC_FAILED. The second block fails with CUBLAS_STATUS_NOT_INITIALIZED.
You will notice a commented out section where I attempt to shutdown and re-initialize CUBLAS. If you uncomment that, you will see that the cublasInit() also fails with code 1 (CUBLAS_STATUS_NOT_INITIALIZED) – which doesn’t seem possible given the comments in the header file.
I am running on a GTX 560 TI.