I started from a simplest case without GLUT. Now I have the same error.
OptiX Error: Invalid context (Details: Function “RTresult _rtContextCompile(RTcontext)” caught exception: Unable to set the CUDA device., [3735714])
(/home/jlmiao/NVIDIA-OptiX-SDK-3.6.0-linux64/SDK/sample3/sample3.c:147)
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <optix.h>
#include <sutil.h>
int main(int argc, char *argv[])
{
RTcontext context;
RTbuffer buffer;
RT_CHECK_ERROR(rtContextCreate(&context));
/*Buffer*/
RTvariable var_buffer;
RT_CHECK_ERROR(rtBufferCreate(context, RT_BUFFER_OUTPUT, &buffer));
RT_CHECK_ERROR(rtBufferSetFormat(buffer, RT_FORMAT_FLOAT4));
RT_CHECK_ERROR(rtBufferSetSize2D(buffer, 256u, 256u));
RT_CHECK_ERROR(rtContextDeclareVariable(context, "result_buffer", &var_buffer));
RT_CHECK_ERROR(rtVariableSetObject(var_buffer,buffer));
/*EntryPoint and Raytype*/
RT_CHECK_ERROR(rtContextSetRayTypeCount(context,1));
RT_CHECK_ERROR(rtContextSetEntryPointCount(context,1));
/*Program*/
char path_to_ptx[512];
RTprogram ray_gen_program;
RTvariable draw_color;
sprintf(path_to_ptx, "%s/%s", sutilSamplesPtxDir(), "sample3_generated_draw_color.cu.ptx");
RT_CHECK_ERROR(rtProgramCreateFromPTXFile(context,path_to_ptx, "draw_solid_color",&ray_gen_program));
RT_CHECK_ERROR(rtProgramDeclareVariable(ray_gen_program, "draw_color",&draw_color));
RT_CHECK_ERROR(rtVariableSet3f(draw_color,0.462f,0.725f,0.0f));
RT_CHECK_ERROR(rtContextSetRayGenerationProgram(context,0,ray_gen_program));
/*Run*/
RT_CHECK_ERROR(rtContextValidate(context));
RT_CHECK_ERROR(rtContextCompile(context));
RT_CHECK_ERROR(rtContextDestroy(context));
return 0;
}