Real Time Rendering - how to get the loop correct ?

Accelerated Computing CUDA CUDA Programming and Performance
1

Hello guys!

I am just a beginner to CUDA and I started CUDA Raytracer project (I am using SDL to display the result).
I want to see how it will perform in real time but I cannot get the rendering loop correct.
I am doing this:

int main(int argc, char** argv)
{
	if (!initGraphics(RES_X, RES_Y))
	{
		return -1;
	}

	cudaDeviceSetLimit(cudaLimitStackSize, STACK_SIZE);
	

	// allocate memory for vfb on the GPU
	Color* dev_vfb;
	cudaMalloc((void**)&dev_vfb, sizeof(Color) * RES_X * RES_Y);
	
	// memcpy HostToDevice
	cudaMemcpy(dev_vfb, vfb_linear, sizeof(Color) * RES_X * RES_Y, cudaMemcpyHostToDevice);


	Uint8* keystate = SDL_GetKeyState(NULL);
	while (true)
	{
		if (keystate[SDLK_UP])
		{
			// do something
		}
		if (keystate[SDLK_DOWN])
		{
			// do something
		}
		if (keystate[SDLK_RIGHT])
		{
			// do something
		}
		if (keystate[SDLK_LEFT])
		{
			// do something
		}
		if (keystate[SDLK_ESCAPE])
		{
			return 0;
		}
		
		// call kernels
		cudaRenderer(dev_vfb);

		cudaMemcpy(vfb_linear, dev_vfb, sizeof(Color) * RES_X * RES_Y, cudaMemcpyDeviceToHost);
				
		// convert the 1D device pixel array (vfb_linear) to 2D pixel array (vfb)
		convertDeviceToHostBuffer();

		// display the scene using SDL
		displayVFB(vfb);
	}


	// free memory	
	freeDeviceMemory();
	cudaFree(dev_vfb);
	
	closeGraphics();

	return EXIT_SUCCESS;
}

dev_vfb is the array (1D) I use on the GPU. When I am done, I copy dev_vfb to linear_vfb (that is on the CPU). I convert linear_vfb to 2D array - vfb and then I pass it to SDL for displaying.

When I run it my program is “not responding”
How can I arrange my loop with all that memory passing from CPU to GPU and vice verca ?

Contents of displayVFB() if needed:

void displayVFB(Color vfb[RES_Y][RES_X])
{
	int rs = screen->format->Rshift;
	int gs = screen->format->Gshift;
	int bs = screen->format->Bshift;

	for (int y = 0; y < screen->h; y++)
	{
		Uint32* row = (Uint32*) ((Uint8*) screen->pixels + y * screen->pitch);
		for (int x = 0; x < screen->w; x++)
			row[x] = vfb[y][x].toRGB32(rs, gs, bs);
	}
	SDL_Flip(screen);
}