#include <iostream>
#include <stdio.h>
#include <conio.h>
#include <windows.h>
#include <process.h>
#include <time.h>

#include <vector_types.h>
#include "cutil.h"
#include "cublas.h"
#include <cuda_runtime.h>


const int MAX_GPU_COUNT = 2;

//create an array of 2 event Handles
HANDLE hEvent[MAX_GPU_COUNT];
HANDLE StartEvent[MAX_GPU_COUNT];

typedef struct{
	int dataN;
	float pointA[32];
	float pointB[32];
	float AtimesB[32];
} AREA;

AREA plan[2];

//CUDA fuctions
extern "C" void vecDOTvec(float* DEVICE1, float* DEVICE2, float* RESULT, int ROW);
////////////////////////////////////////////////////////////////////////////////
//First Thread Function
static void Thread1 (LPVOID lpParam){
	CUDA_SAFE_CALL(cudaSetDevice(0));
    float *d_Data1, *d_Data2, *d_Mul;
    cublasStatus status;	
	status = cublasInit();
	status = cublasAlloc(plan[0].dataN * 1, sizeof(float), (void**)&d_Data1);
	status = cublasAlloc(plan[0].dataN * 1, sizeof(float), (void**)&d_Data2);
	status = cublasAlloc(plan[0].dataN * 1, sizeof(float), (void**)&d_Mul);

	while (true){
		WaitForMultipleObjects(MAX_GPU_COUNT, StartEvent, TRUE, INFINITE);
		CUDA_SAFE_CALL(cudaMemcpy(d_Data1, plan[0].pointA, sizeof(float)*32, cudaMemcpyHostToDevice));
		CUDA_SAFE_CALL(cudaMemcpy(d_Data2, plan[0].pointB, sizeof(float)*32, cudaMemcpyHostToDevice));
		vecDOTvec(d_Data1, d_Data2, d_Mul, plan[0].dataN);
		CUDA_SAFE_CALL(cudaThreadSynchronize());
		status = cublasGetVector(plan[0].dataN,sizeof(float), d_Mul, 1, plan[0].AtimesB, 1);
		ResetEvent(StartEvent[0]);
		SetEvent(hEvent[0]);
	}
    cublasFree(d_Data1);
	cublasFree(d_Data2);
	cublasFree(d_Mul);
}
//Second Thread Function
static void Thread2 (LPVOID lpParam){
	CUDA_SAFE_CALL(cudaSetDevice(1));
    float *d_Data1, *d_Data2, *d_Mul;
    cublasStatus status;	
	status = cublasInit();
	status = cublasAlloc(plan[1].dataN * 1, sizeof(float), (void**)&d_Data1);
	status = cublasAlloc(plan[1].dataN * 1, sizeof(float), (void**)&d_Data2);
	status = cublasAlloc(plan[1].dataN * 1, sizeof(float), (void**)&d_Mul);

	while (true){
		WaitForMultipleObjects(MAX_GPU_COUNT, StartEvent, TRUE, INFINITE);
		CUDA_SAFE_CALL(cudaMemcpy(d_Data1, plan[1].pointA, sizeof(float)*32, cudaMemcpyHostToDevice));
		CUDA_SAFE_CALL(cudaMemcpy(d_Data2, plan[1].pointB, sizeof(float)*32, cudaMemcpyHostToDevice));
		vecDOTvec(d_Data1, d_Data2, d_Mul, plan[1].dataN);
		CUDA_SAFE_CALL(cudaThreadSynchronize());		
		status = cublasGetVector(plan[1].dataN,sizeof(float), d_Mul, 1, plan[1].AtimesB, 1);
		ResetEvent(StartEvent[1]);
		SetEvent(hEvent[1]);
	}
    cublasFree(d_Data1);
	cublasFree(d_Data2);
	cublasFree(d_Mul);
}

void
main(int argc, char** argv)
{
	int i=0, a;
	float time;
	unsigned int hTimer;
	cublasStatus status;

    CUT_SAFE_CALL( cutCreateTimer(&hTimer) );

	for (a = 0; a < MAX_GPU_COUNT; a++){
		plan[a].dataN=32;
		for(int j = 0; j < 32; j++){
			plan[a].pointA[j] = j;//(float)rand() / (float)RAND_MAX;
			plan[a].pointB[j] = j*10;//(float)rand() / (float)RAND_MAX;
		}
	}

	for (int i=0; i<MAX_GPU_COUNT; i++){
		hEvent[i] = CreateEvent( NULL, TRUE, TRUE, NULL );
		StartEvent[i] = CreateEvent( NULL, TRUE, TRUE, NULL );
	}

	//start two new threads
	_beginthread( Thread1, 0, NULL );
	_beginthread( Thread2, 0, NULL );
	Sleep(3000);
	//loop runs 10 times
	//loop prints data and then waits for new data to be calculated
	//before printing again
	CUT_SAFE_CALL( cutResetTimer(hTimer) );
    CUT_SAFE_CALL( cutStartTimer(hTimer) );

	while (i<4){
		for (int j=0; j<MAX_GPU_COUNT; j++)
			SetEvent(StartEvent[j]);

		++i;

		WaitForMultipleObjects(MAX_GPU_COUNT, hEvent, TRUE, INFINITE);
		for (int j=0; j<MAX_GPU_COUNT; j++)	
			ResetEvent(hEvent[j]);

		printf("\n Thread1, round: %d \n",i);
		printf("vec_A	 vec_B 	   AxB \n",i);
		for(int i=0; i<32; ++i)
			printf("%f %f %f\n",plan[0].pointA[i], plan[0].pointB[i], plan[0].AtimesB[i]);

		printf("\n Thread2, round: %d \n",i);
		for(int i=0; i<32; ++i)
			printf("%f %f %f\n",plan[1].pointA[i], plan[1].pointB[i], plan[1].AtimesB[i]);		

		for(int j = 0; j < 32; j++){
		    plan[0].pointA[j] = plan[0].pointA[j]+1;
			plan[1].pointA[j] = plan[1].pointA[j]+1;
		}
	}

	for (int j=0; j<MAX_GPU_COUNT; j++){
		ResetEvent(StartEvent[j]);
		ResetEvent(hEvent[j]);
	}

	CUT_SAFE_CALL( cutStopTimer(hTimer) );
    time = cutGetTimerValue(hTimer);
    printf("\n Time (CUDA) (ms.)      : %f\n", time);

	CUT_EXIT(argc, argv);
}