DoF lens shader

I wouldn’t use the Windows rand() function in a progressive ray tracer at all. RAND_MAX is 32767 which means there are only 15 bits in that RNG.

I had implemented a simple thin lens camera for depth of field as a callable program before. The code below is from a different application than the OptiX Introduction samples, but there should be no problem to add it to the OptiX introduction samples.
Make sure to add a new LENS_SHADER_THIN_LENS enum, add the callable program to the map of programs and adjust the GUI to make it selectable.

// TODO: Add this to the lens_shaders.cu

// Let the sensor be 35 mm wide. Scene is defined in meters!
#define SENSOR_WIDTH 0.035f

// .x = lensRadius
// .y = sensorDistance
// .z = (focalDistance + sensorDistance) / sensorDistance
// .w = aspect ratio == length(sysCameraU) / length(sysCameraV)
rtDeclareVariable(float4, sysThinLens, , );
...

// TODO: Adjust the signature to the OptiX Introduction, returning origin and direction instead of setting the prd.pos and prd.wi.
RT_CALLABLE_PROGRAM void lens_shader_thinlens(PerRayData& prd, const float2 pixel, const float2 screen, const float2 sample)
{
  const float2 fragment = pixel + sample;                    // Jitter the sub-pixel location
  const float2 ndc      = (fragment / screen) * 2.0f - 1.0f; // Normalized device coordinates in range [-1, 1].

  // First calculate the point on the lens through which we look.
  const float theta = 2.0f * M_PIf * sample.x;
  const float r     = sqrtf(sample.y) * sysThinLens.x; // sysThinLens.x is the lens radius.
  const float3 lens = make_float3(r * cosf(theta), r * -sinf(theta), 0.0f);

  // Compute the point on the sensor, behind the lens.
  const float3 sensor = make_float3(-ndc.x * SENSOR_WIDTH * sysThinLens.w, 
                                    -ndc.y * SENSOR_WIDTH, 
                                    -sysThinLens.y);

  // Compute point on the focal plane.
  const float3 focal = (sensor - sysThinLens.z * sensor) - lens;

  const float3 U = optix::normalize(sysCameraU);
  const float3 V = optix::normalize(sysCameraV);
  const float3 W = optix::normalize(sysCameraW); // DAR PERF Do this inside the host code when this camera type is selected.

  prd.pos = sysCameraPosition + lens.x * U + lens.y * V; // + lens.z * W; // lens.z is 0.0f
  prd.wi  = optix::normalize(focal.x * U + focal.y * V + focal.z * W);
}
// TODO: Host code which fills in the sysThinLens pre-calculated parameters.

// GUI Variables controlling the thin lens camera:

float m_cameraFocalLength;
float m_cameraAperture;

// Defaults:
, m_cameraFocalLength(100.0f) // GUI is in mm, will be sent in meters to match the model coordinates.
, m_cameraAperture(32.0f) // Range [1.0, 64.0f]!

void Application::updateThinLens()
{
  // Thin lens data:
  float f = (m_cameraAperture - 1.0f) / 63.0f; // (64.0f - 1.0f); // Immediate values 1.0f and 64.0f are from the aperture GUI range!
  f = f * f * 63.0f + 1.0f;  // f = powf((m_cameraAperture - 1.0f) / (64.0f - 1.0f), 2.0f) * (64.0f - 1.0f) + 1.0f; 
  
  const float focalLength = m_cameraFocalLength * 0.001f; // GUI focal length is in millimeters, but the scene is modeled in meters!
  const float lensRadius  = 0.5f * focalLength / f;       // sysThinLens.x
  
  const float cameraFocalDistance = m_pinholeCamera.m_distance; // Distance in meters.

  const float focalDistance = std::max(focalLength + 0.001f, cameraFocalDistance);  // Set a minimum focal distance bigger than the focalLength.
  const float sensorDistance = 1.0f / (1.0f / focalLength - 1.0f / focalDistance);  // sysThinLens.y
  const float t = (cameraFocalDistance + sensorDistance) / sensorDistance;          // sysThinLens.z
  const float aspect = m_pinholeCamera.getAspectRatio();                            // sysThinLens.w

  m_context["sysThinLens"]->setFloat(lensRadius, sensorDistance, t, aspect);
}

The pinhole camera distance from the center of interest defines the focus plane. I added an event handler which affects only that distance. Something like CTRL + Right Mouse Button Drag or the like. Have fun.