Ray Tracing multiple matrix object transformations

I have an issue with multiple objects transformations using 4x4 matrices
If the object is single, it is rendered normally, but for 2+ it has some artifacts
I think it is the issue with the procedure of matrices application (maybe I am doing it at a wrong place)

Pseudocode:

fn render(ray: Ray, coords: vec2<u32>, scan_depth: u32) {
    var attenuation = 1.0;
    
    // If I had only one transform, which I'd apply to the ray HERE,
    // result would be the same as on screenshot 2, but for multiple
    // chunks/transforms it won't work

    for (var i = 0u; i < scan_depth; i++) {
        var hit_record = HitRecord();

        if chunks_hit(ray, 0.001, 3.40282347e+38, &hit_record, i) {  
            // ...
        } else {
            // ...
        }
    }

    color_buffer[index] = vec4<f32>(vec3<f32>(0.0), 1.0);
    return;
}

fn chunks_hit(ray: Ray, t_min: f32, t_max: f32, record: HitRecord, q: u32) -> bool {
    var temp = HitRecord();
    var hit_anything = false;
    var closest_so_far = t_max;

    for (var i = 0; i < CHUNK_COUNT); i++) {
        // Without this condition, this won't work 
        // for any number of objects (screenshot 3)
        if q == 0 {
            ray.origin = (transforms_buffer[i].inverse_matrix * vec4<f32>(ray.origin, 1.0)).xyz;
            ray.direction = (transforms_buffer[i].inverse_matrix * vec4<f32>(ray.direction, 0.0)).xyz;
        }

        if Chunk::hit(i, ray, t_min, closest_so_far, &temp) {
            hit_anything = true;
            closest_so_far = temp.t;
            *record = temp;
        }
    }

    return hit_anything;
}

Hello & welcome @sanerhp.mail,
the ray directions must be normalized when using multiple TLAS instances.
That might be wrong when having scaling in the transforms.