Hi! It’s been a few days I’m trying to figure out why after my mesh shader invocation I’ve random pixel colors on the screen. But I can’t find anything wrong in my code :
-My descriptors are corrects. Everytihign works fine with a vertex shader, I’ve no validations errors too. But with my mesh shader I’ve troubles! The problem is after my mesh shader invocation here :
#version 460
extension GL_EXT_mesh_shader : require
extension GL_EXT_debug_printf : enable
extension GL_EXT_nonuniform_qualifier : enable
define NB_PRIMITIVE_TYPES 6
define MAX_FRAMES_IN_FLIGHT 2
define MAX_BONES_INFLUENCE 4
layout (local_size_x = 1, local_size_y = 1, local_size_z = 1) in;
// Déclaration du type de primitive + limites
layout (triangles, max_vertices = 255, max_primitives = 85) out;
layout(location = 0) out vec4 fragColor;
layout(location = 1) out vec2 fragTexCoord;
layout(location = 2) out vec3 normal;
layout(location = 3) flat out int materialID;
layout(location = 4) flat out int primitiveType;
layout(location = 5) flat out int currentFrame;
layout(location = 6) flat out int flag;
struct MeshDrawCommand {
uint groupXCount;
uint groupYCount;
uint groupZCount;
};
struct LODLevel {
int index_offset;
int index_count;
int meshletOffset;
int meshletCount;
};
struct TaskData {
MeshDrawCommand meshDrawCommand;
uint baseInstance;
uint vertexCount;
uint primitiveCount;
uint vertexOffset;
uint clusterOffset;
};
struct AABB {
vec3 center;
vec3 size;
};
struct Cluster {
AABB volume;
int meshletOffset;
int meshletCount;
int id;
int lodLevel;
int leaf;
int submeshId;
};
struct MeshLet {
uint id;
uint vertexOffset;
uint indexOffset;
uint nbVertices;
uint nbIndexes;
uint minVertex;
uint maxVertex;
int clusterId;
uint lod;
int voxel;
int rendered;
int submeshId;
vec3 mins;
vec3 maxs;
};
struct SubMesh {
AABB globalBounds;
int vertexOffset;
int indexOffset;
int primitiveType;
int materialId;
int nbVertices;
int nbIndexes;
int lodOffset;
int id;
int lodLevel;
int objectId;
int meshletOffset;
int meshletCount;
};
struct Vertex {
vec3 position; ///< 3D position of the vertex
uint color; ///< Color of the vertex
vec2 texCoords; ///< Coordinates of the texture’s pixel to map to the vertex
vec3 normal;
//bone indexes which will influence this vertex
int m_BoneIDs[MAX_BONES_INFLUENCE];
//weights from each bone
float m_Weights[MAX_BONES_INFLUENCE];
int submeshId;
};
struct ModelData {
mat4 modelMatrix;
mat4 shadowProjMatrix;
mat4 borderMatrices;
};
struct LodLevel {
uint index_offset;
uint nbLevels;
int meshletOffset;
int meshletCount;
};
struct TaskPayload {
int taskId;
};
taskPayloadSharedEXT TaskPayload payload;
layout (push_constant) uniform PushConstant {
mat4 projMatrix;
mat4 viewMatrix;
int primitiveType;
int currentFrame;
} pc;
layout (std430, set = 0, binding = 0) buffer TastDataSSBO {
TaskData taskData;
} taskDatas[NB_PRIMITIVE_TYPES * MAX_FRAMES_IN_FLIGHT];
layout(std430, set = 0, binding = 1) buffer TaskCountSSBO {
uint taskCount;
} TaskCounts[NB_PRIMITIVE_TYPESMAX_FRAMES_IN_FLIGHT];
layout (std430, set = 0, binding = 2) buffer ModelDataSSBO {
ModelData modelData[];
} modelDataBuffer[NB_PRIMITIVE_TYPES * MAX_FRAMES_IN_FLIGHT];
layout (std430, set = 0, binding = 3) buffer ClustersSSBO {
Cluster clusters[];
} clusterDatas[MAX_FRAMES_IN_FLIGHTNB_PRIMITIVE_TYPES];
layout (std430, set = 0, binding = 4) buffer vertexBufferSSBO {
Vertex vertices;
} vertexData[NB_PRIMITIVE_TYPES];
layout (std430, set = 0, binding = 5) buffer indexBufferSSBO {
uint indexes;
} indexData[NB_PRIMITIVE_TYPES];
layout (std430, set=0, binding = 6) buffer LodLevelBuffer {
LODLevel lodLevel;
} lodLevelData;
layout (std430, set = 0, binding = 7) buffer SubMeshDataSSBO {
SubMesh subMeshData;
} SubMeshDatas;
layout (std430, set = 0, binding = 8) buffer MeshletsSSBO {
MeshLet meshlet;
} meshletDatas;
layout (std430, set = 0, binding = 9) buffer VerticesSSBO {
Vertex vertices;
} outputVertices[MAX_FRAMES_IN_FLIGHT];
vec4 unpackColor(uint c)
{
float r = float((c >> 24) & 0xFFu) / 255.0;
float g = float((c >> 16) & 0xFFu) / 255.0;
float b = float((c >> 8) & 0xFFu) / 255.0;
float a = float((c ) & 0xFFu) / 255.0;
return vec4(r, g, b, a);
}
void main() {
TaskData taskData = taskDatas[pc.primitiveType*MAX_FRAMES_IN_FLIGHT+pc.currentFrame].taskData[payload.taskId];
if (pc.currentFrame >= MAX_FRAMES_IN_FLIGHT || pc.primitiveType > 3) {
debugPrintfEXT("Current frame : %i %i", pc.currentFrame, pc.primitiveType);
}
if (taskData.vertexCount != 0)
debugPrintfEXT("Error : %i", taskData.vertexCount);
//Modèles 3D.
if (taskData.vertexCount == 0) {
Cluster cluster = clusterDatas[pc.primitiveType*MAX_FRAMES_IN_FLIGHT+pc.currentFrame].clusters[taskData.clusterOffset+gl_WorkGroupID.y];
SubMesh subMesh = SubMeshDatas.subMeshData[cluster.submeshId];
//debugPrintfEXT("Submesh material : %i, %i", subMesh.id, subMesh.materialId);
if (gl_WorkGroupID.z >= cluster.meshletCount) {
return;
}
MeshLet meshlet = meshletDatas.meshlet[cluster.meshletOffset+gl_WorkGroupID.z];
if (meshlet.rendered == 1) {
//debugPrintfEXT("Rendered");
return;
}
if (cluster.submeshId != subMesh.id || cluster.submeshId != meshlet.submeshId || meshlet.submeshId != subMesh.id) {
debugPrintfEXT("error submesh %i %i %i lod : %i %i %i\n", cluster.submeshId, meshlet.submeshId, subMesh.id, cluster.lodLevel, subMesh.lodLevel, meshlet.lod);
}
//debugPrintfEXT("mesh shader");
//Instance id = taskId (base instance) + workGroupId.x (instance id).
uint instanceId = taskDatas[pc.primitiveType*MAX_FRAMES_IN_FLIGHT+pc.currentFrame].taskData[payload.taskId].baseInstance + gl_WorkGroupID.x;
//debugPrintfEXT("Instance id : %i", gl_WorkGroupID.x);
mat4 modelMatrix = modelDataBuffer[pc.primitiveType*MAX_FRAMES_IN_FLIGHT+pc.currentFrame].modelData[instanceId].modelMatrix;
//debugPrintfEXT("model matrix : %v4f \n%v4f \n%v4f \n%v4f\n", modelMatrix[0], modelMatrix[1], modelMatrix[2], modelMatrix[3]);
//Avoid to render it twice if the meshlet is between two clusters. (Deduplication)
//debugPrintfEXT("mesh shader");
meshletDatas.meshlet[cluster.meshletOffset+gl_WorkGroupID.z].rendered = 1;
uint firstIndex = subMesh.indexOffset + lodLevelData.lodLevel[subMesh.lodOffset+cluster.lodLevel].index_offset + meshlet.indexOffset;
//debugPrintfEXT("First index : %i,", firstIndex);
//3 = Triangles.
uint maxVerts = 255u;
uint maxPrims = 85u;
uint vertexCount = min(meshlet.nbVertices, maxVerts);
uint primitiveCount = min(meshlet.nbIndexes / 3u, maxPrims);
/*if (primitiveCount > maxPrims)
debugPrintfEXT("vertex count : %i, primitive count : %i", vertexCount, primitiveCount);*/
if (pc.primitiveType == 3) {
SetMeshOutputsEXT(vertexCount, primitiveCount);
for (uint i = 0; i < vertexCount; i++) {
/*if (flag[i] == 0 || flag[i] > 1 || flag[i] < 0 || materialID[i] < 0 || materialID[i] >= 7 || primitiveType[i] != 3 || currentFrame[i] < 0 || currentFrame[i] >= MAX_FRAMES_IN_FLIGHT) {
return;
}
debugPrintfEXT("Flag : %i %i %i %i", flag[i], primitiveType[i], materialID[i], currentFrame[i]);*/
uint v = meshlet.vertexOffset + subMesh.vertexOffset + i;
if (v >= 77067)
debugPrintfEXT("v : %i", v);
vec4 color = unpackColor(vertexData[pc.primitiveType].vertices[v].color);
if (color.r < 0.9 || color.g < 0.9 || color.b < 0.9)
debugPrintfEXT("error : %v4f", color);
gl_MeshVerticesEXT[i].gl_Position = pc.projMatrix * pc.viewMatrix * modelMatrix * vec4(vertexData[pc.primitiveType].vertices[v].position, 1.0);
//debugPrintfEXT("vertex position %v3f : ",gl_MeshVerticesEXT[i].gl_Position);
fragColor[i] = color;
//debugPrintfEXT("color : %v4f", fragColor[i]);
fragTexCoord[i] = vertexData[pc.primitiveType].vertices[v].texCoords;
//debugPrintfEXT("tex coords : %v2f", fragTexCoord[i]);
normal[i] = mat3(transpose(inverse(modelMatrix))) * vertexData[pc.primitiveType].vertices[v].normal;
//debugPrintfEXT("normal %v3f", normal[i]);
materialID[i] = subMesh.materialId;
primitiveType[i] = pc.primitiveType;
currentFrame[i] = pc.currentFrame;
flag[i] = 1;
//debugPrintfEXT("materialId %i", subMesh.materialId);
//debugPrintfEXT("position : %i", i);
}
for (uint tri = 0; tri < primitiveCount; tri++) {
//debugPrintfEXT("gen tri : %i", tri);
uint g0 = indexData[pc.primitiveType].indexes[firstIndex + tri * 3 + 0];
uint g1 = indexData[pc.primitiveType].indexes[firstIndex + tri * 3 + 1];
uint g2 = indexData[pc.primitiveType].indexes[firstIndex + tri * 3 + 2];
if (firstIndex + tri * 3 + 2 >= 383400) {
debugPrintfEXT("erreur : %i", firstIndex + tri * 3 + 2);
}
//debugPrintfEXT("indexes : %i %i %i",g0, g1, g2);
uint i0 = g0 - meshlet.vertexOffset;
uint i1 = g1 - meshlet.vertexOffset;
uint i2 = g2 - meshlet.vertexOffset;
//debugPrintfEXT("indexes : %i, %i, %i", i0, i1, i2);
if (i0 >= vertexCount || i1 >= vertexCount || i2 >= vertexCount) {
//debugPrintfEXT("erreur : %i %i %i %i,%i, %i, %i, %i, %i, %i",g0, g1, g2, i0, i1, i2, meshlet.nbIndexes, meshlet.nbVertices, firstIndex, meshlet.vertexOffset);
//continue;
}
gl_PrimitiveTriangleIndicesEXT[tri] = uvec3(i0, i1, i2);
}
} else {
SetMeshOutputsEXT(0, 0);
}
//Face de heightmap ou voxel.
} else {
//debugPrintfEXT("Height Map");
SetMeshOutputsEXT(taskData.vertexCount, taskData.primitiveCount);
for (uint i = 0; i < taskData.vertexCount; i++) {
gl_MeshVerticesEXT[i].gl_Position = vec4(outputVertices[pc.currentFrame].vertices[taskData.vertexOffset+i].position, 1);
}
for (uint tri = 0; tri < taskData.primitiveCount; tri++) {
gl_PrimitiveTriangleIndicesEXT[tri*2] = uvec3(tri*4, tri*4+1, tri*4+2);
gl_PrimitiveTriangleIndicesEXT[tri*2+1] = uvec3(tri*4, tri*4+2, tri*4+3);
}
}
}
The driver (the studio driver for developpers on my rtx 5070 TI), rasterize random fragments with random inputs values here :
#version 460
extension GL_ARB_separate_shader_objects : enable
extension GL_EXT_nonuniform_qualifier : enable
extension GL_EXT_debug_printf : enable
define NB_PRIMITIVE_TYPES 6
define MAX_FRAMES_IN_FLIGHT 2
const uint MAX_TEXTURES = 1024;
layout (push_constant) uniform PushConstant {
layout(offset=136) int currentImageIndex;
} pc;
struct MaterialData {
uint diffuseTextureIndex;
uint specularTextureIndex;
uint normalTextureIndex;
uint metalnessTextureIndex;
uint roughnessTextureIndex;
uint aoTextureIndex;
uint emissiveTextureIndex;
uint materialType;
uint taskOffset;
uint nbVertices;
uint nbIndexes;
int instanceGroupId;
uint vertsInstanceSet;
uint materialId;
uint nbBuffers;
int reflectable;
int refractable;
int pad;
};
layout (std430, set = 0, binding = 10) buffer MaterialDataSSBO {
MaterialData materialData;
} materialDataBuffer[NB_PRIMITIVE_TYPES * MAX_FRAMES_IN_FLIGHT];
layout(set = 0, binding = 11) uniform sampler2D diffuseTextures[MAX_TEXTURES];
layout(set = 1, binding = 0) uniform sampler2D specularTextures[MAX_TEXTURES];
layout(set = 2, binding = 0) uniform sampler2D normalTextures[MAX_TEXTURES];
layout(set = 3, binding = 0) uniform sampler2D metalnessTextures[MAX_TEXTURES];
layout(set = 4, binding = 0) uniform sampler2D roughnessTextures[MAX_TEXTURES];
layout(set = 5, binding = 0) uniform sampler2D aoTextures[MAX_TEXTURES];
layout(set = 6, binding = 0) uniform sampler2D emissiveTextures[MAX_TEXTURES];
layout(location = 0) in vec4 fragColor;
layout(location = 1) in vec2 fragTexCoord;
layout(location = 2) in vec3 normal;
layout(location = 3) flat in int materialID;
layout(location = 4) flat in int primitiveType;
layout(location = 5) flat in int currentFrame;
layout(location = 6) flat in int flag;
layout(location = 0) out vec4 outColor;
void main() {
float epsilon = 0.0000001;
if (flag == 0 || flag > 1 || flag < 0 || materialID < 0 || materialID >= 7 || primitiveType != 3 || currentFrame < 0 || currentFrame >= MAX_FRAMES_IN_FLIGHT
|| fragColor.r < 1 - epsilon || fragColor.g < 1 - epsilon || fragColor.g < 1 - epsilon || fragColor.a < 1 - epsilon) {
outColor = vec4(0, 0, 0, 0);
return;
}
/*if (fragColor.r != 1 || fragColor.g != 1 || fragColor.b != 1)
debugPrintfEXT("Flag : %i %i %i %i %v4f", flag, primitiveType, materialID, currentFrame, fragColor);*/
//if (materialID > 6)
//debugPrintfEXT("material id %i ", materialID);
/*if (primitiveType > 6 || currentFrame > MAX_FRAMES_IN_FLIGHT) {
outColor = vec4(0, 0, 0, 0);
return;
}*/
MaterialData mat = materialDataBuffer[primitiveType * MAX_FRAMES_IN_FLIGHT+currentFrame].materialData[materialID];
vec2 uv = fragTexCoord;
// --- Diffuse ---
vec4 diffuse = fragColor;
//diffuse = vec4(1, 1, 1, 1);
/*if (fragColor.r < 0.99 || fragColor.g < 0.99 || fragColor.b < 0.99) {
//debugPrintfEXT("Error %v4f", fragColor);
outColor = vec4(0, 0, 0, 0);
return;
}*/
//debugPrintfEXT("material id %i, diffuse : %i", materialID, mat.diffuseTextureIndex);
if (mat.diffuseTextureIndex > 0 && mat.diffuseTextureIndex < MAX_TEXTURES) {
diffuse *= texture(diffuseTextures[mat.diffuseTextureIndex-1], uv);
}
//diffuse = vec4(1, 1, 1, 1);
// --- Normal map ---
vec3 N = normalize(normal);
if (mat.normalTextureIndex > 0 && mat.normalTextureIndex < MAX_TEXTURES) {
vec3 nmap = texture(normalTextures[mat.normalTextureIndex-1], uv).xyz * 2.0 - 1.0;
N = normalize(nmap);
}
// --- Specular ---
float specularStrength = 1.0;
if (mat.specularTextureIndex > 0 && mat.specularTextureIndex < MAX_TEXTURES) {
//debugPrintfEXT("Specular");
specularStrength = texture(specularTextures[mat.specularTextureIndex-1], uv).r;
}
// --- AO ---
float ao = 1.0;
if (mat.aoTextureIndex > 0 && mat.aoTextureIndex < MAX_TEXTURES) {
//debugPrintfEXT("AO");
ao = texture(aoTextures[mat.aoTextureIndex-1], uv).r;
}
// --- Emissive ---
vec4 emissive = vec4(0.0);
if (mat.emissiveTextureIndex > 0 && mat.emissiveTextureIndex < MAX_TEXTURES) {
//debugPrintfEXT("Emissive");
emissive = vec4(texture(emissiveTextures[mat.emissiveTextureIndex-1], uv).rgb, 0);
}
// --- Éclairage simple ---
vec3 L = normalize(vec3(0.5, 1.0, 0.3));
float diff = max(dot(-N, L), 0.0);
// Specular simple
vec3 V = normalize(vec3(0,0,1));
vec3 H = normalize(L + V);
float spec = pow(max(dot(N, H), 0.0), 32.0) * specularStrength;
//debugPrintfEXT("texture index : %i, normal : %v3f, fragTexCoord %v2f, color : %v4f, fragColor : %v4f", mat.diffuseTextureIndex, normal, fragTexCoord, diffuse * diff * ao + spec + emissive, fragColor);
//debugPrintfEXT("normal : %v3f", N);
//outColor = diffuse * diff * ao + spec + emissive;
outColor = diffuse;
};
I check everything, if I haven’t had overflow indexes in my mesh shader, or invalid datas, but everyithing is correct so. I’m just wondering if it’s my code (probably) or if it’s a driver bug… It’s strange to have unitialized datas…, as inputs in my fragment shader. It’s something which never happened to me. Even with aligment issue I’ve no random uninitialized input datas in my shader like this. I checked my barrier too, seems to be good.