Slight difference between original image from OpenCV (HTTP) and DeepStream gRPC

Please provide complete information as applicable to your setup.

• Hardware Platform (Jetson / GPU) 4090
• DeepStream Version 7.0
• JetPack Version (valid for Jetson only)
• TensorRT Version
• NVIDIA GPU Driver Version (valid for GPU only)
• Issue Type( questions, new requirements, bugs)
• How to reproduce the issue ? (This is for bugs. Including which sample app is using, the configuration files content, the command line used and other details for reproducing)
• Requirement details( This is for new requirement. Including the module name-for which plugin or for which sample application, the function description)

import os
import numpy as np
import tensorrt as trt
import pycuda.driver as cuda
import pycuda.autoinit
import triton_python_backend_utils as pb_utils
import torch
import torchvision.ops as ops
import cv2

TRT_LOGGER = trt.Logger(trt.Logger.WARNING)


class TritonPythonModel:

    def initialize(self, args):
        self.logger = pb_utils.Logger

    def execute(self, requests):
        responses = []
        for request in requests:
            try:
                in_tensor = pb_utils.get_input_tensor_by_name(request, "INPUT0")
                input_data = in_tensor.as_numpy()  # (B, 3, H, W), float32, RGB [0-255]

                B = input_data.shape[0]
                orig_shapes = []
                original_images_rgb = []
                images_rgb_for_yolox = []

                for i in range(B):
                    img_chw = input_data[i]  # (3, H, W)
                    img_hwc = np.transpose(img_chw, (1, 2, 0))  # (H, W, 3), RGB
                    original_images_rgb.append(img_hwc.copy())
                    images_rgb_for_yolox.append(img_hwc)
                    orig_shapes.append(img_hwc.shape)

                    # 🖼️ SAVE INPUT IMAGE (RGB) for debugging
                    # Input is RGB float32 [0-255] → convert to uint8
                    img_uint8 = img_hwc.astype(np.uint8)  # Still RGB!
                    # OpenCV saves in BGR, so convert RGB → BGR
                    img_bgr = cv2.cvtColor(img_uint8, cv2.COLOR_RGB2BGR)
                    save_path = f"/saved/{self.save_count}.jpg"
                    cv2.imwrite(save_path, img_bgr)
                    self.save_count += 1


    def finalize(self):
        self.stream.synchronize()
        self.logger.log_info("✅ YOLOX+ReID CPU backend finalized.")
import cv2
import numpy as np
import tritonclient.http as httpclient

def get_frame_from_video(cap, frame_id, one_based=False):
    """Đọc frame thứ frame_id từ video."""
    if one_based:
        frame_id = frame_id - 1
    cap.set(cv2.CAP_PROP_POS_FRAMES, frame_id)
    ret, frame = cap.read()
    if not ret:
        raise ValueError(f"Không đọc được frame {frame_id}")
    return frame


def main():
    # ---- Cấu hình ----
    reid_file = "/ssd1/lelong/pytorch/AI_CITY/PoseTrack/result/reid/scene_094/camera_0961.npy"
    bbox_file = "/ssd1/lelong/pytorch/AI_CITY/PoseTrack/result/detection/scene_094/camera_0961.txt"
    video_path = "/ssd1/lelong/pytorch/AI_CITY/PoseTrack/dataset/test/scene_094/camera_0961/video.mp4"
    triton_url = "192.168.1.90:32000"
    model_name = "yolox_reid_ensemble"  # Model ensemble đã triển khai

    # ---- Load dữ liệu ----
    reid_feats_saved = np.load(reid_file, mmap_mode="r")  # (N, D)
    detections = read_detections(bbox_file)
    cap = cv2.VideoCapture(video_path)

    # ---- Chọn 1 mẫu để test ----
    idx = 2
    frame_id, gt_bbox, conf = detections[idx]
    feat_saved = reid_feats_saved[idx]  # (D,)

    # Lấy frame full (frame_id trong file là 1-based hay 0-based?)
    # Giả sử file txt dùng 1-based → chuyển sang 0-based
    frame = get_frame_from_video(cap, frame_id, one_based=False)

    # Chuẩn bị input cho ensemble: (1, 3, H, W), FP32 [0-255]
    input_data = np.expand_dims(np.transpose(frame[:, :, ::-1], (2, 0, 1)), axis=0).astype(np.float32)

    # Gửi tới Triton ensemble
    triton_client = httpclient.InferenceServerClient(url=triton_url)
    input_tensor = httpclient.InferInput("INPUT0", input_data.shape, "FP32")
    input_tensor.set_data_from_numpy(input_data)
    outputs = [
        httpclient.InferRequestedOutput("OUTPUT0"),
        httpclient.InferRequestedOutput("REID_EMBEDDINGS")
    ]
    try:
        result = triton_client.infer(model_name=model_name, inputs=[input_tensor], outputs=outputs)
        dets = result.as_numpy("OUTPUT0")[0]        # (N, 6)
        reids = result.as_numpy("REID_EMBEDDINGS")[0]  # (N, D)
    except Exception as e:
        print("❌ Lỗi khi gọi model ensemble:")
        print(e)
        cap.release()
        return

if __name__ == "__main__":
    main()

Above is the Triton HTTP and Python backend code I use to save images (the input image is in RGB, C, H, W format).

import os
import numpy as np
import tensorrt as trt
import triton_python_backend_utils as pb_utils
import torch
import torchvision.ops as ops
import cupy as cp
from torchvision.ops import roi_align
import cv2  # <-- Thêm để lưu ảnh

TRT_LOGGER = trt.Logger(trt.Logger.WARNING)


class TritonPythonModel:

    def initialize(self, args):
        self.logger = pb_utils.Logger


    def execute(self, requests):
        responses = []
        for request in requests:
            try:
                in_tensor = pb_utils.get_input_tensor_by_name(request, "INPUT0")
                input_dlpack = in_tensor.to_dlpack()
                input_torch = torch.from_dlpack(input_dlpack)
                if not input_torch.is_cuda:
                    input_torch = input_torch.cuda()

                # Ensure input is float32 in [0, 255]
                if input_torch.dtype == torch.uint8:
                    input_torch = input_torch.float()
                elif input_torch.dtype == torch.float32:
                    if input_torch.max() <= 1.0:
                        input_torch = input_torch * 255.0
                else:
                    input_torch = input_torch.float()

                B = input_torch.shape[0]
                if not (1 <= B <= self.max_batch_size):
                    raise ValueError(f"Batch size must be 1–{self.max_batch_size}, got {B}")

                # 🔥 CRITICAL: DeepStream sends RGB → convert to BGR for YOLOX
                input_torch_bgr = input_torch.clone()
                input_torch_bgr[:, [0, 2], :, :] = input_torch_bgr[:, [2, 0], :, :]  # RGB → BGR

                orig_shapes = [(input_torch[i].shape[1], input_torch[i].shape[2]) for i in range(B)]
                # Keep original RGB for ReID!
                original_images_rgb = [input_torch[i] for i in range(B)]

                # 🔽 DEBUG: Save ALL original RGB images as BGR (for correct OpenCV display) 🔽
                for img_rgb in original_images_rgb:
                    # img_rgb: (3, H, W), float32, [0,255], RGB
                    img_hwc_rgb = img_rgb.permute(1, 2, 0).cpu().numpy()  # (H, W, 3), RGB
                    img_hwc_bgr = img_hwc_rgb[:, :, ::-1]  # RGB → BGR
                    img_uint8 = np.clip(img_hwc_bgr, 0, 255).astype(np.uint8)
                    save_path = os.path.join(self.debug_save_dir, f"{self.count}.jpg")
                    cv2.imwrite(save_path, img_uint8)
                    self.count += 1
                # 🔼 DEBUG: Save done 🔼


    def finalize(self):
        self.logger.log_info("✅ YOLOX+ReID backend finalized.")

config.pbtxt

name: "yolox_reid_ensemble"
backend: "python"
max_batch_size: 4

input [
  {
    name: "INPUT0"
    data_type: TYPE_FP32
    #format: FORMAT_NCHW
    #dims: [3, 1080, 1920]
    dims: [1080, 1920, 3]
  }
]

output [
  {
    name: "OUTPUT0"
    data_type: TYPE_FP32
    dims: [-1, 6]
  },
  {
    name: "REID_EMBEDDINGS"
    data_type: TYPE_FP32
    dims: [-1, 2048]
  }
]

instance_group [
  {
    kind: KIND_GPU
    count: 1
    gpus: [0]
  }
]

parameters: {
  key: "FORCE_CPU_ONLY_INPUT_TENSORS"
  value: {
    string_value:"no"
  }
}

My input video is H265 and 1920x1080. My nvinferserver configuration

infer_config {
  unique_id: 1
  gpu_ids: [0]
  max_batch_size: 4
  
  backend {
    triton {
      model_name: "yolox_reid_ensemble"
      version: -1
      grpc {
        url: "192.168.1.90:32001"
        enable_cuda_buffer_sharing: true
      }
    }
    disable_warmup: true
  }

  preprocess {
    network_format: IMAGE_FORMAT_RGB 
    tensor_order: TENSOR_ORDER_LINEAR
    maintain_aspect_ratio: 0
    frame_scaling_hw: FRAME_SCALING_HW_DEFAULT
    frame_scaling_filter: 0
    symmetric_padding: 0
    normalize {
        scale_factor: 1.0
        channel_offsets: [0.0,0.0,0.0]
    }
  }

  postprocess {
    other {}

  }


  extra {
    copy_input_to_host_buffers: false
  }

} 

input_control {
  process_mode : PROCESS_MODE_FULL_FRAME
  #operate_on_gie_id: 1
  #operate_on_class_ids: [0]
  interval : 0
}

output_control {
  output_tensor_meta: true
}
  

When I save the image 0.jpg and compare it with the 0.jpg from the Triton HTTP version, there’s a slight difference.
If the input is the 0.jpg from HTTP, my ReID cosine similarity compared to the PyTorch version is 0.9999, but if I use the image from DeepStream and Triton, the cosine similarity drops to 0.97.

When comparing the two saved images from the two versions with

import cv2
import numpy as np

def compare_images_pixel_diff(img1_path, img2_path, threshold=0):
    """
    Compare two images and return:
    - Number of differing pixels (absolute difference > threshold)
    - MSE and PSNR
    - Maximum absolute pixel difference (averaged across channels)
    - Mean absolute pixel difference (MAE)
    - Color and grayscale difference images
    """
    img1 = cv2.imread(img1_path).astype(np.float32)
    img2 = cv2.imread(img2_path).astype(np.float32)

    if img1 is None or img2 is None:
        raise FileNotFoundError("One or both image files not found.")

    if img1.shape != img2.shape:
        print(f"Warning: Image dimensions differ! {img1.shape} vs {img2.shape}")
        h, w = max(img1.shape[0], img2.shape[0]), max(img1.shape[1], img2.shape[1])
        img1 = cv2.resize(img1, (w, h), interpolation=cv2.INTER_LINEAR)
        img2 = cv2.resize(img2, (w, h), interpolation=cv2.INTER_LINEAR)

    # Compute absolute pixel-wise difference (per channel)
    diff = np.abs(img1 - img2)  # Shape: (H, W, 3)

    # Convert to grayscale by averaging across channels
    diff_gray = np.mean(diff, axis=2)  # Shape: (H, W)

    # Compute metrics
    num_diff_pixels = np.sum(diff_gray > threshold)
    max_diff = np.max(diff_gray)          # Max deviation (mean across channels)
    mean_diff = np.mean(diff_gray)        # Mean Absolute Error (MAE)

    mse = np.mean((img1 - img2) ** 2)
    if mse == 0:
        psnr = float('inf')
    else:
        psnr = 10 * np.log10((255.0 ** 2) / mse)

    return {
        "num_diff_pixels": int(num_diff_pixels),
        "max_diff": float(max_diff),
        "mean_diff": float(mean_diff),
        "mse": float(mse),
        "psnr": float(psnr),
        "diff_image": diff.astype(np.uint8),
        "diff_gray": diff_gray.astype(np.uint8)
    }

# === Usage ===
if __name__ == "__main__":
    img1_path = "/ssd1/lelong/mge_ai_cloud/triton/saved/1.jpg"
    img2_path = "/ssd1/lelong/mge_ai_cloud/triton/saved_CPU/0.jpg"

    try:
        result = compare_images_pixel_diff(img1_path, img2_path, threshold=1.0)

        print(f"🔹 Number of differing pixels (difference > 1): {result['num_diff_pixels']:,}")
        print(f"🔹 Max pixel deviation (mean across channels): {result['max_diff']:.2f}")
        print(f"🔹 Mean absolute error (MAE): {result['mean_diff']:.4f}")
        print(f"🔹 MSE: {result['mse']:.4f}")
        print(f"🔹 PSNR: {result['psnr']:.2f} dB")

        if result['num_diff_pixels'] == 0:
            print("✅ Images are identical!")
        else:
            print("⚠️ Images differ.")

        cv2.imwrite("diff_color.jpg", result['diff_image'])
        cv2.imwrite("diff_gray.jpg", result['diff_gray'])
        print("🖼️ Difference images saved: diff_color.jpg, diff_gray.jpg")

    except Exception as e:
        print(f"Error: {e}")
🔹 Number of differing pixels (difference > 1): 400,791
🔹 Max pixel deviation (mean across channels): 10.00
🔹 Mean absolute error (MAE): 0.7882
🔹 MSE: 1.5493
🔹 PSNR: 46.23 dB
⚠️ Images differ.

The input resolution is 1920×1080, and the nvinferserver calls Triton with a config.pbtxt where the input is also defined as (3×1080×1920). So in theory, the input should remain unchanged.
However, it seems that some resizing is happening somewhere, because when I change the frame_scaling_filter values, the saved input image—although still similar—changes depending on the frame_scaling_filter setting. Currently, I using new streammux, it doesn’t has width, height property.

test result:

frame_scaling_filter: 0, 2,3,4,5,6 → Cosine similarity: 0.96118206

frame_scaling_filter: 1 → Cosine similarity: 0.8941058

Thanks

Thanks for the sharing! I are checking. will get back to you.

  1. Thanks for the sharing! I can reproduce a “frame_scaling_filter: 1” issue, namely, even the out relution is the same, the content will be modified if using bilinear algo. we will continue to investigate. currently you can use other algo instead.
  2. About the accuracy drop issue, whch sample are you testing or referring to? what is the complete media pipeline? cou you dump the pipeline graph by referring this faq?
  3. Please refer to my test to avoid accuracy loss. the source is a RGBA image. I dumped RGB data after scaling(using nearest algo) in nvinferserver, after comparing the RGB with source RGBA, only alpha part is removed. Hence in Python backend, the R,B,G data will be the same with source.
USE_NEW_NVSTREAMMUX=yes  gst-launch-1.0 filesrc location=/home/rec/raw/960x544-rgba.rgb ! videoparse width=960 height=544 format=rgba framerate=1/1 ! nvvideoconvert  ! 'video/x-raw(memory:NVMM),format=RGBA'   ! mux.sink_0 nvstreammux name=mux batch-size=1   ! nvinferserver config-file-path=./dstest1_pgie_nvinferserver_config.txt ! nvvideoconvert ! 'video/x-raw(memory:NVMM),format=RGBA' ! nvdsosd ! nvvideoconvert ! 'video/x-raw,format=I420' ! jpegenc ! filesink location=7.jpeg

nvinferserver is opensource. Here is the dump code in CropSurfaceConverter::resizeBatchinfer_preprocess.cpp (25.1 KB)

My pipeline is very simple. It consists of uridecodebin → streammux → pgie → sgie → fakesink.

The original video is 1920x1080, encoded in H.265.
In the PGIE, due to some computations, I use a Python backend to output bounding boxes for objects along with their corresponding ReID features (with max_object padding set to 40 for DeepStream batch compatibility).

Then, in the PGIE callback, I extract the tensor output from the metadata and create new objects using pyds.nvds_add_obj_meta_to_frame, including mask_params to store the ReID information.

In the SGIE, I use an HRNet model to detect keypoints for the person bounding boxes using nvinfer with engine

reid_yolox.svg.zip (4.9 KB)

  1. I’m using frame_scaling_filter = 0, but it seems there’s still an issue — the input image appears to be slightly altered (even though I’m already using frame_scaling_filter, right?). As I mentioned above, the ReID cosine similarity dropped from 0.9999 to 0.96.
  2. How can I reduce the inaccuracy with my pipeline?
    What did you mean by “Please refer to my test to avoid accuracy loss”?
    When sending images to Triton, the input isn’t RGBA, right? It seems the input is actually NV12.
>>> New pad created with caps: video/x-raw(memory:NVMM), format=(string)NV12, width=(int)1920, height=(int)1080, interlace-mode=(string)progressive, multiview-mode=(string)mono, multiview-flags=(GstVideoMultiviewFlagsSet)0:ffffffff:/right-view-first/left-flipped/left-flopped/right-flipped/right-flopped/half-aspect/mixed-mono, pixel-aspect-ratio=(fraction)1/1, colorimetry=(string)bt709, framerate=(fraction)25/1, nvbuf-memory-type=(string)nvbuf-mem-cuda-device, gpu-id=(int)0
>>> Structure name: video/x-raw
>>> Memory features: memory:NVMM
>>> New pad created with caps: video/x-raw(memory:NVMM), format=(string)NV12, width=(int)1920, height=(int)1080, interlace-mode=(string)progressive, multiview-mode=(string)mono, multiview-flags=(GstVideoMultiviewFlagsSet)0:ffffffff:/right-view-first/left-flipped/left-flopped/right-flipped/right-flopped/half-aspect/mixed-mono, pixel-aspect-ratio=(fraction)1/1, colorimetry=(string)bt709, framerate=(fraction)25/1, nvbuf-memory-type=(string)nvbuf-mem-cuda-device, gpu-id=(int)0
>>> Structure name: video/x-raw
>>> Memory features: memory:NVMM
  1. Do you need the model weights I’m using, the Triton Python backend code, the DeepStream probe function, and the files I use for testing?

My video:

ffprobe -v error -select_streams v:0 -show_entries stream=pix_fmt -of default=noprint_wrappers=1 /Users/longle/Documents/test/nvidia/videos/scene_094/video_960.mp4

pix_fmt=yuvj420p -> Hope this helps you out.

If there are any related bugs that have already been fixed in DeepStream version 7.1 or 8.0, I can give them a try

deepstream 7.1 releases notes; nvvideoconvert to support UYVY (8-bit YCbCr-4:2:2) on x86/dGPU

deepstream 8.0 releases notes: nvvideoconvert: Support U16 formats
related or not.

Thanks

  1. In DeepStream, the format after decoding is NV12. In nvinferserver, the input format will be converted to RBA with NvBufSurfTransformAsync() for normalization.
  2. regarding my test, I mean, maybe the format conversion causes the accuracy loss. in my test, the input format is RGBA. in python backend, the R,G,B data is still the same with intput. you can dump the intermediate data to compare. Here is the faq Dump NV12 NvBufSurface into a YUV file. regarding nvvideoconvert, nvvideoconvert support more format conversion in latter versions, but these formats are not used in your python code.

“we will continue to investigate. currently you can use other algo instead.”

When you find the cause, could you please share a patch to fix it? Thank you.

  1. After checking ,even if the dest resolution is the same with the input, NvBufSurfTransform() which is used to resize will do interpolation scaling for the different colorformat with the set scaling filter. “frame_scaling_filter=0” represents nearest interpolation. AYK, Using this interpolation , if the resolution does not change, the result will be the same as the input.
  2. regarding “frame_scaling_filter: 1” issue, based on the analysis in step1, it is not a bug. if you don’t want to modify the content, you can use nearest interpolation instead.
  3. regarding “Cosine similarity” decrease from 0.9999 to 0.97, Is this still an DeepStream issue to support? Thanks! as writen in my last comment, Please compare the two pipeline and compare the intermediate data to find which conversion causes the accuracy loss. to extrem, for example, you can use RGBA source in both DeepStream and Python, there will no any date loss. Please refer to my test on Oct 10.

There is no update from you for a period, assuming this is not an issue anymore. Hence we are closing this topic. If need further support, please open a new one. Thanks.