Xavier NX Dense Optical Flow Weird Result

I am using Jetson Xavier NX, VPI version is 2.4.

I call the VPI dense optical flow, the results are weird.

When the camera is static, VPI’s optical flow still outputs flow images with a lot of noise. This is
not normal and very different from the results of CPU version Opencv.

My VPI’s backend is NVENC. I noticed that in the sample code of VPI 2.4, the backend is OFA. However, on Xavier NX, OFA backend is not supported. Are the abnormal results caused by the backend difference? (Just my speculation)
Can I use dense optical flow with the NVENC backend in VPI 2.4 to get correct results, or are there any misuses in my code?

The attachments are the code and results.

Thanks a lot.
results

import cv2
import sys
import vpi
import numpy as np
from os import path
from argparse import ArgumentParser
from contextlib import contextmanager
import time


# ----------------------------
# Some utility functions

def process_motion_vectors(mv):
    with mv.rlock_cpu() as data:
        # convert S10.5 format to float
        flow = np.float32(data)/(1<<5)
        
    # with mv.rlock():
    #     # convert S10.5 format to float
    #     flow = np.float32(mv.cpu())/(1<<5)

    # Create an image where the motion vector angle is
    # mapped to a color hue, and intensity is proportional
    # to vector's magnitude
    print("flow:",flow.shape)
    magnitude, angle = cv2.cartToPolar(flow[:,:,0], flow[:,:,1], angleInDegrees=True)
    hsv = np.ndarray([flow.shape[0], flow.shape[1], 3], np.float32)
    
    #################
    # magnitude, angle = cv2.cartToPolar(flow[:,:,0], flow[:,:,1])
    

    # hsv[..., 0] = angle * 180 / np.pi / 2
    # hsv[..., 2] = cv2.normalize(magnitude, None, 0, 255, cv2.NORM_MINMAX)
    
    # motion_flow = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
    # motion_flow = cv2.cvtColor(motion_flow, cv2.COLOR_BGR2GRAY)
    ##################
    
    
    clip = 5.0
    cv2.threshold(magnitude, clip, clip, cv2.THRESH_TRUNC, magnitude)

    # build the hsv image
    hsv[:,:,0] = angle / 2
    hsv[:,:,1] = np.ones((angle.shape[0], angle.shape[1]), np.float32)
    hsv[:,:,2] = magnitude / clip

    ##Convert HSV to BGR8
    bgr = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
    return np.uint8(bgr*255)
    #return motion_flow


# ----------------------------
# Parse command line arguments

parser = ArgumentParser()
parser.add_argument('backend', choices=['nvenc'],
                    help='Backend to be used for processing')

parser.add_argument('input',
                    help='Input video to be processed')

parser.add_argument('quality', choices=['low', 'medium', 'high'],
                    help='Quality setting')

args = parser.parse_args();

assert args.backend == 'nvenc'
backend = vpi.Backend.NVENC

if args.quality == "low":
    quality = vpi.OptFlowQuality.LOW
elif args.quality == "medium":
    quality = vpi.OptFlowQuality.MEDIUM
else:
    assert args.quality == "high"
    quality = vpi.OptFlowQuality.HIGH

# -----------------------------
# Open input and output videos

inVideo = cv2.VideoCapture(0)

if int(cv2.__version__.split('.')[0]) >= 3:
    extOutputVideo = '.mp4'
    fourcc = cv2.VideoWriter_fourcc(*'avc1')
    inSize = (int(inVideo.get(cv2.CAP_PROP_FRAME_WIDTH)), int(inVideo.get(cv2.CAP_PROP_FRAME_HEIGHT)))
    fps = inVideo.get(cv2.CAP_PROP_FPS)
else:
    # MP4 support with OpenCV-2.4 has issues, we'll use
    # avi/mpeg instead.
    extOutputVideo = '.avi'
    fourcc = cv2.cv.CV_FOURCC('M','P','E','G')
    inSize = (int(inVideo.get(cv2.cv.CV_CAP_PROP_FRAME_WIDTH)), int(inVideo.get(cv2.cv.CV_CAP_PROP_FRAME_HEIGHT)))
    fps = inVideo.get(cv2.cv.CV_CAP_PROP_FPS)

if backend == vpi.Backend.NVENC:
    # NVENC always returns 1/4th resolution
    outSize = (inSize[0]//4, inSize[1]//4)
else:
    outSize = inSize

outVideo = cv2.VideoWriter('denseoptflow_mv_python'+str(sys.version_info[0])+'_'+args.backend+extOutputVideo,
                            fourcc, fps, outSize)

#---------------------------------
# Main processing loop

prevFrame = None

idFrame = 0
ksize = (3, 3)
sigmaX = 2
while True:
    # Read one input frame
    ret, cvFrame = inVideo.read()


    #cvFrame_bl = cv2.medianBlur(cvFrame, 5)
   # cvFrame = cv2.resize(cvFrame,None,fx=0.25,fy=0.25)
   # print(cvFrame_bl.shape)
    
    #cvFrame_bl = cv2.cvtColor(cvFrame_bl, cv2.COLOR_BGR2GRAY)
    if not ret:
        break

    # Convert it to NV12_ER format to be used by VPI
    # No single backend can convert from OpenCV's BGR8 to NV12_ER_BL
    # required by the algorithm. We must do in two steps using CUDA and VIC.
    curFrame = vpi.asimage(cvFrame, vpi.Format.BGR8) \
                .convert(vpi.Format.NV12_ER, backend=vpi.Backend.CUDA) \
                .convert(vpi.Format.NV12_ER_BL, backend=vpi.Backend.VIC)

    # Need at least 2 frames to start processing
    if prevFrame is not None:
        print("Processing frame {}".format(idFrame))

        # Calculate the motion vectors from previous to current frame
        start_t = time.time()
        with backend:
            motion_vectors = vpi.optflow_dense(prevFrame, curFrame, quality = vpi.OptFlowQuality.LOW)
        end_t = time.time()
        print(f"optical flow elapsed:{(end_t - start_t)*1000} ms")
        # Turn motion vectors into an image
        motion_image = process_motion_vectors(motion_vectors)

        cv2.imshow('motion_image',motion_image)
        cv2.imshow('cvFrame',cvFrame)
        cv2.waitKey(1)
        # Save it to output video
        #outVideo.write(motion_image)

    # Prepare next iteration
    prevFrame = curFrame
    idFrame += 1

# vim: ts=8:sw=4:sts=4:et:ai

Hi,

Could you try to adjust the parameter of dense optical flow?
For example, using low quality to see if improved.

Thanks.

I have tried all three qualities, there is not much difference. The noise still exist.

Hi,

Could you check if you can get the same output as the document below?
The sample uses NVENC can be found in the VPI 2.3 document:

https://docs.nvidia.com/vpi/2.3/algo_optflow_dense.html

Thanks.

This topic was automatically closed 14 days after the last reply. New replies are no longer allowed.