I am using Jetson Xavier NX, VPI version is 2.4.
I call the VPI dense optical flow, the results are weird.
When the camera is static, VPI’s optical flow still outputs flow images with a lot of noise. This is
not normal and very different from the results of CPU version Opencv.
My VPI’s backend is NVENC. I noticed that in the sample code of VPI 2.4, the backend is OFA. However, on Xavier NX, OFA backend is not supported. Are the abnormal results caused by the backend difference? (Just my speculation)
Can I use dense optical flow with the NVENC backend in VPI 2.4 to get correct results, or are there any misuses in my code?
The attachments are the code and results.
Thanks a lot.
import cv2
import sys
import vpi
import numpy as np
from os import path
from argparse import ArgumentParser
from contextlib import contextmanager
import time
# ----------------------------
# Some utility functions
def process_motion_vectors(mv):
with mv.rlock_cpu() as data:
# convert S10.5 format to float
flow = np.float32(data)/(1<<5)
# with mv.rlock():
# # convert S10.5 format to float
# flow = np.float32(mv.cpu())/(1<<5)
# Create an image where the motion vector angle is
# mapped to a color hue, and intensity is proportional
# to vector's magnitude
print("flow:",flow.shape)
magnitude, angle = cv2.cartToPolar(flow[:,:,0], flow[:,:,1], angleInDegrees=True)
hsv = np.ndarray([flow.shape[0], flow.shape[1], 3], np.float32)
#################
# magnitude, angle = cv2.cartToPolar(flow[:,:,0], flow[:,:,1])
# hsv[..., 0] = angle * 180 / np.pi / 2
# hsv[..., 2] = cv2.normalize(magnitude, None, 0, 255, cv2.NORM_MINMAX)
# motion_flow = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
# motion_flow = cv2.cvtColor(motion_flow, cv2.COLOR_BGR2GRAY)
##################
clip = 5.0
cv2.threshold(magnitude, clip, clip, cv2.THRESH_TRUNC, magnitude)
# build the hsv image
hsv[:,:,0] = angle / 2
hsv[:,:,1] = np.ones((angle.shape[0], angle.shape[1]), np.float32)
hsv[:,:,2] = magnitude / clip
##Convert HSV to BGR8
bgr = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
return np.uint8(bgr*255)
#return motion_flow
# ----------------------------
# Parse command line arguments
parser = ArgumentParser()
parser.add_argument('backend', choices=['nvenc'],
help='Backend to be used for processing')
parser.add_argument('input',
help='Input video to be processed')
parser.add_argument('quality', choices=['low', 'medium', 'high'],
help='Quality setting')
args = parser.parse_args();
assert args.backend == 'nvenc'
backend = vpi.Backend.NVENC
if args.quality == "low":
quality = vpi.OptFlowQuality.LOW
elif args.quality == "medium":
quality = vpi.OptFlowQuality.MEDIUM
else:
assert args.quality == "high"
quality = vpi.OptFlowQuality.HIGH
# -----------------------------
# Open input and output videos
inVideo = cv2.VideoCapture(0)
if int(cv2.__version__.split('.')[0]) >= 3:
extOutputVideo = '.mp4'
fourcc = cv2.VideoWriter_fourcc(*'avc1')
inSize = (int(inVideo.get(cv2.CAP_PROP_FRAME_WIDTH)), int(inVideo.get(cv2.CAP_PROP_FRAME_HEIGHT)))
fps = inVideo.get(cv2.CAP_PROP_FPS)
else:
# MP4 support with OpenCV-2.4 has issues, we'll use
# avi/mpeg instead.
extOutputVideo = '.avi'
fourcc = cv2.cv.CV_FOURCC('M','P','E','G')
inSize = (int(inVideo.get(cv2.cv.CV_CAP_PROP_FRAME_WIDTH)), int(inVideo.get(cv2.cv.CV_CAP_PROP_FRAME_HEIGHT)))
fps = inVideo.get(cv2.cv.CV_CAP_PROP_FPS)
if backend == vpi.Backend.NVENC:
# NVENC always returns 1/4th resolution
outSize = (inSize[0]//4, inSize[1]//4)
else:
outSize = inSize
outVideo = cv2.VideoWriter('denseoptflow_mv_python'+str(sys.version_info[0])+'_'+args.backend+extOutputVideo,
fourcc, fps, outSize)
#---------------------------------
# Main processing loop
prevFrame = None
idFrame = 0
ksize = (3, 3)
sigmaX = 2
while True:
# Read one input frame
ret, cvFrame = inVideo.read()
#cvFrame_bl = cv2.medianBlur(cvFrame, 5)
# cvFrame = cv2.resize(cvFrame,None,fx=0.25,fy=0.25)
# print(cvFrame_bl.shape)
#cvFrame_bl = cv2.cvtColor(cvFrame_bl, cv2.COLOR_BGR2GRAY)
if not ret:
break
# Convert it to NV12_ER format to be used by VPI
# No single backend can convert from OpenCV's BGR8 to NV12_ER_BL
# required by the algorithm. We must do in two steps using CUDA and VIC.
curFrame = vpi.asimage(cvFrame, vpi.Format.BGR8) \
.convert(vpi.Format.NV12_ER, backend=vpi.Backend.CUDA) \
.convert(vpi.Format.NV12_ER_BL, backend=vpi.Backend.VIC)
# Need at least 2 frames to start processing
if prevFrame is not None:
print("Processing frame {}".format(idFrame))
# Calculate the motion vectors from previous to current frame
start_t = time.time()
with backend:
motion_vectors = vpi.optflow_dense(prevFrame, curFrame, quality = vpi.OptFlowQuality.LOW)
end_t = time.time()
print(f"optical flow elapsed:{(end_t - start_t)*1000} ms")
# Turn motion vectors into an image
motion_image = process_motion_vectors(motion_vectors)
cv2.imshow('motion_image',motion_image)
cv2.imshow('cvFrame',cvFrame)
cv2.waitKey(1)
# Save it to output video
#outVideo.write(motion_image)
# Prepare next iteration
prevFrame = curFrame
idFrame += 1
# vim: ts=8:sw=4:sts=4:et:ai