After adding a sensor using RTSP in Sensor Management page, RTSP URLs generated by RTSP Proxy on the Dashboard page.
When I just added a sensor, these RTSP URLs can be played directly using VLC. After a day, all of them can’t be played using VLC anymore, but the Live Streams page can still play the stream (it should be using these RTSP URLs to convert to WebRTC).
Can you share your setup information? Can you share the vst log?
My Environment:
Jetson Orin NX 16GB
Jetpack 6.0 [L4T 36.3.0]
Cam setting:
VST config:
{
"network":
{
"http_port": 81,
"server_domain_name":"",
"stunurl_list": ["stun.l.google.com:19302","stun1.l.google.com:19302"],
"static_turnurl_list": [],
"use_coturn_auth_secret": false,
"coturn_turnurl_list_with_secret": [],
"use_twilio_stun_turn": false,
"twilio_account_sid": "",
"twilio_auth_token": "",
"use_reverse_proxy": false,
"reverse_proxy_server_address": "REVERSE_PROXY_SERVER_ADDRESS:100",
"ntp_servers": [],
"max_webrtc_out_connections": 16,
"max_webrtc_in_connections": 16,
"webservice_access_control_list":"",
"rtsp_server_port": -1,
"rtsp_preferred_network_iface": "eth0",
"rtsp_in_base_udp_port_num": -1,
"rtsp_out_base_udp_port_num": -1,
"rtsp_streaming_over_tcp": false,
"rtsp_server_reclamation_client_timeout_sec": 10,
"rx_socket_buffer_size":1000000,
"tx_socket_buffer_size":1000000,
"stream_monitor_interval_secs": 2,
"rtp_udp_port_range" : "31000-31200",
"udp_latency_ms": 200,
"udp_drop_on_latency": false,
"webrtc_latency_ms": 1000,
"enable_frame_drop": true,
"webrtc_video_quality_tunning":
{
"resolution_2160":
{
"bitrate_start" : 20000, "bitrate_range" : [15000,25000],
"qp_range_I" : [0,30], "qp_range_P" : [0,51]
},
"resolution_1440":
{
"bitrate_start" : 10000, "bitrate_range" : [8000,15000],
"qp_range_I" : [10,30], "qp_range_P" : [10,30]
},
"resolution_1080":
{
"bitrate_start" : 5000, "bitrate_range" : [3000,8000],
"qp_range_I" : [10,30], "qp_range_P" : [10,30]
},
"resolution_720":
{
"bitrate_start" : 3000, "bitrate_range" : [2000,5000],
"qp_range_I" : [10,30], "qp_range_P" : [10,30]
},
"resolution_480":
{
"bitrate_start" : 1000, "bitrate_range" : [800,3000],
"qp_range_I" : [10,30], "qp_range_P" : [10,30]
}
},
"webrtc_peer_conn_timeout_sec": 10,
"enable_grpc": false,
"grpc_server_port": "50051",
"webrtc_in_audio_sender_max_bitrate": 128000,
"webrtc_in_video_degradation_preference": "resolution",
"webrtc_in_video_sender_max_framerate": 30,
"remote_vst_address": "",
"webrtc_port_range": {"min":0, "max":0},
"enable_websocket_pingpong": false,
"websocket_keep_alive_ms": 5000
},
"onvif":
{
"device_discovery_timeout_secs":10,
"onvif_request_timeout_secs":20,
"device_discovery_freq_secs":15,
"device_discovery_interfaces": ["eth1"],
"max_devices_supported": 8,
"default_bitrate_kbps": 8000,
"default_framerate": 30,
"default_resolution": "1920x1080",
"default_gov_length": 60
},
"data":
{
"storage_config_file": "/root/vst_release/configs/vst_storage.json",
"storage_threshold_percentage": 95,
"storage_monitoring_frequency_secs": 2,
"nv_streamer_directory_path": "/home/vst/vst_release/streamer_videos/",
"nv_streamer_loop_playback":true,
"nv_streamer_seekable":false,
"nv_streamer_max_upload_file_size_MB": 10000,
"nv_streamer_media_container_supported": ["mp4","mkv"],
"nv_streamer_metadata_container_supported": ["json"],
"nv_streamer_rtsp_server_output_buffer_size_kb": 1000,
"supported_video_codecs": ["h264","h265"],
"supported_audio_codecs": ["pcmu","pcma","mpeg4-generic"],
"enable_aging_policy": false,
"max_video_download_size_MB":1000,
"always_recording": true,
"event_recording": true,
"event_record_length_secs": 10,
"record_buffer_length_secs": 2,
"use_software_path": false,
"use_webrtc_inbuilt_encoder": "h264",
"webrtc_in_fixed_resolution": "1280x720",
"webrtc_in_max_framerate": 30,
"webrtc_in_video_bitrate_thresold_percentage": 50,
"webrtc_in_passthrough": false,
"webrtc_sender_quality": "pass_through",
"enable_rtsp_server_sei_metadata": false,
"enable_proxy_server_sei_metadata": false,
"gpu_indices": [],
"webrtc_out_enable_insert_sps_pps" : true,
"webrtc_out_set_iframe_interval" : 30,
"webrtc_out_set_idr_interval" : 256,
"webrtc_out_min_drc_interval" : 5,
"webrtc_out_encode_fallback_option" : "software",
"device_name" : "VST",
"device_location" : "",
"enable_dec_low_latency_mode": true,
"webrtc_out_default_resolution": ""
},
"notifications":
{
"enable_notification": true,
"use_message_broker": "redis",
"message_broker_topic": "vst.event",
"message_broker_payload_key": "sensor.id",
"message_broker_metadata_topic": "test",
"redis_server_env_var":"localhost:6379",
"kafka_server_address": "localhost:9092"
},
"debug":
{
"enable_perf_logging":true,
"enable_qos_monitoring":true,
"qos_logfile_path":"/root/vst_release/webroot/log/",
"qos_data_capture_interval_sec":1,
"qos_data_publish_interval_sec":5,
"enable_gst_debug_probes":true,
"enable_prometheus":true,
"prometheus_port": "8080",
"enable_highlighting_logs":false,
"enable_debug_apis": true,
"dump_webrtc_input_stats": false,
"enable_frameid_in_webrtc_stream": false,
"enable_network_bandwidth_notification" : false,
"enable_latency_logging": true
},
"overlay":
{
"video_metadata_server": "",
"video_metadata_query_batch_size_num_frames": 300,
"use_video_metadata_protobuf": false,
"enable_gem_drawing": true,
"analytic_server_address": "",
"overlay_text_font_type": "DejaVuSansMono.ttf"
},
"security":
{
"use_https": false,
"use_rtsp_authentication": false,
"use_http_digest_authentication": false,
"use_multi_user": false,
"enable_user_cleanup": false,
"session_max_age_sec": 2592000,
"multi_user_extra_options": ["Secure", "SameSite=none"],
"nv_org_id": "",
"nv_ngc_key": ""
}
}
VST Logs:
vst.log-20241111.gz (1.7 MB)
vst.log-20241112.gz (1.9 MB)
Then I tested it and found that after restarting the container everything worked fine, but I still don’t know why it happened.
So, is it one 100% reproducable issue?
Yes, it’s happened several times.
Is there anything I can do to check? Or do I need to give you any more information to help?
Can you capture the vst log when you try to playback RTSP with VLC? What kind of error are you see from both VST and VLC when you try to playback it?
I use VLC to play the sensor with Id 50c4d2b6-f616-4ae3-b26d-06ef68669f00, VLC just keeps loading (more than 10 mins now) without any message, you can see from the VST log that it is still recording normally.
VST Log:
vst.log (96.7 KB)
Can you have a try if you can playback the RTSP proxy URL with below command?
$ gst-launch-1.0 rtspsrc protocols=tcp location=rtsp://<JETSON_IP>:8554/live/799515b0-a473-4b43-a21f-7709f2a74012 ! decodebin ! nv3dsink sync=false qos=false
Returns:
Setting pipeline to PAUSED ...
Pipeline is live and does not need PREROLL ...
Progress: (open) Opening Stream
Pipeline is PREROLLED ...
Prerolled, waiting for progress to finish...
Progress: (connect) Connecting to rtsp://10.0.0.167:8554/live/50c4d2b6-f616-4ae3-b26d-06ef68669f00
Progress: (open) Retrieving server options
ERROR: from element /GstPipeline:pipeline0/GstRTSPSrc:rtspsrc0: Could not read from resource.
Additional debug info:
../gst/rtsp/gstrtspsrc.c(6523): gst_rtsp_src_receive_response (): /GstPipeline:pipeline0/GstRTSPSrc:rtspsrc0:
Could not receive message. (Timeout while waiting for server response)
ERROR: pipeline doesn't want to preroll.
Setting pipeline to NULL ...
ERROR: from element /GstPipeline:pipeline0/GstRTSPSrc:rtspsrc0: Could not read from resource.
Additional debug info:
../gst/rtsp/gstrtspsrc.c(6621): gst_rtspsrc_try_send (): /GstPipeline:pipeline0/GstRTSPSrc:rtspsrc0:
Could not receive message. (Timeout while waiting for server response)
ERROR: pipeline doesn't want to preroll.
Freeing pipeline ...
vst logs:
[2024-11-14T9:38:48.373Z:281471285717984:network_utils.cpp:772: getProbeMatch] Probe port is still not opened
[2024-11-14T9:38:49.373Z:281471285717984:network_utils.cpp:772: getProbeMatch] Probe port is still not opened
[2024-11-14T9:38:49.523Z:281467384228832:gstmux.cpp:1411: sendEOS] Sending EOS in MUX pipeline for camera id = 29cfc891-36b2-4d33-8b3b-a05308036d11
[2024-11-14T9:38:49.530Z:281467384228832:gstmux.cpp:191: updateDbRow] Success: Update Database with duration 61338 and FPS = 30 for file /root/store/video/29cfc891-36b2-4d33-8b3b-a05308036d11/640x480/2024/11/14/10/1731576052246.mkv with timestamp : Thu Nov 14 09:20:52 2024
[2024-11-14T9:38:49.530Z:281467384228832:storage_management.cpp:487: addFileInProtectList] Added file into the in protect list filePath:/root/store/video/29cfc891-36b2-4d33-8b3b-a05308036d11/640x480/2024/11/14/10/1731576052246.mkv removeOrAdd:0
[2024-11-14T9:38:49.535Z:281467384228832:gstmux.cpp:1379: setNULLState] Transitioning Pipeline into NULL State for camera ID = 29cfc891-36b2-4d33-8b3b-a05308036d11
[2024-11-14T9:38:49.536Z:281467384228832:gstmux.cpp:1395: setNULLState] State change to NULL success for camera ID = 29cfc891-36b2-4d33-8b3b-a05308036d11
[2024-11-14T9:38:49.536Z:281467384228832:gstmux.cpp:1290: setPlayState] Transitioning Pipeline into PLAYING State for camera ID = 29cfc891-36b2-4d33-8b3b-a05308036d11
[2024-11-14T9:38:49.536Z:281467384228832:gstmux.cpp:1311: setPlayState] GST_STATE_CHANGE_ASYNC (for PLAYING state) for camera ID = 29cfc891-36b2-4d33-8b3b-a05308036d11
[2024-11-14T9:38:49.540Z:281467384228832:gstmux.cpp:691: insertRowInDB] [Success:] Insert into Database for file /root/store/video/29cfc891-36b2-4d33-8b3b-a05308036d11/640x480/2024/11/14/10/1731576113617.mkv with timestamp : Thu Nov 14 09:21:53 2024
[2024-11-14T9:38:49.540Z:281467384228832:storage_management.cpp:487: addFileInProtectList] Added file into the in protect list filePath:/root/store/video/29cfc891-36b2-4d33-8b3b-a05308036d11/640x480/2024/11/14/10/1731576113617.mkv removeOrAdd:1
[2024-11-14T9:38:50.373Z:281471285717984:network_utils.cpp:772: getProbeMatch] Probe port is still not opened
[2024-11-14T9:38:50.880Z:281467046849504:stream_monitor.cpp:695: onConnectionTimeout] [streamMonitor] onConnectionTimeout for table_2, url:rtsp://10.0.0.167:8554/live/c707c3f2-fedb-4404-82a4-2d6b0d98f573
[2024-11-14T9:38:50.881Z:281466588097504:stream_monitor.cpp:954: frameMonitorTask] Exited the thread frameMonitorTask name: table_2
[2024-11-14T9:38:51.373Z:281471285717984:network_utils.cpp:772: getProbeMatch] Probe port is still not opened
[2024-11-14T9:38:52.374Z:281471285717984:network_utils.cpp:772: getProbeMatch] Probe port is still not opened
[2024-11-14T9:38:52.546Z:281467358866400:gstmux.cpp:947: event_probe] Initial condition
[2024-11-14T9:38:52.546Z:281467358866400:gstmux.cpp:948: event_probe] Resolution = 1280 x 720
[2024-11-14T9:38:52.546Z:281467358866400:gstmux.cpp:949: event_probe] Framerate = 0 / 1
[2024-11-14T9:38:52.546Z:281467358866400:gstmux.cpp:950: event_probe] Stream Format = avc
[2024-11-14T9:38:52.546Z:281467392682976:gstmux.cpp:814: busWatchFunc] GST_MESSAGE_ASYNC_DONE for camera ID = 29cfc891-36b2-4d33-8b3b-a05308036d11
[2024-11-14T9:38:52.770Z:281467536403424:fps_display.cpp:80: displayFPS] Unique ID = recording_87620200-0206-4772-9dd8-de75465a5e41 FPS = { 14.944919, 15.070065, 14.987581, 14.969339, 14.998051, 14.914609, 15.058314, 14.932319, 14.932069, 15.058712, 14.967078 }
[2024-11-14T9:38:53.374Z:281471285717984:network_utils.cpp:772: getProbeMatch] Probe port is still not opened
[2024-11-14T9:38:54.374Z:281471285717984:network_utils.cpp:772: getProbeMatch] Probe port is still not opened
[2024-11-14T9:38:55.374Z:281471285717984:network_utils.cpp:772: getProbeMatch] Probe port is still not opened
[2024-11-14T9:38:55.881Z:281467046849504:stream_monitor.cpp:690: restartConnection] Retrying rtsp connection for:table_2, retry count:3
[2024-11-14T9:38:56.374Z:281471285717984:network_utils.cpp:772: getProbeMatch] Probe port is still not opened
[2024-11-14T9:38:56.893Z:281467375774688:stream_monitor.cpp:695: onConnectionTimeout] [streamMonitor] onConnectionTimeout for test1114, url:rtsp://10.0.0.167:8554/live/dfc9cf30-38ae-41f7-9714-b6e0b1adfb9d
[2024-11-14T9:38:56.893Z:281467367320544:stream_monitor.cpp:954: frameMonitorTask] Exited the thread frameMonitorTask name: test1114
[2024-11-14T9:38:57.374Z:281471285717984:network_utils.cpp:772: getProbeMatch] Probe port is still not opened
[2024-11-14T9:38:57.901Z:281471805156320:stream_monitor.cpp:1719: qosMeasurementTask] [streamMonitor] Max limit reached for reconnection retry for:table_2
[2024-11-14T9:38:57.901Z:281471805156320:stream_monitor.cpp:526: ~QosMeasurementRecord] ~QosMeasurementRecord - table_2
[2024-11-14T9:38:57.901Z:281471805156320:stream_monitor.cpp:581: stopQoS] [streamMonitor] stopQoS for table_2
[2024-11-14T9:38:57.901Z:281471805156320:stream_monitor.cpp:1092: ~QosRtspClient] ~QosRtspClient: name:table_2, uri:rtsp://10.0.0.167:8554/live/c707c3f2-fedb-4404-82a4-2d6b0d98f573
[2024-11-14T9:38:57.901Z:281471805156320:livevideosource.h:137: Stop] VideoSource::stop
[2024-11-14T9:38:57.903Z:281471805156320:livevideosource.h:149: Stop] VideoSource::stop : done
[2024-11-14T9:38:57.903Z:281471805156320:livevideosource.h:117: ~VideoSource] ~VideoSource
[2024-11-14T9:38:57.903Z:281471805156320:rtspconnectionclient.cpp:122: ~RTSPConnection] ~RTSPConnection
[2024-11-14T9:38:58.374Z:281471285717984:network_utils.cpp:772: getProbeMatch] Probe port is still not opened
[2024-11-14T9:38:59.374Z:281471285717984:network_utils.cpp:772: getProbeMatch] Probe port is still not opened
[2024-11-14T9:39:0.1Z:281471277263840:network_utils.cpp:541: sendProbe] Probe port is still not opened
[2024-11-14T9:39:0.375Z:281471285717984:network_utils.cpp:772: getProbeMatch] Probe port is still not opened
[2024-11-14T9:39:1.375Z:281471285717984:network_utils.cpp:772: getProbeMatch] Probe port is still not opened
[2024-11-14T9:39:1.893Z:281467375774688:stream_monitor.cpp:690: restartConnection] Retrying rtsp connection for:test1114, retry count:2
[2024-11-14T9:39:2.375Z:281471285717984:network_utils.cpp:772: getProbeMatch] Probe port is still not opened
[2024-11-14T9:39:3.375Z:281471285717984:network_utils.cpp:772: getProbeMatch] Probe port is still not opened
[2024-11-14T9:39:3.909Z:281471805156320:stream_monitor.cpp:1182: startRtspClient] starting RtspClient for test1114, uri:rtsp://10.0.0.167:8554/live/dfc9cf30-38ae-41f7-9714-b6e0b1adfb9d
[2024-11-14T9:39:3.909Z:281471805156320:stream_monitor.cpp:526: ~QosMeasurementRecord] ~QosMeasurementRecord - test1114
[2024-11-14T9:39:3.909Z:281471805156320:stream_monitor.cpp:581: stopQoS] [streamMonitor] stopQoS for test1114
[2024-11-14T9:39:3.909Z:281471805156320:stream_monitor.cpp:1092: ~QosRtspClient] ~QosRtspClient: name:test1114, uri:rtsp://10.0.0.167:8554/live/dfc9cf30-38ae-41f7-9714-b6e0b1adfb9d
[2024-11-14T9:39:3.909Z:281471805156320:livevideosource.h:137: Stop] VideoSource::stop
[2024-11-14T9:39:3.915Z:281471805156320:livevideosource.h:149: Stop] VideoSource::stop : done
[2024-11-14T9:39:3.915Z:281471805156320:livevideosource.h:117: ~VideoSource] ~VideoSource
[2024-11-14T9:39:3.915Z:281471805156320:rtspconnectionclient.cpp:122: ~RTSPConnection] ~RTSPConnection
0
[2024-11-14T9:39:3.915Z:281471805156320:stream_monitor.cpp:1085: QosRtspClient] QosRtspClient: name:test1114, uri:rtsp://10.0.0.167:8554/live/dfc9cf30-38ae-41f7-9714-b6e0b1adfb9d
[2024-11-14T9:39:3.916Z:281467367320544:stream_monitor.cpp:931: frameMonitorTask] Started the thread frameMonitorTask name: test1114
[2024-11-14T9:39:4.375Z:281471285717984:network_utils.cpp:772: getProbeMatch] Probe port is still not opened
[2024-11-14T9:39:4.916Z:281467375774688:rtspconnectionclient.cpp:290: sendNextCommand] [CLIENT] Sending Describe command
[2024-11-14T9:39:5.375Z:281471285717984:network_utils.cpp:772: getProbeMatch] Probe port is still not opened
[2024-11-14T9:39:6.375Z:281471285717984:network_utils.cpp:772: getProbeMatch] Probe port is still not opened
[2024-11-14T9:39:6.912Z:281471813610464:stream_monitor.cpp:371: livenessMonitorTask] CURL error:Timeout was reached [28] for url:rtsp://0.0.0.123
[2024-11-14T9:39:7.376Z:281471285717984:network_utils.cpp:772: getProbeMatch] Probe port is still not opened
[2024-11-14T9:39:7.913Z:281471813610464:stream_monitor.cpp:1535: isUrlBlackListed] [streamMonitor] removing from qos-blacklist url:rtsp://10.0.0.167:8554/live/4e748af6-6ddf-42cc-bf2b-c12ee12083f6
[2024-11-14T9:39:8.376Z:281471285717984:network_utils.cpp:772: getProbeMatch] Probe port is still not opened
[2024-11-14T9:39:8.916Z:281471805156320:stream_monitor.cpp:1182: startRtspClient] starting RtspClient for 你好, uri:rtsp://10.0.0.167:8554/live/4e748af6-6ddf-42cc-bf2b-c12ee12083f6
0
[2024-11-14T9:39:8.916Z:281471805156320:stream_monitor.cpp:1085: QosRtspClient] QosRtspClient: name:你好, uri:rtsp://10.0.0.167:8554/live/4e748af6-6ddf-42cc-bf2b-c12ee12083f6
[2024-11-14T9:39:8.916Z:281467046849504:stream_monitor.cpp:931: frameMonitorTask] Started the thread frameMonitorTask name: 你好
[2024-11-14T9:39:9.376Z:281471285717984:network_utils.cpp:772: getProbeMatch] Probe port is still not opened
[2024-11-14T9:39:9.916Z:281466579643360:rtspconnectionclient.cpp:290: sendNextCommand] [CLIENT] Sending Describe command
[2024-11-14T9:39:10.376Z:281471285717984:network_utils.cpp:772: getProbeMatch] Probe port is still not opened
[2024-11-14T9:39:11.376Z:281471285717984:network_utils.cpp:772: getProbeMatch] Probe port is still not opened
[2024-11-14T9:39:12.376Z:281471285717984:network_utils.cpp:772: getProbeMatch] Probe port is still not opened
[2024-11-14T9:39:13.376Z:281471285717984:network_utils.cpp:772: getProbeMatch] Probe port is still not opened
[2024-11-14T9:39:14.376Z:281471285717984:network_utils.cpp:772: getProbeMatch] Probe port is still not opened
[2024-11-14T9:39:14.916Z:281467375774688:stream_monitor.cpp:695: onConnectionTimeout] [streamMonitor] onConnectionTimeout for test1114, url:rtsp://10.0.0.167:8554/live/dfc9cf30-38ae-41f7-9714-b6e0b1adfb9d
[2024-11-14T9:39:14.916Z:281467367320544:stream_monitor.cpp:954: frameMonitorTask] Exited the thread frameMonitorTask name: test1114
[2024-11-14T9:39:15.377Z:281471285717984:network_utils.cpp:772: getProbeMatch] Probe port is still not opened
[2024-11-14T9:39:16.0Z:281471277263840:network_utils.cpp:541: sendProbe] Probe port is still not opened
[2024-11-14T9:39:16.377Z:281471285717984:network_utils.cpp:772: getProbeMatch] Probe port is still not opened
[2024-11-14T9:39:17.377Z:281471285717984:network_utils.cpp:772: getProbeMatch] Probe port is still not opened
[2024-11-14T9:39:18.272Z:281467553311712:fps_display.cpp:80: displayFPS] Unique ID = recording_50c4d2b6-f616-4ae3-b26d-06ef68669f00 FPS = { 15.015220, 15.034482, 14.924789, 14.979851, 14.970236, 15.012393, 14.982786, 14.979447, 14.979844, 14.981674, 14.976842, 15.075889 }
[2024-11-14T9:39:18.377Z:281471285717984:network_utils.cpp:772: getProbeMatch] Probe port is still not opened
[2024-11-14T9:39:19.377Z:281471285717984:network_utils.cpp:772: getProbeMatch] Probe port is still not opened
[2024-11-14T9:39:19.916Z:281467375774688:stream_monitor.cpp:690: restartConnection] Retrying rtsp connection for:test1114, retry count:3
[2024-11-14T9:39:19.916Z:281466579643360:stream_monitor.cpp:695: onConnectionTimeout] [streamMonitor] onConnectionTimeout for 你好, url:rtsp://10.0.0.167:8554/live/4e748af6-6ddf-42cc-bf2b-c12ee12083f6
[2024-11-14T9:39:19.916Z:281467046849504:stream_monitor.cpp:954: frameMonitorTask] Exited the thread frameMonitorTask name: 你好
[2024-11-14T9:39:20.377Z:281471285717984:network_utils.cpp:772: getProbeMatch] Probe port is still not opened
I can playback the RTSP proxy URL after 3 days on Orin AGX. I don’t see the issue in my side.
I have 4 cameras and when I unplug one of them, the problem occurs in about 10-15 minutes, I’ve tried it a few times already.