Jetpack 5.1.2
8gb Orin Nano
CUDA 11.4.315
cuDNN 8.6.0.166
TensorRT 8.5.2.2
VPI 2.3.9
Vulkan 1.3.204
OpenCV 4.5.4 with CUDA YES
L4T 35.4.1
Interfaces:
wlan0: 192.168.50.69
docker0: 172.17.0.1
# GPU family of target platform. Supported values: tegra, non-tegra
riva_target_gpu_family="tegra"
# Name of tegra platform that is being used. Supported tegra platforms: orin, xavier
riva_tegra_platform="orin"
# Enable or Disable Riva Services
# For any language other than en-US: service_enabled_nlp must be set to false
service_enabled_asr=true
service_enabled_nlp=false
service_enabled_tts=false
service_enabled_nmt=false
# Configure translation services
# Text-to-Text translation (T2T):
# - service_enabled_nmt must be set to true
# - Uncomment desired model for source and target languages in models_nmt field
# Speech-to-Text translation (S2T):
# - service_enabled_asr, service_enabled_nmt must be set to true
# - Set language code of input speech in the asr_language_code field
# - Uncomment desired model for source and target languages in models_nmt field
# Speech-to-Speech translation (S2S):
# - service_enabled_asr, service_enabled_nmt, service_enabled_tts must be set to true
# - Set language code of input speech in the asr_language_code field
# - Uncomment desired model for source and target languages in models_nmt field
# - Set language code of output speech in the tts_language_code field
# Enable Riva Enterprise
# If enrolled in Enterprise, enable Riva Enterprise by setting configuration
# here. You must explicitly acknowledge you have read and agree to the EULA.
# RIVA_API_KEY=<ngc api key>
# RIVA_API_NGC_ORG=<ngc organization>
# RIVA_EULA=accept
# Language code to fetch ASR models of a specific language
# Supported language codes: ar-AR, en-US, en-GB, de-DE, es-ES, es-US, fr-FR, hi-IN, it-IT, ja-JP, ru-RU, ko-KR, pt-BR, zh-CN, es-en-US, ja-en-JP
# For multiple languages enter space separated language codes.
asr_language_code=("en-US")
# ASR acoustic model architecture
# Supported values are: conformer, conformer_xl (en-US + amd64 only), citrinet_1024, citrinet_256 (en-US + arm64 only), jasper (en-US + amd64 only), quartznet (en-US + amd64 only)
asr_acoustic_model=("conformer")
# ASR acoustic model architecture variant
# Supported values for the architecture are:
# conformer: unified(de-DE, ja-JP and zh-CN only), ml_cs(es-en-US only), unified_ml_cs(ja-en-JP only)
# For the default model, keep the field empty
asr_acoustic_model_variant=("")
# ASR decoder type to be used
# If you'd like to use greedy decoder for ASR instead of flashlight/os2s decoder then set the below $use_asr_greedy_decoder to true
use_asr_greedy_decoder=false
# Language code to fetch TTS models of a specific language
# Supported language codes: en-US, es-ES, it-IT, de-DE
# For multiple languages enter space separated language codes
tts_language_code=("en-US")
# Specify one or more GPUs to use
# specifying more than one GPU is currently an experimental feature, and may result in undefined behaviours.
gpus_to_use="device=0"
# Specify the encryption key to use to deploy models
MODEL_DEPLOY_KEY="tlt_encode"
# Locations to use for storing models artifacts
#
# If an absolute path is specified, the data will be written to that location
# Otherwise, a Docker volume will be used (default).
#
# riva_init.sh will create a `rmir` and `models` directory in the volume or
# path specified.
#
# RMIR ($riva_model_loc/rmir)
# Riva uses an intermediate representation (RMIR) for models
# that are ready to deploy but not yet fully optimized for deployment. Pretrained
# versions can be obtained from NGC (by specifying NGC models below) and will be
# downloaded to $riva_model_loc/rmir by `riva_init.sh`
#
# Custom models produced by NeMo or TLT and prepared using riva-build
# may also be copied manually to this location $(riva_model_loc/rmir).
#
# Models ($riva_model_loc/models)
# During the riva_init process, the RMIR files in $riva_model_loc/rmir
# are inspected and optimized for deployment. The optimized versions are
# stored in $riva_model_loc/models. The riva server exclusively uses these
# optimized versions.
riva_model_loc="riva-model-repo"
if [[ $riva_target_gpu_family == "tegra" ]]; then
riva_model_loc="`pwd`/model_repository"
fi
# The default RMIRs are downloaded from NGC by default in the above $riva_rmir_loc directory
# If you'd like to skip the download from NGC and use the existing RMIRs in the $riva_rmir_loc
# then set the below $use_existing_rmirs flag to true. You can also deploy your set of custom
# RMIRs by keeping them in the riva_rmir_loc dir and use this quickstart script with the
# below flag to deploy them all together.
use_existing_rmirs=false
# Ports to expose for Riva services
riva_speech_api_port="50051"
# NGC orgs
riva_ngc_org="nvidia"
riva_ngc_team="riva"
riva_ngc_image_version="2.13.0"
riva_ngc_model_version="2.13.0"
########## ASR MODELS ##########
models_asr=()
for lang_code in ${asr_language_code[@]}; do
modified_lang_code="${lang_code//-/_}"
modified_lang_code=${modified_lang_code,,}
decoder=""
if [ "$use_asr_greedy_decoder" = true ]; then
decoder="_gre"
fi
if [[ ${asr_acoustic_model_variant} != "" ]]; then
if [[ ${asr_acoustic_model} == "conformer" && ${asr_acoustic_model_variant} != "unified" && ${asr_acoustic_model_variant} != "ml_cs" && ${asr_acoustic_model_variant} != "unified_ml_cs" ]]; then
echo "Valid variants for Conformer are: unified, ml_cs and unified_ml_cs."
exit 1
elif [[ ${asr_acoustic_model} != "conformer" ]]; then
echo "Invalid variant for ${asr_acoustic_model}."
exit 1
fi
asr_acoustic_model_variant="_${asr_acoustic_model_variant}"
fi
if [[ ${asr_acoustic_model} == "conformer_xl" && ${lang_code} != "en-US" ]]; then
echo "Conformer-XL acoustic model is only available for language code en-US."
exit 1
fi
if [[ ${asr_acoustic_model_variant} == "_unified" && ${lang_code} != "de-DE" && ${lang_code} != "ja-JP" && ${lang_code} != "zh-CN" ]]; then
echo "Unified Conformer acoustic model is only available for language code de-DE, ja-JP and zh-CN."
exit 1
fi
if [[ ${asr_acoustic_model_variant} == "_ml_cs" && ${lang_code} != "es-en-US" ]]; then
echo "Multilingual Code Switch Conformer acoustic model is only available for language code es-en-US."
exit 1
fi
if [[ ${asr_acoustic_model_variant} == "_unified_ml_cs" && ${lang_code} != "ja-en-JP" ]]; then
echo "Unified Multilingual Code Switch Conformer acoustic model is only available for language code ja-en-JP."
exit 1
fi
if [[ $riva_target_gpu_family == "tegra" ]]; then
if [[ ${asr_acoustic_model} == "jasper" || \
${asr_acoustic_model} == "quartznet" || \
${asr_acoustic_model} == "conformer_xl" ]]; then
echo "Conformer-XL, Jasper and Quartznet models are not available for arm64 architecture"
exit 1
fi
if [[ ${asr_acoustic_model} == "citrinet_256" && ${lang_code} != "en-US" ]]; then
echo "For arm64 architecture, citrinet_256 acoustic model is only available for language code en-US."
exit 1
fi
models_asr+=(
### Streaming w/ CPU decoder, best latency configuration
"${riva_ngc_org}/${riva_ngc_team}/models_asr_${asr_acoustic_model}${asr_acoustic_model_variant}_${modified_lang_code}_str:${riva_ngc_model_version}-${riva_target_gpu_family}-${riva_tegra_platform}"
### Offline w/ CPU decoder
# "${riva_ngc_org}/${riva_ngc_team}/rmir_asr_${asr_acoustic_model}${asr_acoustic_model_variant}_${modified_lang_code}_ofl${decoder}:${riva_ngc_model_version}"
)
else
if [[ ${asr_acoustic_model} != "conformer" && \
${asr_acoustic_model} != "conformer_xl" && \
${asr_acoustic_model} != "citrinet_1024" && \
${asr_acoustic_model} != "jasper" && \
${asr_acoustic_model} != "quartznet" ]]; then
echo "For amd64 architecture, valid acoustic models are conformer, conformer_xl, citrinet_1024, jasper and quartznet."
exit 1
fi
if [[ (${asr_acoustic_model} == "jasper" || \
${asr_acoustic_model} == "quartznet") && \
${lang_code} != "en-US" ]]; then
echo "jasper and quartznet acoustic models are only available for language code en-US."
exit 1
fi
models_asr+=(
### Streaming w/ CPU decoder, best latency configuration
"${riva_ngc_org}/${riva_ngc_team}/rmir_asr_${asr_acoustic_model}${asr_acoustic_model_variant}_${modified_lang_code}_str${decoder}:${riva_ngc_model_version}"
### Streaming w/ CPU decoder, best throughput configuration
# "${riva_ngc_org}/${riva_ngc_team}/rmir_asr_${asr_acoustic_model}${asr_acoustic_model_variant}_${modified_lang_code}_str_thr${decoder}:${riva_ngc_model_version}"
### Offline w/ CPU decoder
"${riva_ngc_org}/${riva_ngc_team}/rmir_asr_${asr_acoustic_model}${asr_acoustic_model_variant}_${modified_lang_code}_ofl${decoder}:${riva_ngc_model_version}"
)
fi
### Punctuation model
if [[ ${asr_acoustic_model_variant} != "_unified" && ${asr_acoustic_model_variant} != "_unified_ml_cs" ]]; then
pnc_lang=$(echo $modified_lang_code | cut -d "_" -f 1)
pnc_region=${modified_lang_code##*_}
modified_lang_code=${pnc_lang}_${pnc_region}
if [[ $riva_target_gpu_family == "tegra" ]]; then
models_asr+=(
"${riva_ngc_org}/${riva_ngc_team}/models_nlp_punctuation_bert_base_${modified_lang_code}:${riva_ngc_model_version}-${riva_target_gpu_family}-${riva_tegra_platform}"
)
else
models_asr+=(
"${riva_ngc_org}/${riva_ngc_team}/rmir_nlp_punctuation_bert_base_${modified_lang_code}:${riva_ngc_model_version}"
)
fi
fi
done
when trying to load the docker container with bash riva_start.sh i get the following errors in the logs:
$ docker logs riva-speech
/opt/riva/bin/start-riva: line 10: curl: command not found
/opt/riva/bin/start-riva: line 11: [: -ne: unary operator expected
> Triton server is ready...
I0912 19:05:46.787671 22 riva_server.cc:126] Using Insecure Server Credentials
E0912 19:05:46.796759 22 model_registry.cc:288] error: unable to get server status: failed to connect to all addresses; last error: UNKNOWN: ipv4:127.0.0.1:8001: Failed to connect to remote host: Connection refused
I0912 19:05:46.954185 20 pinned_memory_manager.cc:240] Pinned memory pool is created at '0x203b8e000' with size 268435456
I0912 19:05:46.954640 20 cuda_memory_manager.cc:105] CUDA memory pool is created on device 0 with size 1000000000
I0912 19:05:47.271525 20 model_lifecycle.cc:459] loading: conformer-en-US-asr-streaming-ctc-decoder-cpu-streaming:1
I0912 19:05:47.271637 20 model_lifecycle.cc:459] loading: conformer-en-US-asr-streaming-endpointing-streaming:1
I0912 19:05:47.271713 20 model_lifecycle.cc:459] loading: conformer-en-US-asr-streaming-feature-extractor-streaming:1
I0912 19:05:47.271786 20 model_lifecycle.cc:459] loading: riva-punctuation-en-US:1
I0912 19:05:47.271841 20 model_lifecycle.cc:459] loading: riva-trt-conformer-en-US-asr-streaming-am-streaming:1
I0912 19:05:47.272042 20 model_lifecycle.cc:459] loading: riva-trt-riva-punctuation-en-US-nn-bert-base-uncased:1
I0912 19:05:47.403735 20 tensorrt.cc:5444] TRITONBACKEND_Initialize: tensorrt
I0912 19:05:47.403858 20 tensorrt.cc:5454] Triton TRITONBACKEND API version: 1.10
I0912 19:05:47.403870 20 tensorrt.cc:5460] 'tensorrt' TRITONBACKEND API version: 1.10
I0912 19:05:47.403879 20 tensorrt.cc:5488] backend configuration:
{"cmdline":{"auto-complete-config":"false","min-compute-capability":"5.300000","backend-directory":"/opt/tritonserver/backends","default-max-batch-size":"4"}}
I0912 19:05:48.330135 20 tensorrt.cc:5578] TRITONBACKEND_ModelInitialize: riva-trt-conformer-en-US-asr-streaming-am-streaming (version 1)
I0912 19:05:48.331957 20 tensorrt.cc:5627] TRITONBACKEND_ModelInstanceInitialize: riva-trt-conformer-en-US-asr-streaming-am-streaming_0 (GPU device 0)
I0912 19:05:50.092374 20 logging.cc:49] Loaded engine size: 285 MiB
W0912 19:05:50.227172 20 logging.cc:46] Using an engine plan file across different models of devices is not recommended and is likely to affect performance or even cause errors.
I0912 19:05:54.173549 20 logging.cc:49] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +535, GPU +593, now: CPU 1455, GPU 7227 (MiB)
I0912 19:05:54.192756 20 logging.cc:49] [MemUsageChange] TensorRT-managed allocation in engine deserialization: CPU +0, GPU +281, now: CPU 0, GPU 281 (MiB)
I0912 19:05:54.294400 20 logging.cc:49] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +0, GPU +7, now: CPU 883, GPU 6699 (MiB)
I0912 19:05:54.863133 20 logging.cc:49] [MemUsageChange] TensorRT-managed allocation in IExecutionContext creation: CPU +0, GPU +11, now: CPU 0, GPU 292 (MiB)
I0912 19:05:55.081385 20 tensorrt.cc:1547] Created instance riva-trt-conformer-en-US-asr-streaming-am-streaming_0 on GPU 0 with stream priority 0 and optimization profile default[0];
E0912 19:05:56.801369 22 model_registry.cc:288] error: unable to get server status: failed to connect to all addresses; last error: UNKNOWN: ipv4:127.0.0.1:8001: Failed to connect to remote host: Connection refused
E0912 19:06:06.801837 22 model_registry.cc:288] error: unable to get server status: failed to connect to all addresses; last error: UNKNOWN: ipv4:127.0.0.1:8001: Failed to connect to remote host: Connection refused
E0912 19:06:16.802244 22 model_registry.cc:288] error: unable to get server status: failed to connect to all addresses; last error: UNKNOWN: ipv4:127.0.0.1:8001: Failed to connect to remote host: Connection refused
E0912 19:06:26.802716 22 model_registry.cc:288] error: unable to get server status: failed to connect to all addresses; last error: UNKNOWN: ipv4:127.0.0.1:8001: Failed to connect to remote host: Connection refused
E0912 19:06:36.811107 22 model_registry.cc:288] error: unable to get server status: failed to connect to all addresses; last error: UNKNOWN: ipv4:127.0.0.1:8001: Failed to connect to remote host: Connection refused
E0912 19:06:46.811460 22 model_registry.cc:288] error: unable to get server status: failed to connect to all addresses; last error: UNKNOWN: ipv4:127.0.0.1:8001: Failed to connect to remote host: Connection refused
E0912 19:06:56.811830 22 model_registry.cc:288] error: unable to get server status: failed to connect to all addresses; last error: UNKNOWN: ipv4:127.0.0.1:8001: Failed to connect to remote host: Connection refused
I0912 19:06:58.293629 20 model_lifecycle.cc:693] successfully loaded 'riva-trt-conformer-en-US-asr-streaming-am-streaming' version 1
I0912 19:06:58.300656 20 feature-extractor.cc:415] TRITONBACKEND_ModelInitialize: conformer-en-US-asr-streaming-feature-extractor-streaming (version 1)
I0912 19:06:58.348831 20 backend_model.cc:303] model configuration:
{
"name": "conformer-en-US-asr-streaming-feature-extractor-streaming",
"platform": "",
"backend": "riva_asr_features",
"version_policy": {
"latest": {
"num_versions": 1
}
},
"max_batch_size": 1024,
"input": [
{
"name": "AUDIO_SIGNAL",
"data_type": "TYPE_FP32",
"format": "FORMAT_NONE",
"dims": [
-1
],
"is_shape_tensor": false,
"allow_ragged_batch": false,
"optional": false
},
{
"name": "SAMPLE_RATE",
"data_type": "TYPE_UINT32",
"format": "FORMAT_NONE",
"dims": [
1
],
"is_shape_tensor": false,
"allow_ragged_batch": false,
"optional": false
}
],
"output": [
{
"name": "AUDIO_FEATURES",
"data_type": "TYPE_FP32",
"dims": [
80,
-1
],
"label_filename": "",
"is_shape_tensor": false
},
{
"name": "AUDIO_PROCESSED",
"data_type": "TYPE_FP32",
"dims": [
1
],
"label_filename": "",
"is_shape_tensor": false
},
{
"name": "AUDIO_FEATURES_LENGTH",
"data_type": "TYPE_INT32",
"dims": [
1
],
"label_filename": "",
"is_shape_tensor": false
}
],
"batch_input": [],
"batch_output": [],
"optimization": {
"graph": {
"level": 0
},
"priority": "PRIORITY_DEFAULT",
"cuda": {
"graphs": false,
"busy_wait_events": false,
"graph_spec": [],
"output_copy_stream": true
},
"input_pinned_memory": {
"enable": true
},
"output_pinned_memory": {
"enable": true
},
"gather_kernel_buffer_threshold": 0,
"eager_batching": false
},
"sequence_batching": {
"oldest": {
"max_candidate_sequences": 1024,
"preferred_batch_size": [
256,
512
],
"max_queue_delay_microseconds": 1000
},
"max_sequence_idle_microseconds": 60000000,
"control_input": [
{
"name": "START",
"control": [
{
"kind": "CONTROL_SEQUENCE_START",
"int32_false_true": [
0,
1
],
"fp32_false_true": [],
"bool_false_true": [],
"data_type": "TYPE_INVALID"
}
]
},
{
"name": "READY",
"control": [
{
"kind": "CONTROL_SEQUENCE_READY",
"int32_false_true": [
0,
1
],
"fp32_false_true": [],
"bool_false_true": [],
"data_type": "TYPE_INVALID"
}
]
},
{
"name": "END",
"control": [
{
"kind": "CONTROL_SEQUENCE_END",
"int32_false_true": [
0,
1
],
"fp32_false_true": [],
"bool_false_true": [],
"data_type": "TYPE_INVALID"
}
]
},
{
"name": "CORRID",
"control": [
{
"kind": "CONTROL_SEQUENCE_CORRID",
"int32_false_true": [],
"fp32_false_true": [],
"bool_false_true": [],
"data_type": "TYPE_UINT64"
}
]
}
],
"state": []
},
"instance_group": [
{
"name": "conformer-en-US-asr-streaming-feature-extractor-streaming_0",
"kind": "KIND_GPU",
"count": 1,
"gpus": [
0
],
"secondary_devices": [],
"profile": [],
"passive": false,
"host_policy": ""
}
],
"default_model_filename": "",
"cc_model_filenames": {},
"metric_tags": {},
"parameters": {
"streaming": {
"string_value": "True"
},
"window_size": {
"string_value": "0.025"
},
"stddev": {
"string_value": "2.2668, 3.1642, 3.7079, 3.7642, 3.5349, 3.5901, 3.7640, 3.8424, 4.0145, 4.1475, 4.0457, 3.9048, 3.7709, 3.6117, 3.3188, 3.1489, 3.0615, 3.0362, 2.9929, 3.0500, 3.0341, 3.0484, 3.0103, 2.9474, 2.9128, 2.8669, 2.8332, 2.9411, 3.0378, 3.0712, 3.0190, 2.9992, 3.0124, 3.0024, 3.0275, 3.0870, 3.0656, 3.0142, 3.0493, 3.1373, 3.1135, 3.0675, 2.8828, 2.7018, 2.6296, 2.8826, 2.9325, 2.9288, 2.9271, 2.9890, 3.0137, 2.9855, 3.0839, 2.9319, 2.3512, 2.3795, 2.6191, 2.7555, 2.9326, 2.9931, 3.1543, 3.0855, 2.6820, 3.0566, 3.1272, 3.1663, 3.1836, 3.0018, 2.9089, 3.1727, 3.1626, 3.1086, 2.9804, 3.1107, 3.2998, 3.3697, 3.3716, 3.2487, 3.1597, 3.1181"
},
"apply_normalization": {
"string_value": "True"
},
"norm_per_feature": {
"string_value": "True"
},
"dither": {
"string_value": "0.0"
},
"stddev_floor": {
"string_value": "1e-05"
},
"transpose": {
"string_value": "False"
},
"chunk_size": {
"string_value": "0.16"
},
"right_padding_size": {
"string_value": "1.92"
},
"sample_rate": {
"string_value": "16000"
},
"window_stride": {
"string_value": "0.01"
},
"max_execution_batch_size": {
"string_value": "1"
},
"num_features": {
"string_value": "80"
},
"precalc_norm_params": {
"string_value": "False"
},
"precalc_norm_time_steps": {
"string_value": "0"
},
"mean": {
"string_value": "-11.4412, -9.9334, -9.1292, -9.0365, -9.2804, -9.5643, -9.7342, -9.6925, -9.6333, -9.2808, -9.1887, -9.1422, -9.1397, -9.2028, -9.2749, -9.4776, -9.9185, -10.1557, -10.3800, -10.5067, -10.3190, -10.4728, -10.5529, -10.6402, -10.6440, -10.5113, -10.7395, -10.7870, -10.6074, -10.5033, -10.8278, -10.6384, -10.8481, -10.6875, -10.5454, -10.4747, -10.5165, -10.4930, -10.3413, -10.3472, -10.3735, -10.6830, -10.8813, -10.6338, -10.3856, -10.7727, -10.8957, -10.8068, -10.7373, -10.6108, -10.3405, -10.2889, -10.3922, -10.4946, -10.3367, -10.4164, -10.9949, -10.7196, -10.3971, -10.1734, -9.9257, -9.6557, -9.1761, -9.6653, -9.7876, -9.7230, -9.7792, -9.7056, -9.2702, -9.4650, -9.2755, -9.1369, -9.1174, -8.9197, -8.5394, -8.2614, -8.1353, -8.1422, -8.3430, -8.6655"
},
"use_utterance_norm_params": {
"string_value": "False"
},
"left_padding_size": {
"string_value": "1.92"
},
"gain": {
"string_value": "1.0"
}
},
"model_warmup": [],
"model_transaction_policy": {
"decoupled": false
}
}
I0912 19:06:58.350500 20 tensorrt.cc:5578] TRITONBACKEND_ModelInitialize: riva-trt-riva-punctuation-en-US-nn-bert-base-uncased (version 1)
I0912 19:06:58.421547 20 pipeline_library.cc:24] TRITONBACKEND_ModelInitialize: riva-punctuation-en-US (version 1)
WARNING: Logging before InitGoogleLogging() is written to STDERR
W0912 19:06:58.422551 48 parameter_parser.cc:146] Parameter 'attn_mask_tensor_name' set but unused.
W0912 19:06:58.422822 48 parameter_parser.cc:146] Parameter 'bos_token' set but unused.
W0912 19:06:58.422828 48 parameter_parser.cc:146] Parameter 'capit_logits_tensor_name' set but unused.
W0912 19:06:58.422834 48 parameter_parser.cc:146] Parameter 'capitalization_mapping_path' set but unused.
W0912 19:06:58.422838 48 parameter_parser.cc:146] Parameter 'delimiter' set but unused.
W0912 19:06:58.422842 48 parameter_parser.cc:146] Parameter 'eos_token' set but unused.
W0912 19:06:58.422845 48 parameter_parser.cc:146] Parameter 'input_ids_tensor_name' set but unused.
W0912 19:06:58.422848 48 parameter_parser.cc:146] Parameter 'language_code' set but unused.
W0912 19:06:58.422852 48 parameter_parser.cc:146] Parameter 'load_model' set but unused.
W0912 19:06:58.422856 48 parameter_parser.cc:146] Parameter 'model_api' set but unused.
W0912 19:06:58.422860 48 parameter_parser.cc:146] Parameter 'model_family' set but unused.
W0912 19:06:58.422865 48 parameter_parser.cc:146] Parameter 'model_name' set but unused.
W0912 19:06:58.422869 48 parameter_parser.cc:146] Parameter 'pad_chars_with_spaces' set but unused.
W0912 19:06:58.422873 48 parameter_parser.cc:146] Parameter 'pipeline_type' set but unused.
W0912 19:06:58.422878 48 parameter_parser.cc:146] Parameter 'preserve_accents' set but unused.
W0912 19:06:58.422883 48 parameter_parser.cc:146] Parameter 'punct_logits_tensor_name' set but unused.
W0912 19:06:58.422888 48 parameter_parser.cc:146] Parameter 'punctuation_mapping_path' set but unused.
W0912 19:06:58.422894 48 parameter_parser.cc:146] Parameter 'remove_spaces' set but unused.
W0912 19:06:58.422899 48 parameter_parser.cc:146] Parameter 'to_lower' set but unused.
W0912 19:06:58.422904 48 parameter_parser.cc:146] Parameter 'token_type_tensor_name' set but unused.
W0912 19:06:58.422909 48 parameter_parser.cc:146] Parameter 'tokenizer' set but unused.
W0912 19:06:58.422912 48 parameter_parser.cc:146] Parameter 'tokenizer_to_lower' set but unused.
W0912 19:06:58.422917 48 parameter_parser.cc:146] Parameter 'unicode_normalize' set but unused.
W0912 19:06:58.422922 48 parameter_parser.cc:146] Parameter 'unk_token' set but unused.
W0912 19:06:58.422927 48 parameter_parser.cc:146] Parameter 'use_int64_nn_inputs' set but unused.
W0912 19:06:58.422931 48 parameter_parser.cc:146] Parameter 'vocab' set but unused.
W0912 19:06:58.423112 48 parameter_parser.cc:146] Parameter 'attn_mask_tensor_name' set but unused.
W0912 19:06:58.423126 48 parameter_parser.cc:146] Parameter 'bos_token' set but unused.
W0912 19:06:58.423130 48 parameter_parser.cc:146] Parameter 'capit_logits_tensor_name' set but unused.
W0912 19:06:58.423135 48 parameter_parser.cc:146] Parameter 'capitalization_mapping_path' set but unused.
W0912 19:06:58.423137 48 parameter_parser.cc:146] Parameter 'delimiter' set but unused.
W0912 19:06:58.423141 48 parameter_parser.cc:146] Parameter 'eos_token' set but unused.
W0912 19:06:58.423144 48 parameter_parser.cc:146] Parameter 'input_ids_tensor_name' set but unused.
W0912 19:06:58.423148 48 parameter_parser.cc:146] Parameter 'language_code' set but unused.
W0912 19:06:58.423151 48 parameter_parser.cc:146] Parameter 'model_api' set but unused.
W0912 19:06:58.423156 48 parameter_parser.cc:146] Parameter 'model_family' set but unused.
W0912 19:06:58.423158 48 parameter_parser.cc:146] Parameter 'model_name' set but unused.
W0912 19:06:58.423162 48 parameter_parser.cc:146] Parameter 'pad_chars_with_spaces' set but unused.
W0912 19:06:58.423166 48 parameter_parser.cc:146] Parameter 'preserve_accents' set but unused.
W0912 19:06:58.423169 48 parameter_parser.cc:146] Parameter 'punct_logits_tensor_name' set but unused.
W0912 19:06:58.423172 48 parameter_parser.cc:146] Parameter 'punctuation_mapping_path' set but unused.
W0912 19:06:58.423175 48 parameter_parser.cc:146] Parameter 'remove_spaces' set but unused.
W0912 19:06:58.423179 48 parameter_parser.cc:146] Parameter 'to_lower' set but unused.
W0912 19:06:58.423182 48 parameter_parser.cc:146] Parameter 'token_type_tensor_name' set but unused.
W0912 19:06:58.423187 48 parameter_parser.cc:146] Parameter 'tokenizer' set but unused.
W0912 19:06:58.423189 48 parameter_parser.cc:146] Parameter 'tokenizer_to_lower' set but unused.
W0912 19:06:58.423193 48 parameter_parser.cc:146] Parameter 'unicode_normalize' set but unused.
W0912 19:06:58.423197 48 parameter_parser.cc:146] Parameter 'unk_token' set but unused.
W0912 19:06:58.423199 48 parameter_parser.cc:146] Parameter 'use_int64_nn_inputs' set but unused.
W0912 19:06:58.423203 48 parameter_parser.cc:146] Parameter 'vocab' set but unused.
W0912 19:06:58.423228 48 parameter_parser.cc:146] Parameter 'attn_mask_tensor_name' set but unused.
W0912 19:06:58.423233 48 parameter_parser.cc:146] Parameter 'bos_token' set but unused.
W0912 19:06:58.423236 48 parameter_parser.cc:146] Parameter 'capit_logits_tensor_name' set but unused.
W0912 19:06:58.423239 48 parameter_parser.cc:146] Parameter 'capitalization_mapping_path' set but unused.
W0912 19:06:58.423243 48 parameter_parser.cc:146] Parameter 'delimiter' set but unused.
W0912 19:06:58.423246 48 parameter_parser.cc:146] Parameter 'eos_token' set but unused.
W0912 19:06:58.423249 48 parameter_parser.cc:146] Parameter 'input_ids_tensor_name' set but unused.
W0912 19:06:58.423254 48 parameter_parser.cc:146] Parameter 'language_code' set but unused.
W0912 19:06:58.423256 48 parameter_parser.cc:146] Parameter 'model_api' set but unused.
W0912 19:06:58.423260 48 parameter_parser.cc:146] Parameter 'model_family' set but unused.
W0912 19:06:58.423264 48 parameter_parser.cc:146] Parameter 'model_name' set but unused.
W0912 19:06:58.423266 48 parameter_parser.cc:146] Parameter 'pad_chars_with_spaces' set but unused.
W0912 19:06:58.423270 48 parameter_parser.cc:146] Parameter 'preserve_accents' set but unused.
W0912 19:06:58.423274 48 parameter_parser.cc:146] Parameter 'punct_logits_tensor_name' set but unused.
W0912 19:06:58.423276 48 parameter_parser.cc:146] Parameter 'punctuation_mapping_path' set but unused.
W0912 19:06:58.423280 48 parameter_parser.cc:146] Parameter 'remove_spaces' set but unused.
W0912 19:06:58.423283 48 parameter_parser.cc:146] Parameter 'to_lower' set but unused.
W0912 19:06:58.423287 48 parameter_parser.cc:146] Parameter 'token_type_tensor_name' set but unused.
W0912 19:06:58.423305 48 parameter_parser.cc:146] Parameter 'tokenizer_to_lower' set but unused.
W0912 19:06:58.423310 48 parameter_parser.cc:146] Parameter 'unicode_normalize' set but unused.
W0912 19:06:58.423313 48 parameter_parser.cc:146] Parameter 'unk_token' set but unused.
W0912 19:06:58.423316 48 parameter_parser.cc:146] Parameter 'use_int64_nn_inputs' set but unused.
W0912 19:06:58.423322 48 parameter_parser.cc:146] Parameter 'vocab' set but unused.
W0912 19:06:58.423398 48 parameter_parser.cc:146] Parameter 'model_api' set but unused.
W0912 19:06:58.423481 48 parameter_parser.cc:146] Parameter 'model_family' set but unused.
I0912 19:06:58.424102 20 backend_model.cc:303] model configuration:
{
"name": "riva-punctuation-en-US",
"platform": "",
"backend": "riva_nlp_pipeline",
"version_policy": {
"latest": {
"num_versions": 1
}
},
"max_batch_size": 1,
"input": [
{
"name": "PIPELINE_INPUT",
"data_type": "TYPE_STRING",
"format": "FORMAT_NONE",
"dims": [
1
],
"is_shape_tensor": false,
"allow_ragged_batch": false,
"optional": false
}
],
"output": [
{
"name": "PIPELINE_OUTPUT",
"data_type": "TYPE_STRING",
"dims": [
1
],
"label_filename": "",
"is_shape_tensor": false
}
],
"batch_input": [],
"batch_output": [],
"optimization": {
"priority": "PRIORITY_DEFAULT",
"input_pinned_memory": {
"enable": true
},
"output_pinned_memory": {
"enable": true
},
"gather_kernel_buffer_threshold": 0,
"eager_batching": false
},
"instance_group": [
{
"name": "riva-punctuation-en-US_0",
"kind": "KIND_CPU",
"count": 1,
"gpus": [],
"secondary_devices": [],
"profile": [],
"passive": false,
"host_policy": ""
}
],
"default_model_filename": "",
"cc_model_filenames": {},
"metric_tags": {},
"parameters": {
"attn_mask_tensor_name": {
"string_value": "attention_mask"
},
"to_lower": {
"string_value": "true"
},
"input_ids_tensor_name": {
"string_value": "input_ids"
},
"punct_logits_tensor_name": {
"string_value": "punct_logits"
},
"remove_spaces": {
"string_value": "False"
},
"unicode_normalize": {
"string_value": "False"
},
"eos_token": {
"string_value": "[SEP]"
},
"load_model": {
"string_value": "false"
},
"capit_logits_tensor_name": {
"string_value": "capit_logits"
},
"tokenizer": {
"string_value": "wordpiece"
},
"token_type_tensor_name": {
"string_value": "token_type_ids"
},
"pipeline_type": {
"string_value": "punctuation"
},
"model_api": {
"string_value": "/nvidia.riva.nlp.RivaLanguageUnderstanding/PunctuateText"
},
"vocab": {
"string_value": "/data/models/riva-punctuation-en-US/1/f92889b136d2433693cb9127e1aea218_vocab.txt"
},
"delimiter": {
"string_value": " "
},
"model_family": {
"string_value": "riva"
},
"punctuation_mapping_path": {
"string_value": "/data/models/riva-punctuation-en-US/1/bf74918539724a61a0d7703134519ea5_punct_label_ids.csv"
},
"language_code": {
"string_value": "en-US"
},
"preserve_accents": {
"string_value": "false"
},
"tokenizer_to_lower": {
"string_value": "true"
},
"capitalization_mapping_path": {
"string_value": "/data/models/riva-punctuation-en-US/1/56633d0a0d8e459b9c8acd572cfa34b8_capit_label_ids.csv"
},
"unk_token": {
"string_value": "[UNK]"
},
"model_name": {
"string_value": "riva-trt-riva-punctuation-en-US-nn-bert-base-uncased"
},
"pad_chars_with_spaces": {
"string_value": "False"
},
"bos_token": {
"string_value": "[CLS]"
},
"use_int64_nn_inputs": {
"string_value": "False"
}
},
"model_warmup": []
}
I0912 19:06:58.424595 20 pipeline_library.cc:28] TRITONBACKEND_ModelInstanceInitialize: riva-punctuation-en-US_0 (device 0)
I0912 19:06:58.484221 20 tensorrt.cc:5627] TRITONBACKEND_ModelInstanceInitialize: riva-trt-riva-punctuation-en-US-nn-bert-base-uncased_0 (GPU device 0)
I0912 19:06:58.484622 20 model_lifecycle.cc:693] successfully loaded 'riva-punctuation-en-US' version 1
E0912 19:07:06.819216 22 model_registry.cc:288] error: unable to get server status: failed to connect to all addresses; last error: UNKNOWN: ipv4:127.0.0.1:8001: Failed to connect to remote host: Connection refused
/opt/riva/bin/start-riva: line 55: 20 Killed ${CUSTOM_TRITON_ENV} tritonserver --log-verbose=0 --disable-auto-complete-config $model_repos --cuda-memory-pool-byte-size=0:1000000000
One of the processes has exited unexpectedly. Stopping container.
W0912 19:07:08.115984 22 riva_server.cc:196] Signal: 15