Tao Finetuning

davesarmoury · December 20, 2022, 9:42pm

I’m trying to finetune FastPitch and HiFiGAN using Tao and mostly following the notebook from Text to Speech Notebook | NVIDIA NGC

When trying to finetune FastPitch, with the command below:

!tao spectro_gen finetune
-e $SPECS_DIR/spectro_gen/finetune.yaml
-g 1
-k tlt_encode
-r $RESULTS_DIR/spectro_gen/finetune
-m $pretrained_fastpitch_model
train_dataset=$DATA_DIR/$finetune_data_name/merged_train.json
validation_dataset=$DATA_DIR/$finetune_data_name/manifest_val.json
prior_folder=$RESULTS_DIR/spectro_gen/finetune/prior_folder
trainer.max_epochs=200
n_speakers=2
pitch_fmin=$pitch_fmin
pitch_fmax=$pitch_fmax
pitch_avg=$pitch_mean
pitch_std=$pitch_std
trainer.precision=16

it fails with this error:

[NeMo E 2022-12-20 21:25:37 common:503] Model instantiation failed!
Target class: nemo.collections.tts.models.fastpitch.FastPitchModel
Error(s): src path does not exist or it is not a path in nemo file. src value I got was: scripts/tts_dataset_files/cmudict-0.7b_nv22.08. Absolute: /opt/nvidia/tools/scripts/tts_dataset_files/cmudict-0.7b_nv22.08
Traceback (most recent call last):
File “”, line 764, in extract
File “/opt/conda/lib/python3.8/tarfile.py”, line 2060, in extract
tarinfo = self.getmember(member)
File “/opt/conda/lib/python3.8/tarfile.py”, line 1782, in getmember
raise KeyError(“filename %r not found” % name)
KeyError: “filename ‘manifest.yaml’ not found”

During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File “”, line 653, in restore_manifest
File “”, line 767, in extract
File “/opt/conda/lib/python3.8/tarfile.py”, line 2060, in extract
tarinfo = self.getmember(member)
File “/opt/conda/lib/python3.8/tarfile.py”, line 1782, in getmember
raise KeyError(“filename %r not found” % name)
KeyError: “filename ‘./manifest.yaml’ not found”

During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File “”, line 94, in restore_from
File “”, line 404, in restore_from
File “”, line 203, in validate_archive
File “”, line 657, in restore_manifest
TypeError: The indicated file ‘/data/tts_en_fastpitch_v1.4.0/tts_en_fastpitch_align.nemo’ is not an EFF archive

During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File “/opt/NeMo/nemo/core/classes/common.py”, line 482, in from_config_dict
instance = imported_cls(cfg=config, trainer=trainer)
File “/opt/NeMo/nemo/collections/tts/models/fastpitch.py”, line 105, in init
self._setup_tokenizer(tokenizer_conf)
File “/opt/NeMo/nemo/collections/tts/models/fastpitch.py”, line 188, in _setup_tokenizer
g2p_kwargs[“phoneme_dict”] = self.register_artifact(
File “/opt/NeMo/nemo/core/classes/modelPT.py”, line 222, in register_artifact
return self._save_restore_connector.register_artifact(self, config_path, src, verify_src_exists)
File “/opt/NeMo/nemo/core/connectors/save_restore_connector.py”, line 378, in register_artifact
raise FileNotFoundError(
FileNotFoundError: src path does not exist or it is not a path in nemo file. src value I got was: scripts/tts_dataset_files/cmudict-0.7b_nv22.08. Absolute: /opt/nvidia/tools/scripts/tts_dataset_files/cmudict-0.7b_nv22.08

Error executing job with overrides: [‘exp_manager.explicit_log_dir=/results/spectro_gen/finetune’, ‘trainer.gpus=1’, ‘restore_from=/data/tts_en_fastpitch_v1.4.0/tts_en_fastpitch_align.nemo’, ‘encryption_key=tlt_encode’, ‘train_dataset=/data/GLaDOS/merged_train.json’, ‘validation_dataset=/data/GLaDOS/manifest_val.json’, ‘prior_folder=/results/spectro_gen/finetune/prior_folder’, ‘trainer.max_epochs=200’, ‘n_speakers=2’, ‘pitch_fmin=80.0’, ‘pitch_fmax=2048.0’, ‘pitch_avg=165.458’, ‘pitch_std=40.1891’, ‘trainer.precision=16’]
An error occurred during Hydra’s exception formatting:
AssertionError()
Traceback (most recent call last):
File “/opt/conda/lib/python3.8/site-packages/hydra/_internal/utils.py”, line 252, in run_and_report
assert mdl is not None
AssertionError

During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File “</opt/conda/lib/python3.8/site-packages/nvidia_tao_pytorch/conv_ai/tts/spectro_gen/scripts/finetune.py>”, line 3, in
File “”, line 285, in
File “/opt/NeMo/nemo/core/config/hydra_runner.py”, line 104, in wrapper
_run_hydra(
File “/opt/conda/lib/python3.8/site-packages/hydra/_internal/utils.py”, line 377, in _run_hydra
run_and_report(
File “/opt/conda/lib/python3.8/site-packages/hydra/_internal/utils.py”, line 294, in run_and_report
raise ex
File “/opt/conda/lib/python3.8/site-packages/hydra/_internal/utils.py”, line 211, in run_and_report
return func()
File “/opt/conda/lib/python3.8/site-packages/hydra/_internal/utils.py”, line 378, in
lambda: hydra.run(
File “/opt/conda/lib/python3.8/site-packages/hydra/_internal/hydra.py”, line 111, in run
_ = ret.return_value
File “/opt/conda/lib/python3.8/site-packages/hydra/core/utils.py”, line 233, in return_value
raise self._return_value
File “/opt/conda/lib/python3.8/site-packages/hydra/core/utils.py”, line 160, in run_job
ret.return_value = task_function(task_cfg)
File “”, line 224, in main
File “/opt/NeMo/nemo/core/classes/modelPT.py”, line 311, in restore_from
instance = cls._save_restore_connector.restore_from(
File “”, line 94, in restore_from
File “/opt/NeMo/nemo/core/connectors/save_restore_connector.py”, line 235, in restore_from
loaded_params = self.load_config_and_state_dict(
File “/opt/NeMo/nemo/core/connectors/save_restore_connector.py”, line 158, in load_config_and_state_dict
instance = calling_cls.from_config_dict(config=conf, trainer=trainer)
File “/opt/NeMo/nemo/core/classes/common.py”, line 504, in from_config_dict
raise e
File “/opt/NeMo/nemo/core/classes/common.py”, line 496, in from_config_dict
instance = cls(cfg=config, trainer=trainer)
File “/opt/NeMo/nemo/collections/tts/models/fastpitch.py”, line 105, in init
self._setup_tokenizer(tokenizer_conf)
File “/opt/NeMo/nemo/collections/tts/models/fastpitch.py”, line 188, in _setup_tokenizer
g2p_kwargs[“phoneme_dict”] = self.register_artifact(
File “/opt/NeMo/nemo/core/classes/modelPT.py”, line 222, in register_artifact
return self._save_restore_connector.register_artifact(self, config_path, src, verify_src_exists)
File “/opt/NeMo/nemo/core/connectors/save_restore_connector.py”, line 378, in register_artifact
raise FileNotFoundError(
FileNotFoundError: src path does not exist or it is not a path in nemo file. src value I got was: scripts/tts_dataset_files/cmudict-0.7b_nv22.08. Absolute: /opt/nvidia/tools/scripts/tts_dataset_files/cmudict-0.7b_nv22.08

I’m using pretrained fastpitch 1.4.0 from ngc (TTS En FastPitch | NVIDIA NGC). I’ve tried the other 3 versions too, but get equally un awesome results

So many different errors in that output, I don’t even know what ones to start tracking down. The YAML is the default one downloaded from tao spectro_gen download_specs

• Hardware (3090ti, Ryzen 7 5800X, 32Gb)
• Network Type (TAO with FastPitch)
• TLT Version (4.0.0-pyt)

Morganh · December 21, 2022, 3:41am

So, do you mean you can not run default notebook successfully?
If yes, please double check your ~/.tao_mounts.json to confirm the mapping takes effect in order to map your local files into tao container.
Because all the path in the commandline should be inside the tao container.

davesarmoury · December 21, 2022, 1:53pm

@Morganh That is correct, I can’t run the notebook correctly. Below is my taomounts:

{
“Mounts”: [
{
“source”: “/home/davesarmoury/ws/glados_ws/TAO/tmp/data”,
“destination”: “/data”
},
{
“source”: “/home/davesarmoury/ws/glados_ws/TAO/tmp/specs”,
“destination”: “/specs”
},
{
“source”: “/home/davesarmoury/ws/glados_ws/TAO/tmp/results”,
“destination”: “/results”
},
{
“source”: “/home/davesarmoury/.cache”,
“destination”: “/root/.cache”
}
],
“DockerOptions”: {
“shm_size”: “16G”,
“ulimits”: {
“memlock”: -1,
“stack”: 67108864
}
}
}

And the tao command expands-out to be:

tao spectro_gen finetune -e /specs/spectro_gen/finetune.yaml -g 1 -k tlt_encode -r /results/spectro_gen/finetune -m /data/tts_en_fastpitch_v1.4.0/tts_en_fastpitch_align.nemo train_dataset=/data/GLaDOS/merged_train.json validation_dataset=/data/GLaDOS/manifest_val.json prior_folder=/results/spectro_gen/finetune/prior_folder trainer.max_epochs=200 n_speakers=2 pitch_fmin=80.0 pitch_fmax=2048.0 pitch_avg=165.458 pitch_std=40.1891 trainer.precision=16

And a tree of the “data” directory from my taomounts is:

/home/davesarmoury/ws/glados_ws/TAO/tmp/data
├── dataset_convert
│ └── merge
│ ├── dataset_convert.log
│ └── status.json
├── GLaDOS
│ ├── clips_resampled
…
│ ├── manifest.json
│ ├── manifest_resampled.json
│ ├── manifest_train.json
│ ├── manifest_val.json
│ └── merged_train.json
├── LJSpeech
│ ├── ljs_audio_text_test_filelist.txt
│ ├── ljs_audio_text_train_filelist.txt
│ ├── ljs_audio_text_val_filelist.txt
│ ├── ljspeech_test.json
│ ├── ljspeech_train.json
│ ├── ljspeech_val.json
│ ├── metadata.csv
│ ├── README
│ └── wavs
│ ├── LJ001-0001.txt
…
├── tts_en_fastpitch_v1.4.0
│ └── tts_en_fastpitch_align.nemo
└── tts_hifigan_v1.0.0rc1
└── tts_hifigan.nemo

Also, connecting to the container directly shows that the data mountpoint is connected properly:

davesarmoury@armoury-beast:~$ tao spectro_gen
2022-12-21 09:15:15,089 [INFO] root: Registry: [‘nvcr.io’]
2022-12-21 09:15:15,111 [INFO] tlt.components.instance_handler.local_instance: Running command in container: nvcr.io/nvidia/tao/tao-toolkit:4.0.0-pyt
2022-12-21 09:15:15,111 [INFO] tlt.components.instance_handler.local_instance: No commands provided to the launcher
Kicking off an interactive docker session.
NOTE: This container instance will be terminated when you exit.
2022-12-21 09:15:15,120 [WARNING] tlt.components.docker_handler.docker_handler:
Docker will run the commands as root. If you would like to retain your
local host permissions, please add the “user”:“UID:GID” in the
DockerOptions portion of the “/home/davesarmoury/.tao_mounts.json” file. You can obtain your
users UID and GID by using the “id -u” and “id -g” commands on the
terminal.
root@8d388d5bd733:/opt/nvidia/tools# ls /data/
GLaDOS LJSpeech dataset_convert tts_en_fastpitch_v1.4.0 tts_hifigan_v1.0.0rc1
root@8d388d5bd733:/opt/nvidia/tools# ls /data/tts_en_fastpitch_v1.4.0/
tts_en_fastpitch_align.nemo

Morganh · December 22, 2022, 9:50am

Hi,
You was following text-to-speech-finetuning-cvtool.ipynb.

As mentioned in the notebook,

This notebook assumes that you are already familiar with TTS Training using TAO, as described in the text-to-speech-training notebook, and that you have a pretrained TTS model.

Did you run text-to-speech-training notebook successfully?

If yes, please use the .tlt model you have trained and config it in your current “-m $pretrained_fastpitch_model” and retry.

davesarmoury · December 22, 2022, 1:51pm

Hi @Morganh,

I’m trying to run the finetuning notebook from that ngc resource. It fails on the finetuning step. I’m using the pretrained fastpitch 1.4 from ngc as the notebook suggests ngc registry model download-version "nvidia/nemo/tts_en_fastpitch:1.4.0"

Morganh · December 22, 2022, 3:20pm

Could you share the full log ?

davesarmoury · December 22, 2022, 3:42pm

The full output from the command is here:

gist.github.com

https://gist.github.com/davesarmoury/a564f000ff80d440cd2b3df1f68f9120

gistfile1.txt

2022-12-22 10:36:33,950 [INFO] root: Registry: ['nvcr.io']
2022-12-22 10:36:33,984 [INFO] tlt.components.instance_handler.local_instance: Running command in container: nvcr.io/nvidia/tao/tao-toolkit:4.0.0-pyt
2022-12-22 10:36:33,993 [WARNING] tlt.components.docker_handler.docker_handler: 
Docker will run the commands as root. If you would like to retain your
local host permissions, please add the "user":"UID:GID" in the
DockerOptions portion of the "/home/davesarmoury/.tao_mounts.json" file. You can obtain your
users UID and GID by using the "id -u" and "id -g" commands on the
terminal.
[NeMo W 2022-12-22 15:36:37 experimental:27] Module <class 'nemo.collections.tts.torch.tts_tokenizers.IPATokenizer'> is experimental, not ready for production and is not fully supported. Use at your own risk.
[NeMo W 2022-12-22 15:36:37 experimental:27] Module <class 'nemo.collections.tts.models.radtts.RadTTSModel'> is experimental, not ready for production and is not fully supported. Use at your own risk.

This file has been truncated. show original

Also, the complete notebook (with some application-specific modifications for my project) is here:

github.com

davesarmoury/GLaDOS/blob/main/TrainGLaDOS_TAO.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train Adapt Optimize (TAO) Toolkit"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Train Adapt Optimize (TAO) Toolkit  is a python based AI toolkit for taking purpose-built pre-trained AI models and customizing them with your own data. \n",
    "\n",
    "Transfer learning extracts learned features from an existing neural network to a new one. Transfer learning is often used when creating a large training dataset is not feasible. \n",
    "\n",
    "Developers, researchers and software partners building intelligent AI apps and services, can bring their own data to fine-tune pre-trained models instead of going through the hassle of training from scratch."
   ]
  },

This file has been truncated. show original

Thanks,

Dave

Morganh · December 22, 2022, 3:46pm

Hi,
Please try to use another .nemo file . See TTS En FastPitch | NVIDIA NGC

wget ‘https://api.ngc.nvidia.com/v2/models/nvidia/nemo/tts_en_fastpitch/versions/1.8.1/files/tts_en_fastpitch_align.nemo’

davesarmoury · December 22, 2022, 3:58pm

@Morganh

Below are 2 gists, one with 1.0.0 of fastpitch and one with 1.8.0. Neither one works, but with different errors

gist.github.com

https://gist.github.com/davesarmoury/6d8583d0e90423ac99555ae9db60b2eb

gistfile1.txt

tao spectro_gen finetune -e /specs/spectro_gen/finetune.yaml -g 1 -k tlt_encode -r /results/spectro_gen/finetune -m /data/tts_en_fastpitch_v1.8.0/tts_en_fastpitch_align.nemo train_dataset=/data/GLaDOS/merged_train.json validation_dataset=/data/GLaDOS/manifest_val.json prior_folder=/results/spectro_gen/finetune/prior_folder trainer.max_epochs=200 n_speakers=2 pitch_fmin=80.0 pitch_fmax=2048.0 pitch_avg=165.458 pitch_std=40.1891 trainer.precision=16

2022-12-22 10:54:28,805 [INFO] root: Registry: ['nvcr.io']
2022-12-22 10:54:28,827 [INFO] tlt.components.instance_handler.local_instance: Running command in container: nvcr.io/nvidia/tao/tao-toolkit:4.0.0-pyt
2022-12-22 10:54:28,838 [WARNING] tlt.components.docker_handler.docker_handler: 
Docker will run the commands as root. If you would like to retain your
local host permissions, please add the "user":"UID:GID" in the
DockerOptions portion of the "/home/davesarmoury/.tao_mounts.json" file. You can obtain your
users UID and GID by using the "id -u" and "id -g" commands on the
terminal.

This file has been truncated. show original

gistfile2.txt

tao spectro_gen finetune -e /specs/spectro_gen/finetune.yaml -g 1 -k tlt_encode -r /results/spectro_gen/finetune -m /data/tts_en_fastpitch_v1.0.0/tts_en_fastpitch_align.nemo train_dataset=/data/GLaDOS/merged_train.json validation_dataset=/data/GLaDOS/manifest_val.json prior_folder=/results/spectro_gen/finetune/prior_folder trainer.max_epochs=200 n_speakers=2 pitch_fmin=80.0 pitch_fmax=2048.0 pitch_avg=165.458 pitch_std=40.1891 trainer.precision=16

2022-12-22 10:55:59,906 [INFO] root: Registry: ['nvcr.io']
2022-12-22 10:55:59,929 [INFO] tlt.components.instance_handler.local_instance: Running command in container: nvcr.io/nvidia/tao/tao-toolkit:4.0.0-pyt
2022-12-22 10:55:59,939 [WARNING] tlt.components.docker_handler.docker_handler: 
Docker will run the commands as root. If you would like to retain your
local host permissions, please add the "user":"UID:GID" in the
DockerOptions portion of the "/home/davesarmoury/.tao_mounts.json" file. You can obtain your
users UID and GID by using the "id -u" and "id -g" commands on the
terminal.

This file has been truncated. show original

There were downloaded using ngc. I doubt wget will make a difference, but I will try that now just to be sure

davesarmoury · December 22, 2022, 4:06pm

@Morganh

Another gist! I downloaded the 1.8.1 nemo file using the wget command, moved it into the data directory, and re ran the tao command (with updated nemo filepath). Error output in the gist

gist.github.com

https://gist.github.com/davesarmoury/5ba9592d7d5f79679d6b8295df3e5d8f

gistfile1.txt

wget https://api.ngc.nvidia.com/v2/models/nvidia/nemo/tts_en_fastpitch/versions/1.8.1/files/tts_en_fastpitch_align.nemo
mv tts_en_fastpitch_align.nemo tmp/data 

tao spectro_gen finetune -e /specs/spectro_gen/finetune.yaml -g 1 -k tlt_encode -r /results/spectro_gen/finetune -m /data/tts_en_fastpitch_align.nemo train_dataset=/data/GLaDOS/merged_train.json validation_dataset=/data/GLaDOS/manifest_val.json prior_folder=/results/spectro_gen/finetune/prior_folder trainer.max_epochs=200 n_speakers=2 pitch_fmin=80.0 pitch_fmax=2048.0 pitch_avg=165.458 pitch_std=40.1891 trainer.precision=16

2022-12-22 11:02:12,655 [INFO] root: Registry: ['nvcr.io']
2022-12-22 11:02:12,676 [INFO] tlt.components.instance_handler.local_instance: Running command in container: nvcr.io/nvidia/tao/tao-toolkit:4.0.0-pyt
2022-12-22 11:02:12,687 [WARNING] tlt.components.docker_handler.docker_handler: 
Docker will run the commands as root. If you would like to retain your
local host permissions, please add the "user":"UID:GID" in the

This file has been truncated. show original

Morganh · December 22, 2022, 5:47pm

To narrow down, please try to run in 22.05 docker with the 1.4.0 nemo file(wget ‘https://api.ngc.nvidia.com/v2/models/nvidia/nemo/tts_en_fastpitch/versions/1.4.0/files/tts_en_fastpitch_align.nemo’).
For example,
$ docker run --runtime=nvidia -it --rm --entrypoint="" -v /home/yourname:/workspace nvcr.io/nvidia/tao/tao-toolkit-pyt:v3.22.05-py3 /bin/bash

davesarmoury · December 22, 2022, 7:04pm

Trying to run the command in the new docker gives me a configuration error:

root@9c3d30b2a8df:/opt/nvidia/tools# spectro_gen finetune -e /specs/spectro_gen/finetune.yaml -g 1 -k tlt_encode -r /results/spectro_gen/finetune -m /data/tts_en_fastpitch_v1.4.0/tts_en_fastpitch_align.nemo train_dataset=/data/GLaDOS/merged_train.json validation_dataset=/data/GLaDOS/manifest_val.json prior_folder=/results/spectro_gen/finetune/prior_folder trainer.max_epochs=200 n_speakers=2 pitch_fmin=80.0 pitch_fmax=2048.0 pitch_avg=165.458 pitch_std=40.1891 trainer.precision=16
[NeMo W 2022-12-22 19:14:49 init:22] pynini is not installed !
Please run the nemo_text_processing/setup.sh script prior to usage of this toolkit.
[NeMo W 2022-12-22 19:14:52 init:22] pynini is not installed !
Please run the nemo_text_processing/setup.sh script prior to usage of this toolkit.
[NeMo W 2022-12-22 19:14:52 nemo_logging:349] /home/jenkins/agent/workspace/tlt-pytorch-main-nightly/conv_ai/tts/spectro_gen/scripts/finetune.py:273: UserWarning:
‘finetune.yaml’ is validated against ConfigStore schema with the same name.
This behavior is deprecated in Hydra 1.1 and will be removed in Hydra 1.2.
See https://hydra.cc/docs/next/upgrades/1.0_to_1.1/automatic_schema_matching for migration instructions.

Traceback (most recent call last):
File “/opt/conda/lib/python3.8/site-packages/hydra/_internal/config_loader_impl.py”, line 455, in _load_single_config
merged = OmegaConf.merge(schema.config, ret.config)
File “/opt/conda/lib/python3.8/site-packages/omegaconf/omegaconf.py”, line 264, in merge
target.merge_with(*configs[1:])
File “/opt/conda/lib/python3.8/site-packages/omegaconf/basecontainer.py”, line 438, in merge_with
self._format_and_raise(key=None, value=None, cause=e)
File “/opt/conda/lib/python3.8/site-packages/omegaconf/base.py”, line 190, in _format_and_raise
format_and_raise(
File “/opt/conda/lib/python3.8/site-packages/omegaconf/_utils.py”, line 741, in format_and_raise
_raise(ex, cause)
File “/opt/conda/lib/python3.8/site-packages/omegaconf/_utils.py”, line 719, in _raise
raise ex.with_traceback(sys.exc_info()[2]) # set end OC_CAUSE=1 for full backtrace
File “/opt/conda/lib/python3.8/site-packages/omegaconf/basecontainer.py”, line 436, in merge_with
self._merge_with(*others)
File “/opt/conda/lib/python3.8/site-packages/omegaconf/basecontainer.py”, line 460, in _merge_with
BaseContainer._map_merge(self, other)
File “/opt/conda/lib/python3.8/site-packages/omegaconf/basecontainer.py”, line 378, in _map_merge
dest[key] = src._get_node(key)
File “/opt/conda/lib/python3.8/site-packages/omegaconf/dictconfig.py”, line 310, in setitem
self._format_and_raise(
File “/opt/conda/lib/python3.8/site-packages/omegaconf/base.py”, line 190, in _format_and_raise
format_and_raise(
File “/opt/conda/lib/python3.8/site-packages/omegaconf/_utils.py”, line 741, in format_and_raise
_raise(ex, cause)
File “/opt/conda/lib/python3.8/site-packages/omegaconf/_utils.py”, line 719, in _raise
raise ex.with_traceback(sys.exc_info()[2]) # set end OC_CAUSE=1 for full backtrace
File “/opt/conda/lib/python3.8/site-packages/omegaconf/dictconfig.py”, line 308, in setitem
self.__set_impl(key=key, value=value)
File “/opt/conda/lib/python3.8/site-packages/omegaconf/dictconfig.py”, line 318, in __set_impl
self._set_item_impl(key, value)
File “/opt/conda/lib/python3.8/site-packages/omegaconf/basecontainer.py”, line 503, in _set_item_impl
target_node_ref = self._get_node(key)
File “/opt/conda/lib/python3.8/site-packages/omegaconf/dictconfig.py”, line 465, in _get_node
self._validate_get(key)
File “/opt/conda/lib/python3.8/site-packages/omegaconf/dictconfig.py”, line 166, in _validate_get
self._format_and_raise(
File “/opt/conda/lib/python3.8/site-packages/omegaconf/base.py”, line 190, in _format_and_raise
format_and_raise(
File “/opt/conda/lib/python3.8/site-packages/omegaconf/_utils.py”, line 821, in format_and_raise
_raise(ex, cause)
File “/opt/conda/lib/python3.8/site-packages/omegaconf/_utils.py”, line 719, in _raise
raise ex.with_traceback(sys.exc_info()[2]) # set end OC_CAUSE=1 for full backtrace
omegaconf.errors.ConfigKeyError: Key ‘phoneme_dict_path’ not in ‘DefaultConfig’
full_key: phoneme_dict_path
object_type=DefaultConfig

The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File “/home/jenkins/agent/workspace/tlt-pytorch-main-nightly/conv_ai/tts/spectro_gen/scripts/finetune.py”, line 273, in
File “/opt/conda/lib/python3.8/site-packages/nemo/core/config/hydra_runner.py”, line 104, in wrapper
_run_hydra(
File “/opt/conda/lib/python3.8/site-packages/hydra/_internal/utils.py”, line 367, in _run_hydra
run_and_report(
File “/opt/conda/lib/python3.8/site-packages/hydra/_internal/utils.py”, line 214, in run_and_report
raise ex
File “/opt/conda/lib/python3.8/site-packages/hydra/_internal/utils.py”, line 211, in run_and_report
return func()
File “/opt/conda/lib/python3.8/site-packages/hydra/_internal/utils.py”, line 368, in
lambda: hydra.run(
File “/opt/conda/lib/python3.8/site-packages/hydra/_internal/hydra.py”, line 87, in run
cfg = self.compose_config(
File “/opt/conda/lib/python3.8/site-packages/hydra/_internal/hydra.py”, line 564, in compose_config
cfg = self.config_loader.load_configuration(
File “/opt/conda/lib/python3.8/site-packages/hydra/_internal/config_loader_impl.py”, line 146, in load_configuration
return self._load_configuration_impl(
File “/opt/conda/lib/python3.8/site-packages/hydra/_internal/config_loader_impl.py”, line 249, in _load_configuration_impl
cfg = self._compose_config_from_defaults_list(
File “/opt/conda/lib/python3.8/site-packages/hydra/_internal/config_loader_impl.py”, line 515, in _compose_config_from_defaults_list
loaded = self._load_single_config(default=default, repo=repo)
File “/opt/conda/lib/python3.8/site-packages/hydra/_internal/config_loader_impl.py”, line 463, in _load_single_config
raise ConfigCompositionException(
hydra.errors.ConfigCompositionException: Error merging ‘finetune.yaml’ with schema

Morganh · December 23, 2022, 1:12pm

Please run below firstly.

rm -rf /opt/conda/lib/python3.8/site-packages/numba-0.53.1.dist-info/
rm /opt/conda/lib/python3.8/site-packages/llvmlite-0.36.0-py3.8.egg-info
pip uninstall llvmlite
pip uninstall numba
pip install numba==0.48
pip install librosa==0.8.1
pip install pynini

davesarmoury · December 23, 2022, 1:21pm

root@292d6c76bb3d:/opt/nvidia/tools# spectro_gen finetune -e /specs/spectro_gen/finetune.yaml -g 1 -k tlt_encode -r /results/spectro_gen/finetune -m
/data/tts_en_fastpitch_v1.4.0/tts_en_fastpitch_align.nemo train_dataset=/data/GLaDOS/merged_train.json validation_dataset=/data/GLaDOS/manifest_val.json prior_folder=/results/spectro_gen/finetune/prior_folder trainer.max_epochs=200 n_speakers=2 pitch_fmin=80.0 pitch_fmax=2048.0 pitch_avg=165.458 pitch_std=40.1891 trainer.precision=16

[NeMo W 2022-12-23 13:20:12 nemo_logging:349] /home/jenkins/agent/workspace/tlt-pytorch-main-nightly/conv_ai/tts/spectro_gen/scripts/finetune.py:273: UserWarning:
‘finetune.yaml’ is validated against ConfigStore schema with the same name.
This behavior is deprecated in Hydra 1.1 and will be removed in Hydra 1.2.
See https://hydra.cc/docs/next/upgrades/1.0_to_1.1/automatic_schema_matching for migration instructions.

Traceback (most recent call last):
File “/opt/conda/lib/python3.8/site-packages/hydra/_internal/config_loader_impl.py”, line 455, in _load_single_config
merged = OmegaConf.merge(schema.config, ret.config)
File “/opt/conda/lib/python3.8/site-packages/omegaconf/omegaconf.py”, line 264, in merge
target.merge_with(*configs[1:])
File “/opt/conda/lib/python3.8/site-packages/omegaconf/basecontainer.py”, line 438, in merge_with
self._format_and_raise(key=None, value=None, cause=e)
File “/opt/conda/lib/python3.8/site-packages/omegaconf/base.py”, line 190, in _format_and_raise
format_and_raise(
File “/opt/conda/lib/python3.8/site-packages/omegaconf/_utils.py”, line 741, in format_and_raise
_raise(ex, cause)
File “/opt/conda/lib/python3.8/site-packages/omegaconf/_utils.py”, line 719, in _raise
raise ex.with_traceback(sys.exc_info()[2]) # set end OC_CAUSE=1 for full backtrace
File “/opt/conda/lib/python3.8/site-packages/omegaconf/basecontainer.py”, line 436, in merge_with
self._merge_with(*others)
File “/opt/conda/lib/python3.8/site-packages/omegaconf/basecontainer.py”, line 460, in _merge_with
BaseContainer._map_merge(self, other)
File “/opt/conda/lib/python3.8/site-packages/omegaconf/basecontainer.py”, line 378, in _map_merge
dest[key] = src._get_node(key)
File “/opt/conda/lib/python3.8/site-packages/omegaconf/dictconfig.py”, line 310, in setitem
self._format_and_raise(
File “/opt/conda/lib/python3.8/site-packages/omegaconf/base.py”, line 190, in _format_and_raise
format_and_raise(
File “/opt/conda/lib/python3.8/site-packages/omegaconf/_utils.py”, line 741, in format_and_raise
_raise(ex, cause)
File “/opt/conda/lib/python3.8/site-packages/omegaconf/_utils.py”, line 719, in _raise
raise ex.with_traceback(sys.exc_info()[2]) # set end OC_CAUSE=1 for full backtrace
File “/opt/conda/lib/python3.8/site-packages/omegaconf/dictconfig.py”, line 308, in setitem
self.__set_impl(key=key, value=value)
File “/opt/conda/lib/python3.8/site-packages/omegaconf/dictconfig.py”, line 318, in __set_impl
self._set_item_impl(key, value)
File “/opt/conda/lib/python3.8/site-packages/omegaconf/basecontainer.py”, line 503, in _set_item_impl
target_node_ref = self._get_node(key)
File “/opt/conda/lib/python3.8/site-packages/omegaconf/dictconfig.py”, line 465, in _get_node
self._validate_get(key)
File “/opt/conda/lib/python3.8/site-packages/omegaconf/dictconfig.py”, line 166, in _validate_get
self._format_and_raise(
File “/opt/conda/lib/python3.8/site-packages/omegaconf/base.py”, line 190, in _format_and_raise
format_and_raise(
File “/opt/conda/lib/python3.8/site-packages/omegaconf/_utils.py”, line 821, in format_and_raise
_raise(ex, cause)
File “/opt/conda/lib/python3.8/site-packages/omegaconf/_utils.py”, line 719, in _raise
raise ex.with_traceback(sys.exc_info()[2]) # set end OC_CAUSE=1 for full backtrace
omegaconf.errors.ConfigKeyError: Key ‘phoneme_dict_path’ not in ‘DefaultConfig’
full_key: phoneme_dict_path
object_type=DefaultConfig

The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File “/home/jenkins/agent/workspace/tlt-pytorch-main-nightly/conv_ai/tts/spectro_gen/scripts/finetune.py”, line 273, in
File “/opt/conda/lib/python3.8/site-packages/nemo/core/config/hydra_runner.py”, line 104, in wrapper
_run_hydra(
File “/opt/conda/lib/python3.8/site-packages/hydra/_internal/utils.py”, line 367, in _run_hydra
run_and_report(
File “/opt/conda/lib/python3.8/site-packages/hydra/_internal/utils.py”, line 214, in run_and_report
raise ex
File “/opt/conda/lib/python3.8/site-packages/hydra/_internal/utils.py”, line 211, in run_and_report
return func()
File “/opt/conda/lib/python3.8/site-packages/hydra/_internal/utils.py”, line 368, in
lambda: hydra.run(
File “/opt/conda/lib/python3.8/site-packages/hydra/_internal/hydra.py”, line 87, in run
cfg = self.compose_config(
File “/opt/conda/lib/python3.8/site-packages/hydra/_internal/hydra.py”, line 564, in compose_config
cfg = self.config_loader.load_configuration(
File “/opt/conda/lib/python3.8/site-packages/hydra/_internal/config_loader_impl.py”, line 146, in load_configuration
return self._load_configuration_impl(
File “/opt/conda/lib/python3.8/site-packages/hydra/_internal/config_loader_impl.py”, line 249, in _load_configuration_impl
cfg = self._compose_config_from_defaults_list(
File “/opt/conda/lib/python3.8/site-packages/hydra/_internal/config_loader_impl.py”, line 515, in _compose_config_from_defaults_list
loaded = self._load_single_config(default=default, repo=repo)
File “/opt/conda/lib/python3.8/site-packages/hydra/_internal/config_loader_impl.py”, line 463, in _load_single_config
raise ConfigCompositionException(
hydra.errors.ConfigCompositionException: Error merging ‘finetune.yaml’ with schema

Morganh · December 23, 2022, 1:25pm

Can you attach your finetune.yaml ?

davesarmoury · December 23, 2022, 1:32pm

This should be the default one downloaded from tao spectro_gen download_specs
finetune.yaml (3.5 KB)

Morganh · December 23, 2022, 1:41pm

Suggest you to download specs files again.
I cannot find “phoneme_dict_path” in my finetune.yaml.

davesarmoury · December 23, 2022, 1:43pm

@Morganh Is there some other method I should be using to download the finetune.yaml file other than tao spectro_gen download_specs? I’ve deleted the directories and re downloaded all specs multiple times with the same result

Morganh · December 23, 2022, 1:57pm

There is not other method yet. Please run “spectro_gen download_specs” if you run into the 22.05 docker. Then, there is not ‘phoneme_dict_path’ .

davesarmoury · December 23, 2022, 3:00pm

@Morganh It seems closer, but is failing in the 22.05 docker with a new error:

Error executing job with overrides: [‘exp_manager.explicit_log_dir=/results/spectro_gen/finetune’, ‘trainer.gpus=1’, ‘restore_from=/data/tts_en_fastpitch_align.nemo’, ‘encryption_key=tlt_encode’, ‘train_dataset=/data/GLaDOS/LIES_train.json’, ‘validation_dataset=/data/GLaDOS/LIES_train.json’, ‘prior_folder=/results/spectro_gen/finetune/prior_folder’, ‘trainer.max_epochs=200’, ‘n_speakers=2’, ‘pitch_fmin=80.0’, ‘pitch_fmax=2048.0’, ‘pitch_avg=165.458’, ‘pitch_std=40.1891’, ‘trainer.precision=16’]
Traceback (most recent call last):
File “/home/jenkins/agent/workspace/tlt-pytorch-main-nightly/conv_ai/tts/spectro_gen/scripts/finetune.py”, line 273, in
File “/opt/conda/lib/python3.8/site-packages/nemo/core/config/hydra_runner.py”, line 104, in wrapper
_run_hydra(
File “/opt/conda/lib/python3.8/site-packages/hydra/_internal/utils.py”, line 367, in _run_hydra
run_and_report(
File “/opt/conda/lib/python3.8/site-packages/hydra/_internal/utils.py”, line 214, in run_and_report
raise ex
File “/opt/conda/lib/python3.8/site-packages/hydra/_internal/utils.py”, line 211, in run_and_report
return func()
File “/opt/conda/lib/python3.8/site-packages/hydra/_internal/utils.py”, line 368, in
lambda: hydra.run(
File “/opt/conda/lib/python3.8/site-packages/hydra/_internal/hydra.py”, line 110, in run
_ = ret.return_value
File “/opt/conda/lib/python3.8/site-packages/hydra/core/utils.py”, line 233, in return_value
raise self._return_value
File “/opt/conda/lib/python3.8/site-packages/hydra/core/utils.py”, line 160, in run_job
ret.return_value = task_function(task_cfg)
File “/home/jenkins/agent/workspace/tlt-pytorch-main-nightly/conv_ai/tts/spectro_gen/scripts/finetune.py”, line 227, in main
File “/home/jenkins/agent/workspace/tlt-pytorch-main-nightly/conv_ai/tts/spectro_gen/scripts/finetune.py”, line 158, in check_dataset_transcripts
ValueError: Please ensure that your finetuning data came from the finetuning notebook or the Nvidia custom voice tool.

This is happening no matter what dataset I use, even if I create a BS dataset just using only data from LJSpeech (setting speaker 1 and 2 to both be from that dataset)