I was able to run the DOPEWriter yesterday and it was producing no error.
Today I had quite a bunch of changes (not sure which may have caused error) and now while BasicWriter works the DOPEWriter doesn’t work.
#Basic Writer
# Initialize and attach writer
writer = rep.WriterRegistry.get("BasicWriter")
print("***************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************")
output_directory = os.getcwd() + "/_output_headless"
print("Outputting data to ", output_directory)
writer.initialize(
output_dir=output_directory,
rgb=True,
bounding_box_2d_tight=True,
semantic_segmentation=True,
instance_segmentation=True,
distance_to_image_plane=True,
bounding_box_3d=True,
occlusion=True,
normals=True,
)
RESOLUTION = (CONFIG["width"], CONFIG["height"])
driver_rp = rep.create.render_product(str(driver_cam_prim.GetPrimPath()), RESOLUTION)
pallet_rp = rep.create.render_product(pallet_cam, RESOLUTION)
forklift_rp = rep.create.render_product(top_view_cam, RESOLUTION)
writer.attach([driver_rp, forklift_rp, pallet_rp])
This one works and produces the following three folders:
Now when I comment the entire code above and replace it with the following, I get the errors listed below:
def _register_pose_annotator():
NodeConnectionTemplate = SyntheticData.NodeConnectionTemplate
if writer_helper == DOPEWriter:
rep.AnnotatorRegistry.register_annotator_from_node(
name="DopeSync",
input_rendervars=[
NodeConnectionTemplate(
"PostProcessDispatch", attributes_mapping={"outputs:swhFrameNumber": "inputs:syncValue"}
),
NodeConnectionTemplate("CameraParams", attributes_mapping={"outputs:exec": "inputs:execIn"}),
NodeConnectionTemplate("InstanceMapping", attributes_mapping={"outputs:exec": "inputs:execIn"}),
NodeConnectionTemplate("bounding_box_3d", attributes_mapping={"outputs:exec": "inputs:execIn"}),
NodeConnectionTemplate(
"OcclusionSDExportRawArray", attributes_mapping={"outputs:exec": "inputs:execIn"}
),
],
node_type_id="omni.graph.action.SyncGate",
)
rep.AnnotatorRegistry.register_annotator_from_node(
name="dope",
input_rendervars=[
NodeConnectionTemplate("DopeSync", attributes_mapping={"outputs:execOut": "inputs:exec"}),
"CameraParams",
"InstanceMapping",
NodeConnectionTemplate(
"bounding_box_3d", attributes_mapping={"outputs:data": "inputs:boundingBox3d"}
),
NodeConnectionTemplate(
"OcclusionSDExportRawArray", attributes_mapping={"outputs:data": "inputs:occlusion"}
),
],
node_type_id="omni.replicator.isaac.Dope",
init_params={
"width": width,
"height": height,
"cameraRotation": np.array(camera_rotation),
},
output_data_type=np.dtype(
[
("semanticId", "<u4"),
("visibility", "<f4"),
("location", "<f4", (3,)),
("rotation", "<f4", (4,)), # Quaternion
("projected_cuboid", "<f4", (9, 2)), # Includes Center
]
),
output_is_2d=True,
)
def _setup_writer():
"""Setup the OV Replicator dataset writer and attach it to a render product.
"""
if writer_helper == DOPEWriter:
writer = rep.WriterRegistry.get("DOPEWriter")
writer.initialize(
output_dir=os.getcwd() + '/_out_dope_headless',
class_name_to_index_map=CLASS_NAME_TO_INDEX,
use_s3=use_s3,
bucket_name=bucket,
endpoint_url=endpoint,
# rgb=True,
# bounding_box_2d_tight=True,
# semantic_segmentation=True,
# instance_segmentation=True,
# distance_to_image_plane=True,
# bounding_box_3d=True,
# occlusion=True,
# normals=True,
)
RESOLUTION = (CONFIG["width"], CONFIG["height"])
driver_rp = rep.create.render_product(str(driver_cam_prim.GetPrimPath()), RESOLUTION)
pallet_rp = rep.create.render_product(pallet_cam, RESOLUTION)
forklift_rp = rep.create.render_product(top_view_cam, RESOLUTION)
writer.attach([driver_rp, forklift_rp, pallet_rp])
_register_pose_annotator()
_setup_writer()
The error is:
2023-02-08 15:46:31 [32,105ms] [Warning] [carb.flatcache.plugin] UsdRelationship /Replicator/SDGPipeline/OgnGroup_08.inputs:prims has multiple targets, which is not supported
/buildAgent/work/f25a4639a4b1bdc1/source/foundation/FdFoundation.cpp (162) : invalid operation : Foundation object exists already. Only one instance per process can be created.
inspect.signature(rep.randomizer.scatter_2d): (surface_prims: Union[omni.replicator.core.scripts.utils.utils.ReplicatorItem, List[str]], seed: int = None, offset: int = 0, check_for_collisions: bool = False, input_prims: Union[omni.replicator.core.scripts.utils.utils.ReplicatorItem, List[str]] = None)
----------------------------------------------------------------
signature of rep.AnnotatorRegistry.register_annotator_from_node: (name: str, input_rendervars: List[Union[str, list, omni.syntheticdata.scripts.SyntheticData.SyntheticData.NodeConnectionTemplate]], node_type_id: str, init_params: dict = None, render_product_idxs: tuple = (0,), output_rendervars: List[Union[str, list]] = None, output_data_type: Any = None, output_is_2d: bool = False, output_channels: int = 1, is_gpu_enabled: bool = True) -> omni.replicator.core.scripts.annotators.Annotator
---------------------------------------------------
class to index: {'Cardbox': 1, 'Forklift': 2, 'Pallet': 3}
2023-02-08 15:46:43 [44,118ms] [Warning] [carb.flatcache.plugin] PathToAttributesMap (0x1d5084a0) contains attributes with duplicate name "camera" with different types but same per-element size. Data may become corrupted during request to move elements between buckets!
2023-02-08 15:46:44 [44,817ms] [Warning] [carb.scenerenderer-rtx.plugin] Auto-enabling Sampled Direct Lighting because scene has 12 lights (above 10 light threshold)
2023-02-08 15:47:05 [66,085ms] [Error] [omni.graph.core.plugin] /Replicator/SDGPipeline/OgnScatter2D: Assertion raised in compute - Randomization timed out, cannot find a configuration to randomize prim locations and prevent collisions.
File "/home/mona/.local/share/ov/pkg/isaac_sim-2022.2.0/extscache/omni.replicator.core-1.6.4+104.1.lx64.r.cp37/omni/replicator/core/ogn/python/_impl/nodes/OgnScatter2D.py", line 247, in compute
"Randomization timed out, cannot find a configuration to randomize prim locations and prevent collisions."
2023-02-08 15:47:08 [69,227ms] [Error] [omni.graph.core.plugin] /Render/PostProcess/SDGPipeline/MultipleRenderProducts_3Products_DOPEWriterWriter: Assertion raised in compute - cannot reshape array of size 84 into shape (1024,1024,newaxis)
File "/home/mona/.local/share/ov/pkg/isaac_sim-2022.2.0/extscache/omni.replicator.core-1.6.4+104.1.lx64.r.cp37/omni/replicator/core/ogn/python/_impl/nodes/OgnWriter.py", line 86, in compute
params, annotator_params, params["__device"].get()
File "/home/mona/.local/share/ov/pkg/isaac_sim-2022.2.0/extscache/omni.replicator.core-1.6.4+104.1.lx64.r.cp37/omni/replicator/core/scripts/utils/annotator_utils.py", line 215, in _get_annotator_data
**common_params,
File "/home/mona/.local/share/ov/pkg/isaac_sim-2022.2.0/extscache/omni.replicator.core-1.6.4+104.1.lx64.r.cp37/omni/replicator/core/scripts/utils/annotator_utils.py", line 93, in _reshape_output_attr
return np.squeeze(data.reshape(height, width, -1))
2023-02-08 15:47:11 [72,030ms] [Error] [omni.graph.core.plugin] /Render/PostProcess/SDGPipeline/MultipleRenderProducts_3Products_DOPEWriterWriter: Assertion raised in compute - cannot reshape array of size 78 into shape (1024,1024,newaxis)
File "/home/mona/.local/share/ov/pkg/isaac_sim-2022.2.0/extscache/omni.replicator.core-1.6.4+104.1.lx64.r.cp37/omni/replicator/core/ogn/python/_impl/nodes/OgnWriter.py", line 86, in compute
params, annotator_params, params["__device"].get()
File "/home/mona/.local/share/ov/pkg/isaac_sim-2022.2.0/extscache/omni.replicator.core-1.6.4+104.1.lx64.r.cp37/omni/replicator/core/scripts/utils/annotator_utils.py", line 215, in _get_annotator_data
**common_params,
File "/home/mona/.local/share/ov/pkg/isaac_sim-2022.2.0/extscache/omni.replicator.core-1.6.4+104.1.lx64.r.cp37/omni/replicator/core/scripts/utils/annotator_utils.py", line 93, in _reshape_output_attr
return np.squeeze(data.reshape(height, width, -1))
2023-02-08 15:47:14 [74,670ms] [Error] [omni.graph.core.plugin] /Render/PostProcess/SDGPipeline/MultipleRenderProducts_3Products_DOPEWriterWriter: Assertion raised in compute - cannot reshape array of size 94 into shape (1024,1024,newaxis)
File "/home/mona/.local/share/ov/pkg/isaac_sim-2022.2.0/extscache/omni.replicator.core-1.6.4+104.1.lx64.r.cp37/omni/replicator/core/ogn/python/_impl/nodes/OgnWriter.py", line 86, in compute
params, annotator_params, params["__device"].get()
File "/home/mona/.local/share/ov/pkg/isaac_sim-2022.2.0/extscache/omni.replicator.core-1.6.4+104.1.lx64.r.cp37/omni/replicator/core/scripts/utils/annotator_utils.py", line 215, in _get_annotator_data
**common_params,
File "/home/mona/.local/share/ov/pkg/isaac_sim-2022.2.0/extscache/omni.replicator.core-1.6.4+104.1.lx64.r.cp37/omni/replicator/core/scripts/utils/annotator_utils.py", line 93, in _reshape_output_attr
return np.squeeze(data.reshape(height, width, -1))
2023-02-08 15:47:17 [77,828ms] [Error] [omni.graph.core.plugin] /Render/PostProcess/SDGPipeline/MultipleRenderProducts_3Products_DOPEWriterWriter: Assertion raised in compute - cannot reshape array of size 88 into shape (1024,1024,newaxis)
File "/home/mona/.local/share/ov/pkg/isaac_sim-2022.2.0/extscache/omni.replicator.core-1.6.4+104.1.lx64.r.cp37/omni/replicator/core/ogn/python/_impl/nodes/OgnWriter.py", line 86, in compute
params, annotator_params, params["__device"].get()
File "/home/mona/.local/share/ov/pkg/isaac_sim-2022.2.0/extscache/omni.replicator.core-1.6.4+104.1.lx64.r.cp37/omni/replicator/core/scripts/utils/annotator_utils.py", line 215, in _get_annotator_data
**common_params,
File "/home/mona/.local/share/ov/pkg/isaac_sim-2022.2.0/extscache/omni.replicator.core-1.6.4+104.1.lx64.r.cp37/omni/replicator/core/scripts/utils/annotator_utils.py", line 93, in _reshape_output_attr
return np.squeeze(data.reshape(height, width, -1))
2023-02-08 15:47:19 [80,527ms] [Error] [omni.graph.core.plugin] /Render/PostProcess/SDGPipeline/MultipleRenderProducts_3Products_DOPEWriterWriter: Assertion raised in compute - cannot reshape array of size 86 into shape (1024,1024,newaxis)
File "/home/mona/.local/share/ov/pkg/isaac_sim-2022.2.0/extscache/omni.replicator.core-1.6.4+104.1.lx64.r.cp37/omni/replicator/core/ogn/python/_impl/nodes/OgnWriter.py", line 86, in compute
params, annotator_params, params["__device"].get()
File "/home/mona/.local/share/ov/pkg/isaac_sim-2022.2.0/extscache/omni.replicator.core-1.6.4+104.1.lx64.r.cp37/omni/replicator/core/scripts/utils/annotator_utils.py", line 215, in _get_annotator_data
**common_params,
File "/home/mona/.local/share/ov/pkg/isaac_sim-2022.2.0/extscache/omni.replicator.core-1.6.4+104.1.lx64.r.cp37/omni/replicator/core/scripts/utils/annotator_utils.py", line 93, in _reshape_output_attr
return np.squeeze(data.reshape(height, width, -1))
2023-02-08 15:47:22 [83,502ms] [Error] [omni.graph.core.plugin] /Render/PostProcess/SDGPipeline/MultipleRenderProducts_3Products_DOPEWriterWriter: Assertion raised in compute - cannot reshape array of size 82 into shape (1024,1024,newaxis)
File "/home/mona/.local/share/ov/pkg/isaac_sim-2022.2.0/extscache/omni.replicator.core-1.6.4+104.1.lx64.r.cp37/omni/replicator/core/ogn/python/_impl/nodes/OgnWriter.py", line 86, in compute
params, annotator_params, params["__device"].get()
File "/home/mona/.local/share/ov/pkg/isaac_sim-2022.2.0/extscache/omni.replicator.core-1.6.4+104.1.lx64.r.cp37/omni/replicator/core/scripts/utils/annotator_utils.py", line 215, in _get_annotator_data
**common_params,
File "/home/mona/.local/share/ov/pkg/isaac_sim-2022.2.0/extscache/omni.replicator.core-1.6.4+104.1.lx64.r.cp37/omni/replicator/core/scripts/utils/annotator_utils.py", line 93, in _reshape_output_attr
return np.squeeze(data.reshape(height, width, -1))
2023-02-08 15:47:25 [86,369ms] [Error] [omni.graph.core.plugin] /Render/PostProcess/SDGPipeline/MultipleRenderProducts_3Products_DOPEWriterWriter: Assertion raised in compute - cannot reshape array of size 86 into shape (1024,1024,newaxis)
File "/home/mona/.local/share/ov/pkg/isaac_sim-2022.2.0/extscache/omni.replicator.core-1.6.4+104.1.lx64.r.cp37/omni/replicator/core/ogn/python/_impl/nodes/OgnWriter.py", line 86, in compute
params, annotator_params, params["__device"].get()
File "/home/mona/.local/share/ov/pkg/isaac_sim-2022.2.0/extscache/omni.replicator.core-1.6.4+104.1.lx64.r.cp37/omni/replicator/core/scripts/utils/annotator_utils.py", line 215, in _get_annotator_data
**common_params,
File "/home/mona/.local/share/ov/pkg/isaac_sim-2022.2.0/extscache/omni.replicator.core-1.6.4+104.1.lx64.r.cp37/omni/replicator/core/scripts/utils/annotator_utils.py", line 93, in _reshape_output_attr
return np.squeeze(data.reshape(height, width, -1))
2023-02-08 15:47:28 [89,164ms] [Error] [omni.graph.core.plugin] /Render/PostProcess/SDGPipeline/MultipleRenderProducts_3Products_DOPEWriterWriter: Assertion raised in compute - cannot reshape array of size 96 into shape (1024,1024,newaxis)
File "/home/mona/.local/share/ov/pkg/isaac_sim-2022.2.0/extscache/omni.replicator.core-1.6.4+104.1.lx64.r.cp37/omni/replicator/core/ogn/python/_impl/nodes/OgnWriter.py", line 86, in compute
params, annotator_params, params["__device"].get()
File "/home/mona/.local/share/ov/pkg/isaac_sim-2022.2.0/extscache/omni.replicator.core-1.6.4+104.1.lx64.r.cp37/omni/replicator/core/scripts/utils/annotator_utils.py", line 215, in _get_annotator_data
**common_params,
File "/home/mona/.local/share/ov/pkg/isaac_sim-2022.2.0/extscache/omni.replicator.core-1.6.4+104.1.lx64.r.cp37/omni/replicator/core/scripts/utils/annotator_utils.py", line 93, in _reshape_output_attr
return np.squeeze(data.reshape(height, width, -1))
2023-02-08 15:47:31 [92,078ms] [Error] [omni.graph.core.plugin] /Render/PostProcess/SDGPipeline/MultipleRenderProducts_3Products_DOPEWriterWriter: Assertion raised in compute - cannot reshape array of size 76 into shape (1024,1024,newaxis)
File "/home/mona/.local/share/ov/pkg/isaac_sim-2022.2.0/extscache/omni.replicator.core-1.6.4+104.1.lx64.r.cp37/omni/replicator/core/ogn/python/_impl/nodes/OgnWriter.py", line 86, in compute
params, annotator_params, params["__device"].get()
File "/home/mona/.local/share/ov/pkg/isaac_sim-2022.2.0/extscache/omni.replicator.core-1.6.4+104.1.lx64.r.cp37/omni/replicator/core/scripts/utils/annotator_utils.py", line 215, in _get_annotator_data
**common_params,
File "/home/mona/.local/share/ov/pkg/isaac_sim-2022.2.0/extscache/omni.replicator.core-1.6.4+104.1.lx64.r.cp37/omni/replicator/core/scripts/utils/annotator_utils.py", line 93, in _reshape_output_attr
return np.squeeze(data.reshape(height, width, -1))
2023-02-08 15:47:33 [94,458ms] [Error] [omni.graph.core.plugin] /Render/PostProcess/SDGPipeline/MultipleRenderProducts_3Products_DOPEWriterWriter: Assertion raised in compute - cannot reshape array of size 90 into shape (1024,1024,newaxis)
File "/home/mona/.local/share/ov/pkg/isaac_sim-2022.2.0/extscache/omni.replicator.core-1.6.4+104.1.lx64.r.cp37/omni/replicator/core/ogn/python/_impl/nodes/OgnWriter.py", line 86, in compute
params, annotator_params, params["__device"].get()
File "/home/mona/.local/share/ov/pkg/isaac_sim-2022.2.0/extscache/omni.replicator.core-1.6.4+104.1.lx64.r.cp37/omni/replicator/core/scripts/utils/annotator_utils.py", line 215, in _get_annotator_data
**common_params,
File "/home/mona/.local/share/ov/pkg/isaac_sim-2022.2.0/extscache/omni.replicator.core-1.6.4+104.1.lx64.r.cp37/omni/replicator/core/scripts/utils/annotator_utils.py", line 93, in _reshape_output_attr
return np.squeeze(data.reshape(height, width, -1))
2023-02-08 15:47:35 [95,655ms] [Error] [omni.gpucompute-cuda.plugin] CUDA error in freeAsync: operation not supported: ../../../source/plugins/omni.gpucompute-cuda/GpuCompute-Cuda.cpp: 145 (device 0)
2023-02-08 15:47:35 [95,655ms] [Error] [omni.gpucompute-cuda.plugin] CUDA error in freeAsync: operation not supported: ../../../source/plugins/omni.gpucompute-cuda/GpuCompute-Cuda.cpp: 145 (device 0)
2023-02-08 15:47:35 [95,655ms] [Error] [omni.gpucompute-cuda.plugin] CUDA error in freeAsync: operation not supported: ../../../source/plugins/omni.gpucompute-cuda/GpuCompute-Cuda.cpp: 145 (device 0)
2023-02-08 15:47:35 [95,655ms] [Error] [omni.gpucompute-cuda.plugin] CUDA error in freeAsync: operation not supported: ../../../source/plugins/omni.gpucompute-cuda/GpuCompute-Cuda.cpp: 145 (device 0)
2023-02-08 15:47:35 [95,655ms] [Error] [omni.gpucompute-cuda.plugin] CUDA error in freeAsync: operation not supported: ../../../source/plugins/omni.gpucompute-cuda/GpuCompute-Cuda.cpp: 145 (device 0)
2023-02-08 15:47:35 [95,655ms] [Error] [omni.gpucompute-cuda.plugin] CUDA error in freeAsync: operation not supported: ../../../source/plugins/omni.gpucompute-cuda/GpuCompute-Cuda.cpp: 145 (device 0)
2023-02-08 15:47:35 [95,655ms] [Error] [omni.gpucompute-cuda.plugin] CUDA error in freeAsync: operation not supported: ../../../source/plugins/omni.gpucompute-cuda/GpuCompute-Cuda.cpp: 145 (device 0)
2023-02-08 15:47:35 [95,655ms] [Error] [omni.gpucompute-cuda.plugin] CUDA error in freeAsync: operation not supported: ../../../source/plugins/omni.gpucompute-cuda/GpuCompute-Cuda.cpp: 145 (device 0)
2023-02-08 15:47:35 [95,655ms] [Error] [omni.gpucompute-cuda.plugin] CUDA error in freeAsync: operation not supported: ../../../source/plugins/omni.gpucompute-cuda/GpuCompute-Cuda.cpp: 145 (device 0)
2023-02-08 15:47:35 [95,655ms] [Error] [omni.gpucompute-cuda.plugin] CUDA error in freeAsync: operation not supported: ../../../source/plugins/omni.gpucompute-cuda/GpuCompute-Cuda.cpp: 145 (device 0)
2023-02-08 15:47:35 [95,655ms] [Error] [omni.gpucompute-cuda.plugin] CUDA error in freeAsync: operation not supported: ../../../source/plugins/omni.gpucompute-cuda/GpuCompute-Cuda.cpp: 145 (device 0)
2023-02-08 15:47:35 [95,655ms] [Error] [omni.gpucompute-cuda.plugin] CUDA error in freeAsync: operation not supported: ../../../source/plugins/omni.gpucompute-cuda/GpuCompute-Cuda.cpp: 145 (device 0)
2023-02-08 15:47:35 [95,655ms] [Error] [omni.gpucompute-cuda.plugin] CUDA error in freeAsync: operation not supported: ../../../source/plugins/omni.gpucompute-cuda/GpuCompute-Cuda.cpp: 145 (device 0)
2023-02-08 15:47:35 [95,655ms] [Error] [omni.gpucompute-cuda.plugin] CUDA error in freeAsync: operation not supported: ../../../source/plugins/omni.gpucompute-cuda/GpuCompute-Cuda.cpp: 145 (device 0)
2023-02-08 15:47:35 [95,655ms] [Error] [omni.gpucompute-cuda.plugin] CUDA error in freeAsync: operation not supported: ../../../source/plugins/omni.gpucompute-cuda/GpuCompute-Cuda.cpp: 145 (device 0)
2023-02-08 15:47:35 [95,655ms] [Error] [omni.gpucompute-cuda.plugin] CUDA error in freeAsync: operation not supported: ../../../source/plugins/omni.gpucompute-cuda/GpuCompute-Cuda.cpp: 145 (device 0)
2023-02-08 15:47:35 [95,655ms] [Error] [omni.gpucompute-cuda.plugin] CUDA error in freeAsync: operation not supported: ../../../source/plugins/omni.gpucompute-cuda/GpuCompute-Cuda.cpp: 145 (device 0)
2023-02-08 15:47:35 [95,770ms] [Warning] [carb] Plugin interface for a client: omni.hydratexture.plugin was already released.
2023-02-08 15:47:35 [95,770ms] [Warning] [carb] Plugin interface for a client: omni.hydratexture.plugin was already released.
[95.782s] Simulation App Shutting Down
And no images get produces.
This happens when for the code I copied for DOPEWriter, I convert output_is_2d=False, to output_is_2d=True,
and I have no error when it is False and it produces something like this:
and here’s the messages I get due to run (but images are produced):
2023-02-08 15:53:58 [44,825ms] [Warning] [carb.flatcache.plugin] PathToAttributesMap (0x1ca36140) contains attributes with duplicate name "camera" with different types but same per-element size. Data may become corrupted during request to move elements between buckets!
2023-02-08 15:53:59 [45,488ms] [Warning] [carb.scenerenderer-rtx.plugin] Auto-enabling Sampled Direct Lighting because scene has 12 lights (above 10 light threshold)
2023-02-08 15:54:20 [66,517ms] [Error] [omni.graph.core.plugin] /Replicator/SDGPipeline/OgnScatter2D: Assertion raised in compute - Randomization timed out, cannot find a configuration to randomize prim locations and prevent collisions.
File "/home/mona/.local/share/ov/pkg/isaac_sim-2022.2.0/extscache/omni.replicator.core-1.6.4+104.1.lx64.r.cp37/omni/replicator/core/ogn/python/_impl/nodes/OgnScatter2D.py", line 247, in compute
"Randomization timed out, cannot find a configuration to randomize prim locations and prevent collisions."
2023-02-08 15:54:23 [69,805ms] [Warning] [omni.replicator.core.scripts.writers] Writer version not specified in the writer metadata
2023-02-08 15:54:50 [96,985ms] [Error] [omni.gpucompute-cuda.plugin] CUDA error in freeAsync: operation not supported: ../../../source/plugins/omni.gpucompute-cuda/GpuCompute-Cuda.cpp: 145 (device 0)
2023-02-08 15:54:50 [96,985ms] [Error] [omni.gpucompute-cuda.plugin] CUDA error in freeAsync: operation not supported: ../../../source/plugins/omni.gpucompute-cuda/GpuCompute-Cuda.cpp: 145 (device 0)
2023-02-08 15:54:50 [96,985ms] [Error] [omni.gpucompute-cuda.plugin] CUDA error in freeAsync: operation not supported: ../../../source/plugins/omni.gpucompute-cuda/GpuCompute-Cuda.cpp: 145 (device 0)
2023-02-08 15:54:50 [96,985ms] [Error] [omni.gpucompute-cuda.plugin] CUDA error in freeAsync: operation not supported: ../../../source/plugins/omni.gpucompute-cuda/GpuCompute-Cuda.cpp: 145 (device 0)
2023-02-08 15:54:50 [96,985ms] [Error] [omni.gpucompute-cuda.plugin] CUDA error in freeAsync: operation not supported: ../../../source/plugins/omni.gpucompute-cuda/GpuCompute-Cuda.cpp: 145 (device 0)
2023-02-08 15:54:50 [96,985ms] [Error] [omni.gpucompute-cuda.plugin] CUDA error in freeAsync: operation not supported: ../../../source/plugins/omni.gpucompute-cuda/GpuCompute-Cuda.cpp: 145 (device 0)
2023-02-08 15:54:50 [96,985ms] [Error] [omni.gpucompute-cuda.plugin] CUDA error in freeAsync: operation not supported: ../../../source/plugins/omni.gpucompute-cuda/GpuCompute-Cuda.cpp: 145 (device 0)
2023-02-08 15:54:50 [96,985ms] [Error] [omni.gpucompute-cuda.plugin] CUDA error in freeAsync: operation not supported: ../../../source/plugins/omni.gpucompute-cuda/GpuCompute-Cuda.cpp: 145 (device 0)
2023-02-08 15:54:50 [96,985ms] [Error] [omni.gpucompute-cuda.plugin] CUDA error in freeAsync: operation not supported: ../../../source/plugins/omni.gpucompute-cuda/GpuCompute-Cuda.cpp: 145 (device 0)
2023-02-08 15:54:50 [96,985ms] [Error] [omni.gpucompute-cuda.plugin] CUDA error in freeAsync: operation not supported: ../../../source/plugins/omni.gpucompute-cuda/GpuCompute-Cuda.cpp: 145 (device 0)
2023-02-08 15:54:50 [96,985ms] [Error] [omni.gpucompute-cuda.plugin] CUDA error in freeAsync: operation not supported: ../../../source/plugins/omni.gpucompute-cuda/GpuCompute-Cuda.cpp: 145 (device 0)
2023-02-08 15:54:50 [96,985ms] [Error] [omni.gpucompute-cuda.plugin] CUDA error in freeAsync: operation not supported: ../../../source/plugins/omni.gpucompute-cuda/GpuCompute-Cuda.cpp: 145 (device 0)
2023-02-08 15:54:50 [96,985ms] [Error] [omni.gpucompute-cuda.plugin] CUDA error in freeAsync: operation not supported: ../../../source/plugins/omni.gpucompute-cuda/GpuCompute-Cuda.cpp: 145 (device 0)
2023-02-08 15:54:50 [96,985ms] [Error] [omni.gpucompute-cuda.plugin] CUDA error in freeAsync: operation not supported: ../../../source/plugins/omni.gpucompute-cuda/GpuCompute-Cuda.cpp: 145 (device 0)
2023-02-08 15:54:50 [96,985ms] [Error] [omni.gpucompute-cuda.plugin] CUDA error in freeAsync: operation not supported: ../../../source/plugins/omni.gpucompute-cuda/GpuCompute-Cuda.cpp: 145 (device 0)
2023-02-08 15:54:50 [96,985ms] [Error] [omni.gpucompute-cuda.plugin] CUDA error in freeAsync: operation not supported: ../../../source/plugins/omni.gpucompute-cuda/GpuCompute-Cuda.cpp: 145 (device 0)
2023-02-08 15:54:50 [96,985ms] [Error] [omni.gpucompute-cuda.plugin] CUDA error in freeAsync: operation not supported: ../../../source/plugins/omni.gpucompute-cuda/GpuCompute-Cuda.cpp: 145 (device 0)
2023-02-08 15:54:50 [97,156ms] [Warning] [carb] Plugin interface for a client: omni.hydratexture.plugin was already released.
2023-02-08 15:54:50 [97,156ms] [Warning] [carb] Plugin interface for a client: omni.hydratexture.plugin was already released.
[97.170s] Simulation App Shutting Down