Hi
Trying to run Downloadable NIM on RTX 4090
What I an doing wrong / Compose Output:
llama-3.1-8b-instruct-RTX-1 | raise ValueError(f"The model {request.model} does not exist.")
llama-3.1-8b-instruct-RTX-1 | ValueError: The model meta/llama-3.1-8b-instruct-RTX does not exist.
llama-3.1-8b-instruct-RTX-1 | await self.middleware_stack(scope, receive, send)
llama-3.1-8b-instruct-RTX-1 | File “/opt/nim/llm/.venv/lib/python3.12/site-packages/starlette/middleware/errors.py”, line 187, in call
llama-3.1-8b-instruct-RTX-1 | raise exc
llama-3.1-8b-instruct-RTX-1 | File “/opt/nim/llm/.venv/lib/python3.12/site-packages/starlette/middleware/errors.py”, line 165, in call
llama-3.1-8b-instruct-RTX-1 | await self.app(scope, receive, _send)
llama-3.1-8b-instruct-RTX-1 | File “/opt/nim/llm/.venv/lib/python3.12/site-packages/starlette/middleware/cors.py”, line 85, in call
llama-3.1-8b-instruct-RTX-1 | await self.app(scope, receive, send)
llama-3.1-8b-instruct-RTX-1 | File “/opt/nim/llm/.venv/lib/python3.12/site-packages/starlette/middleware/exceptions.py”, line 62, in call
llama-3.1-8b-instruct-RTX-1 | await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send)
llama-3.1-8b-instruct-RTX-1 | File “/opt/nim/llm/.venv/lib/python3.12/site-packages/starlette/_exception_handler.py”, line 62, in wrapped_app
llama-3.1-8b-instruct-RTX-1 | raise exc
llama-3.1-8b-instruct-RTX-1 | File “/opt/nim/llm/.venv/lib/python3.12/site-packages/starlette/_exception_handler.py”, line 51, in wrapped_app
llama-3.1-8b-instruct-RTX-1 | await app(scope, receive, sender)
llama-3.1-8b-instruct-RTX-1 | File “/opt/nim/llm/.venv/lib/python3.12/site-packages/starlette/routing.py”, line 715, in call
llama-3.1-8b-instruct-RTX-1 | await self.middleware_stack(scope, receive, send)
llama-3.1-8b-instruct-RTX-1 | File “/opt/nim/llm/.venv/lib/python3.12/site-packages/starlette/routing.py”, line 735, in app
llama-3.1-8b-instruct-RTX-1 | await route.handle(scope, receive, send)
llama-3.1-8b-instruct-RTX-1 | File “/opt/nim/llm/.venv/lib/python3.12/site-packages/starlette/routing.py”, line 288, in handle
llama-3.1-8b-instruct-RTX-1 | await self.app(scope, receive, send)
llama-3.1-8b-instruct-RTX-1 | File “/opt/nim/llm/.venv/lib/python3.12/site-packages/starlette/routing.py”, line 76, in app
llama-3.1-8b-instruct-RTX-1 | await wrap_app_handling_exceptions(app, request)(scope, receive, send)
llama-3.1-8b-instruct-RTX-1 | File “/opt/nim/llm/.venv/lib/python3.12/site-packages/starlette/_exception_handler.py”, line 62, in wrapped_app
llama-3.1-8b-instruct-RTX-1 | raise exc
llama-3.1-8b-instruct-RTX-1 | File “/opt/nim/llm/.venv/lib/python3.12/site-packages/starlette/_exception_handler.py”, line 51, in wrapped_app
llama-3.1-8b-instruct-RTX-1 | await app(scope, receive, sender)
llama-3.1-8b-instruct-RTX-1 | File “/opt/nim/llm/.venv/lib/python3.12/site-packages/starlette/routing.py”, line 73, in app
llama-3.1-8b-instruct-RTX-1 | response = await f(request)
llama-3.1-8b-instruct-RTX-1 | ^^^^^^^^^^^^^^^^
llama-3.1-8b-instruct-RTX-1 | File “/opt/nim/llm/.venv/lib/python3.12/site-packages/fastapi/routing.py”, line 301, in app
llama-3.1-8b-instruct-RTX-1 | raw_response = await run_endpoint_function(
llama-3.1-8b-instruct-RTX-1 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
llama-3.1-8b-instruct-RTX-1 | File “/opt/nim/llm/.venv/lib/python3.12/site-packages/fastapi/routing.py”, line 212, in run_endpoint_function
llama-3.1-8b-instruct-RTX-1 | return await dependant.call(**values)
llama-3.1-8b-instruct-RTX-1 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
llama-3.1-8b-instruct-RTX-1 | File “/opt/nim/llm/nim_llm_sdk/entrypoints/openai/api_server.py”, line 657, in create_chat_completion
llama-3.1-8b-instruct-RTX-1 | ) = self.openai_serving_chat._maybe_get_adapters(request)
llama-3.1-8b-instruct-RTX-1 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
llama-3.1-8b-instruct-RTX-1 | File “/opt/nim/llm/.venv/lib/python3.12/site-packages/vllm/entrypoints/openai/serving_engine.py”, line 235, in _maybe_get_adapters
llama-3.1-8b-instruct-RTX-1 | raise ValueError(f"The model {request.model} does not exist.")
llama-3.1-8b-instruct-RTX-1 | ValueError: The model meta/llama-3.1-8b-instruct-RTX does not exist.