Weights are already downloaded, but failure occurs during load (e.g., local model with adapter but the config hasn't been downloaded). Likely a race condition with cleanup logic.
2023-11-12T05:38:04.192534Z ERROR text_generation_client: router/client/src/lib.rs:33: Server error: Can't find 'adapter_config.json' at '/data/models--arnavgrg--codealpaca_v3/sn
apshots/834b33af35ff5965ea3e4bc18b51ad5d65da7466'
2023-11-12T05:38:04.192612Z INFO text_generation_router::loader: router/src/loader.rs:184: FAILED loading adapter /data/models--arnavgrg--codealpaca_v3/snapshots/834b33af35ff596
5ea3e4bc18b51ad5d65da7466
2023-11-12T05:38:04.192682Z ERROR text_generation_router::queue: router/src/queue.rs:240: adapter /data/models--arnavgrg--codealpaca_v3/snapshots/834b33af35ff5965ea3e4bc18b51ad5d
65da7466 not found in queue_map
Backtrace [{ fn: "text_generation_router::queue::AdapterQueuesState::set_status", file: "./src/queue.rs", line: 241 }, { fn: "text_generation_router::loader::loader_tas[132/1877]
re}}", file: "./src/loader.rs", line: 186 }, { fn: "<core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll", file: "/build/rustc-v6rcRM/rustc-1.66.1+d
fsg0ubuntu1~llvm/library/core/src/future/mod.rs", line: 91 }, { fn: "tokio::runtime::task::core::Core<T,S>::poll::{{closure}}", file: "/root/.cargo/registry/src/github.com-1ecc62
99db9ec823/tokio-1.29.1/src/runtime/task/core.rs", line: 311 }, { fn: "tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut", file: "/root/.cargo/registry/src/github.com-1ecc62
99db9ec823/tokio-1.29.1/src/loom/std/unsafe_cell.rs", line: 14 }, { fn: "tokio::runtime::task::core::Core<T,S>::poll", file: "/root/.cargo/registry/src/github.com-1ecc6299db9ec82
3/tokio-1.29.1/src/runtime/task/core.rs", line: 300 }, { fn: "tokio::runtime::task::harness::poll_future::{{closure}}", file: "/root/.cargo/registry/src/github.com-1ecc6299db9ec8
23/tokio-1.29.1/src/runtime/task/harness.rs", line: 476 }, { fn: "<core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once", file: "/build/ru
stc-v6rcRM/rustc-1.66.1+dfsg0ubuntu1~llvm/library/core/src/panic/unwind_safe.rs", line: 271 }, { fn: "std::panicking::try::do_call", file: "/build/rustc-v6rcRM/rustc-1.66.1+dfsg0
ubuntu1~llvm/library/std/src/panicking.rs", line: 483 }, { fn: "__rust_try" }, { fn: "std::panicking::try", file: "/build/rustc-v6rcRM/rustc-1.66.1+dfsg0ubuntu1~llvm/library/std/
src/panicking.rs", line: 447 }, { fn: "std::panic::catch_unwind", file: "/build/rustc-v6rcRM/rustc-1.66.1+dfsg0ubuntu1~llvm/library/std/src/panic.rs", line: 137 }, { fn: "tokio::
runtime::task::harness::poll_future", file: "/root/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.29.1/src/runtime/task/harness.rs", line: 464 }, { fn: "tokio::runtime::
task::harness::Harness<T,S>::poll_inner", file: "/root/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.29.1/src/runtime/task/harness.rs", line: 198 }, { fn: "tokio::runti
me::task::harness::Harness<T,S>::poll", file: "/root/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.29.1/src/runtime/task/harness.rs", line: 152 }, { fn: "tokio::runtime
::task::raw::poll", file: "/root/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.29.1/src/runtime/task/raw.rs", line: 276 }, { fn: "tokio::runtime::task::raw::RawTask::po
ll", file: "/root/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.29.1/src/runtime/task/raw.rs", line: 200 }, { fn: "tokio::runtime::task::LocalNotified<S>::run", file: "
/root/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.29.1/src/runtime/task/mod.rs", line: 400 }, { fn: "tokio::runtime::scheduler::multi_thread::worker::Context::run_tas
k::{{closure}}", file: "/root/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.29.1/src/runtime/scheduler/multi_thread/worker.rs", line: 639 }, { fn: "tokio::runtime::coop
::with_budget", file: "/root/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.29.1/src/runtime/coop.rs", line: 107 }, { fn: "tokio::runtime::coop::budget", file: "/root/.c
argo/registry/src/github.com-1ecc6299db9ec823/tokio-1.29.1/src/runtime/coop.rs", line: 73 }, { fn: "tokio::runtime::scheduler::multi_thread::worker::Context::run_task", file: "/r
oot/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.29.1/src/runtime/scheduler/multi_thread/worker.rs", line: 575 }, { fn: "tokio::runtime::scheduler::multi_thread::worke
r::Context::run", file: "/root/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.29.1/src/runtime/scheduler/multi_thread/worker.rs", line: 526 }, { fn: "tokio::runtime::sch
eduler::multi_thread::worker::run::{{closure}}::{{closure}}", file: "/root/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.29.1/src/runtime/scheduler/multi_thread/worker.
rs", line: 491 }, { fn: "tokio::runtime::context::scoped::Scoped<T>::set", file: "/root/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.29.1/src/runtime/context/scoped.rs
", line: 40 }, { fn: "tokio::runtime::context::set_scheduler::{{closure}}", file: "/root/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.29.1/src/runtime/context.rs", lin
e: 176 }, { fn: "std::thread::local::LocalKey<T>::try_with", file: "/build/rustc-v6rcRM/rustc-1.66.1+dfsg0ubuntu1~llvm/library/std/src/thread/local.rs", line: 446 }, { fn: "std::
thread::local::LocalKey<T>::with", file: "/build/rustc-v6rcRM/rustc-1.66.1+dfsg0ubuntu1~llvm/library/std/src/thread/local.rs", line: 422 }, { fn: "tokio::runtime::context::set_sc
heduler", file: "/root/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.29.1/src/runtime/context.rs", line: 176 }, { fn: "tokio::runtime::scheduler::multi_thread::worker::
run::{{closure}}", file: "/root/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.29.1/src/runtime/scheduler/multi_thread/worker.rs", line: 486 }, { fn: "tokio::runtime::co
ntext::runtime::enter_runtime", file: "/root/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.29.1/src/runtime/context/runtime.rs", line: 65 }, { fn: "tokio::runtime::sche
duler::multi_thread::worker::run", file: "/root/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.29.1/src/runtime/scheduler/multi_thread/worker.rs", line: 478 }, { fn: "to
kio::runtime::scheduler::multi_thread::worker::Launch::launch::{{closure}}", file: "/root/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.29.1/src/runtime/scheduler/multi
_thread/worker.rs", line: 447 }, { fn: "<tokio::runtime::blocking::task::BlockingTask<T> as core::future::future::Future>::poll", file: "/root/.cargo/registry/src/github.com-1ecc
6299db9ec823/tokio-1.29.1/src/runtime/blocking/task.rs", line: 42 }, { fn: "tokio::runtime::task::core::Core<T,S>::poll::{{closure}}", file: "/root/.cargo/registry/src/github.com
-1ecc6299db9ec823/tokio-1.29.1/src/runtime/task/core.rs", line: 311 }, { fn: "tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut", file: "/root/.cargo/registry/src/github.com
-1ecc6299db9ec823/tokio-1.29.1/src/loom/std/unsafe_cell.rs", line: 14 }, { fn: "tokio::runtime::task::core::Core<T,S>::poll", file: "/root/.cargo/registry/src/github.com-1ecc6299
db9ec823/tokio-1.29.1/src/runtime/task/core.rs", line: 300 }, { fn: "tokio::runtime::task::harness::poll_future::{{closure}}", file: "/root/.cargo/registry/src/github.com-1ecc629
9db9ec823/tokio-1.29.1/src/runtime/task/harness.rs", line: 476 }, { fn: "<core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once", file: "/b
uild/rustc-v6rcRM/rustc-1.66.1+dfsg0ubuntu1~llvm/library/core/src/panic/unwind_safe.rs", line: 271 }, { fn: "std::panicking::try::do_call", file: "/build/rustc-v6rcRM/rustc-1.66.
1+dfsg0ubuntu1~llvm/library/std/src/panicking.rs", line: 483 }, { fn: "__rust_try" }, { fn: "std::panicking::try", file: "/build/rustc-v6rcRM/rustc-1.66.1+dfsg0ubuntu1~llvm/libra
ry/std/src/panicking.rs", line: 447 }, { fn: "std::panic::catch_unwind", file: "/build/rustc-v6rcRM/rustc-1.66.1+dfsg0ubuntu1~llvm/library/std/src/panic.rs", line: 137 }, { fn: "
tokio::runtime::task::harness::poll_future", file: "/root/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.29.1/src/runtime/task/harness.rs", line: 464 }, { fn: "tokio::ru
ntime::task::harness::Harness<T,S>::poll_inner", file: "/root/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.29.1/src/runtime/task/harness.rs", line: 198 }, { fn: "tokio
::runtime::task::harness::Harness<T,S>::poll", file: "/root/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.29.1/src/runtime/task/harness.rs", line: 152 }, { fn: "tokio::
runtime::task::raw::poll", file: "/root/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.29.1/src/runtime/task/raw.rs", line: 276 }, { fn: "tokio::runtime::task::raw::Raw$
ask::poll", file: "/root/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.29.1/src/runtime/task/raw.rs", line: 200 }, { fn: "tokio::runtime::task::UnownedTask<S>::run", fi
le: "/root/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.29.1/src/runtime/task/mod.rs", line: 437 }, { fn: "tokio::runtime::blocking::pool::Task::run", file: "/root/.ca
rgo/registry/src/github.com-1ecc6299db9ec823/tokio-1.29.1/src/runtime/blocking/pool.rs", line: 159 }, { fn: "tokio::runtime::blocking::pool::Inner::run", file: "/root/.cargo/regi
stry/src/github.com-1ecc6299db9ec823/tokio-1.29.1/src/runtime/blocking/pool.rs", line: 513 }, { fn: "tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}}", file: "/
root/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.29.1/src/runtime/blocking/pool.rs", line: 471 }, { fn: "std::sys_common::backtrace::__rust_begin_short_backtrace", fi
le: "/build/rustc-v6rcRM/rustc-1.66.1+dfsg0ubuntu1~llvm/library/std/src/sys_common/backtrace.rs", line: 121 }, { fn: "std::thread::Builder::spawn_unchecked_::{{closure}}::{{closu
re}}", file: "/build/rustc-v6rcRM/rustc-1.66.1+dfsg0ubuntu1~llvm/library/std/src/thread/mod.rs", line: 551 }, { fn: "<core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::
function::FnOnce<()>>::call_once", file: "/build/rustc-v6rcRM/rustc-1.66.1+dfsg0ubuntu1~llvm/library/core/src/panic/unwind_safe.rs", line: 271 }, { fn: "std::panicking::try::do_c
all", file: "/build/rustc-v6rcRM/rustc-1.66.1+dfsg0ubuntu1~llvm/library/std/src/panicking.rs", line: 483 }, { fn: "__rust_try" }, { fn: "std::panicking::try", file: "/build/rustc
-v6rcRM/rustc-1.66.1+dfsg0ubuntu1~llvm/library/std/src/panicking.rs", line: 447 }, { fn: "std::panic::catch_unwind", file: "/build/rustc-v6rcRM/rustc-1.66.1+dfsg0ubuntu1~llvm/lib
rary/std/src/panic.rs", line: 137 }, { fn: "std::thread::Builder::spawn_unchecked_::{{closure}}", file: "/build/rustc-v6rcRM/rustc-1.66.1+dfsg0ubuntu1~llvm/library/std/src/thread
/mod.rs", line: 550 }, { fn: "core::ops::function::FnOnce::call_once{{vtable.shim}}", file: "/build/rustc-v6rcRM/rustc-1.66.1+dfsg0ubuntu1~llvm/library/core/src/ops/function.rs",
line: 251 }, { fn: "<alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once", file: "/build/rustc-v6rcRM/rustc-1.66.1+dfsg0ubuntu1~llvm/library/alloc/src/boxed.
rs", line: 1987 }, { fn: "<alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once", file: "/build/rustc-v6rcRM/rustc-1.66.1+dfsg0ubuntu1~llvm/library/alloc/src/b
oxed.rs", line: 1987 }, { fn: "std::sys::unix::thread::Thread::new::thread_start", file: "/build/rustc-v6rcRM/rustc-1.66.1+dfsg0ubuntu1~llvm/library/std/src/sys/unix/thread.rs",
line: 108 }, { fn: "start_thread", file: "/build/glibc-BHL3KM/glibc-2.31/nptl/pthread_create.c", line: 477 }, { fn: "clone", file: "/build/glibc-BHL3KM/glibc-2.31/misc/../sysdeps
/unix/sysv/linux/x86_64/clone.S", line: 95 }]
thread 'tokio-runtime-worker' panicked at 'called `Option::unwrap()` on a `None` value', router/src/queue.rs:243:23