I receive an overflow error when I try to run training or prediction in VisCy. I tried creating a new environment to see if that fixes it, but that did not work.
> [rank1]: Traceback (most recent call last):
> [rank1]: File "/hpc/mydata/soorya.pradeep/scratch/VisCy/viscy/scripts/DragonFly_vsModel/pretrain.py", line 70, in <module>
> [rank1]: trainer.fit(model(), data(batch_size=32, caching=True, num_workers=16))
> [rank1]: File "/hpc/mydata/soorya.pradeep/viscy/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 543, in fit
> [rank1]: call._call_and_handle_interrupt(
> [rank1]: File "/hpc/mydata/soorya.pradeep/viscy/lib/python3.10/site-packages/lightning/pytorch/trainer/call.py", line 43, in _call_and_handle_interrupt
> [rank1]: return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs)
> [rank1]: File "/hpc/mydata/soorya.pradeep/viscy/lib/python3.10/site-packages/lightning/pytorch/strategies/launchers/subprocess_script.py", line 105, in launch
> [rank1]: return function(*args, **kwargs)
> [rank1]: File "/hpc/mydata/soorya.pradeep/viscy/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 579, in _fit_impl
> [rank1]: self._run(model, ckpt_path=ckpt_path)
> [rank1]: File "/hpc/mydata/soorya.pradeep/viscy/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 948, in _run
> [rank1]: call._call_setup_hook(self) # allow user to set up LightningModule in accelerator environment
> [rank1]: File "/hpc/mydata/soorya.pradeep/viscy/lib/python3.10/site-packages/lightning/pytorch/trainer/call.py", line 94, in _call_setup_hook
> [rank1]: _call_lightning_datamodule_hook(trainer, "setup", stage=fn)
> [rank1]: File "/hpc/mydata/soorya.pradeep/viscy/lib/python3.10/site-packages/lightning/pytorch/trainer/call.py", line 181, in _call_lightning_datamodule_hook
> [rank1]: return fn(*args, **kwargs)
> [rank1]: File "/hpc/mydata/soorya.pradeep/viscy_main/VisCy/viscy/data/combined.py", line 52, in setup
> [rank1]: dm.setup(stage)
> [rank1]: File "/hpc/mydata/soorya.pradeep/viscy_main/VisCy/viscy/data/hcs.py", line 383, in setup
> [rank1]: self._setup_fit(dataset_settings)
> [rank1]: File "/hpc/mydata/soorya.pradeep/viscy_main/VisCy/viscy/data/hcs.py", line 396, in _setup_fit
> [rank1]: train_transform = Compose(
> [rank1]: File "/hpc/mydata/soorya.pradeep/viscy/lib/python3.10/site-packages/monai/transforms/compose.py", line 251, in __init__
> [rank1]: self.set_random_state(seed=get_seed())
> [rank1]: File "/hpc/mydata/soorya.pradeep/viscy/lib/python3.10/site-packages/monai/transforms/compose.py", line 263, in set_random_state
> [rank1]: _transform.set_random_state(seed=self.R.randint(MAX_SEED, dtype="uint32"))
> [rank1]: File "/hpc/mydata/soorya.pradeep/viscy/lib/python3.10/site-packages/monai/transforms/transform.py", line 207, in set_random_state
> [rank1]: _seed = _seed % MAX_SEED
> [rank1]: OverflowError: Python integer 4294967296 out of bounds for uint32
> [rank0]: Traceback (most recent call last):
> [rank0]: File "/hpc/mydata/soorya.pradeep/scratch/VisCy/viscy/scripts/DragonFly_vsModel/pretrain.py", line 70, in <module>
> [rank0]: trainer.fit(model(), data(batch_size=32, caching=True, num_workers=16))
> [rank0]: File "/hpc/mydata/soorya.pradeep/viscy/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 543, in fit
> [rank0]: call._call_and_handle_interrupt(
> [rank0]: File "/hpc/mydata/soorya.pradeep/viscy/lib/python3.10/site-packages/lightning/pytorch/trainer/call.py", line 43, in _call_and_handle_interrupt
> [rank0]: return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs)
> [rank0]: File "/hpc/mydata/soorya.pradeep/viscy/lib/python3.10/site-packages/lightning/pytorch/strategies/launchers/subprocess_script.py", line 105, in launch
> [rank0]: return function(*args, **kwargs)
> [rank0]: File "/hpc/mydata/soorya.pradeep/viscy/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 579, in _fit_impl
> [rank0]: self._run(model, ckpt_path=ckpt_path)
> [rank0]: File "/hpc/mydata/soorya.pradeep/viscy/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 948, in _run
> [rank0]: call._call_setup_hook(self) # allow user to set up LightningModule in accelerator environment
> [rank0]: File "/hpc/mydata/soorya.pradeep/viscy/lib/python3.10/site-packages/lightning/pytorch/trainer/call.py", line 94, in _call_setup_hook
> [rank0]: _call_lightning_datamodule_hook(trainer, "setup", stage=fn)
> [rank0]: File "/hpc/mydata/soorya.pradeep/viscy/lib/python3.10/site-packages/lightning/pytorch/trainer/call.py", line 181, in _call_lightning_datamodule_hook
> [rank0]: return fn(*args, **kwargs)
> [rank0]: File "/hpc/mydata/soorya.pradeep/viscy_main/VisCy/viscy/data/combined.py", line 52, in setup
> [rank0]: dm.setup(stage)
> [rank0]: File "/hpc/mydata/soorya.pradeep/viscy_main/VisCy/viscy/data/hcs.py", line 383, in setup
> [rank0]: self._setup_fit(dataset_settings)
> [rank0]: File "/hpc/mydata/soorya.pradeep/viscy_main/VisCy/viscy/data/hcs.py", line 396, in _setup_fit
> [rank0]: train_transform = Compose(
> [rank0]: File "/hpc/mydata/soorya.pradeep/viscy/lib/python3.10/site-packages/monai/transforms/compose.py", line 251, in __init__
> [rank0]: self.set_random_state(seed=get_seed())
> [rank0]: File "/hpc/mydata/soorya.pradeep/viscy/lib/python3.10/site-packages/monai/transforms/compose.py", line 263, in set_random_state
> [rank0]: _transform.set_random_state(seed=self.R.randint(MAX_SEED, dtype="uint32"))
> [rank0]: File "/hpc/mydata/soorya.pradeep/viscy/lib/python3.10/site-packages/monai/transforms/transform.py", line 207, in set_random_state
> [rank0]: _seed = _seed % MAX_SEED
> [rank0]: OverflowError: Python integer 4294967296 out of bounds for uint32
> [rank2]: Traceback (most recent call last):
> [rank2]: File "/hpc/mydata/soorya.pradeep/scratch/VisCy/viscy/scripts/DragonFly_vsModel/pretrain.py", line 70, in <module>
> [rank2]: trainer.fit(model(), data(batch_size=32, caching=True, num_workers=16))
> [rank2]: File "/hpc/mydata/soorya.pradeep/viscy/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 543, in fit
> [rank2]: call._call_and_handle_interrupt(
> [rank2]: File "/hpc/mydata/soorya.pradeep/viscy/lib/python3.10/site-packages/lightning/pytorch/trainer/call.py", line 43, in _call_and_handle_interrupt
> [rank2]: return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs)
> [rank2]: File "/hpc/mydata/soorya.pradeep/viscy/lib/python3.10/site-packages/lightning/pytorch/strategies/launchers/subprocess_script.py", line 105, in launch
> [rank2]: return function(*args, **kwargs)
> [rank2]: File "/hpc/mydata/soorya.pradeep/viscy/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 579, in _fit_impl
> [rank2]: self._run(model, ckpt_path=ckpt_path)
> [rank2]: File "/hpc/mydata/soorya.pradeep/viscy/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 948, in _run
> [rank2]: call._call_setup_hook(self) # allow user to set up LightningModule in accelerator environment
> [rank2]: File "/hpc/mydata/soorya.pradeep/viscy/lib/python3.10/site-packages/lightning/pytorch/trainer/call.py", line 94, in _call_setup_hook
> [rank2]: _call_lightning_datamodule_hook(trainer, "setup", stage=fn)
> [rank2]: File "/hpc/mydata/soorya.pradeep/viscy/lib/python3.10/site-packages/lightning/pytorch/trainer/call.py", line 181, in _call_lightning_datamodule_hook
> [rank2]: return fn(*args, **kwargs)
> [rank2]: File "/hpc/mydata/soorya.pradeep/viscy_main/VisCy/viscy/data/combined.py", line 52, in setup
> [rank2]: dm.setup(stage)
> [rank2]: File "/hpc/mydata/soorya.pradeep/viscy_main/VisCy/viscy/data/hcs.py", line 383, in setup
> [rank2]: self._setup_fit(dataset_settings)
> [rank2]: File "/hpc/mydata/soorya.pradeep/viscy_main/VisCy/viscy/data/hcs.py", line 396, in _setup_fit
> [rank2]: train_transform = Compose(
> [rank2]: File "/hpc/mydata/soorya.pradeep/viscy/lib/python3.10/site-packages/monai/transforms/compose.py", line 251, in __init__
> [rank2]: self.set_random_state(seed=get_seed())
> [rank2]: File "/hpc/mydata/soorya.pradeep/viscy/lib/python3.10/site-packages/monai/transforms/compose.py", line 263, in set_random_state
> [rank2]: _transform.set_random_state(seed=self.R.randint(MAX_SEED, dtype="uint32"))
> [rank2]: File "/hpc/mydata/soorya.pradeep/viscy/lib/python3.10/site-packages/monai/transforms/transform.py", line 207, in set_random_state
> [rank2]: _seed = _seed % MAX_SEED
> [rank2]: OverflowError: Python integer 4294967296 out of bounds for uint32
> [rank3]: Traceback (most recent call last):
> [rank3]: File "/hpc/mydata/soorya.pradeep/scratch/VisCy/viscy/scripts/DragonFly_vsModel/pretrain.py", line 70, in <module>
> [rank3]: trainer.fit(model(), data(batch_size=32, caching=True, num_workers=16))
> [rank3]: File "/hpc/mydata/soorya.pradeep/viscy/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 543, in fit
> [rank3]: call._call_and_handle_interrupt(
> [rank3]: File "/hpc/mydata/soorya.pradeep/viscy/lib/python3.10/site-packages/lightning/pytorch/trainer/call.py", line 43, in _call_and_handle_interrupt
> [rank3]: return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs)
> [rank3]: File "/hpc/mydata/soorya.pradeep/viscy/lib/python3.10/site-packages/lightning/pytorch/strategies/launchers/subprocess_script.py", line 105, in launch
> [rank3]: return function(*args, **kwargs)
> [rank3]: File "/hpc/mydata/soorya.pradeep/viscy/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 579, in _fit_impl
> [rank3]: self._run(model, ckpt_path=ckpt_path)
> [rank3]: File "/hpc/mydata/soorya.pradeep/viscy/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 948, in _run
> [rank3]: call._call_setup_hook(self) # allow user to set up LightningModule in accelerator environment
> [rank3]: File "/hpc/mydata/soorya.pradeep/viscy/lib/python3.10/site-packages/lightning/pytorch/trainer/call.py", line 94, in _call_setup_hook
> [rank3]: _call_lightning_datamodule_hook(trainer, "setup", stage=fn)
> [rank3]: File "/hpc/mydata/soorya.pradeep/viscy/lib/python3.10/site-packages/lightning/pytorch/trainer/call.py", line 181, in _call_lightning_datamodule_hook
> [rank3]: return fn(*args, **kwargs)
> [rank3]: File "/hpc/mydata/soorya.pradeep/viscy_main/VisCy/viscy/data/combined.py", line 52, in setup
> [rank3]: dm.setup(stage)
> [rank3]: File "/hpc/mydata/soorya.pradeep/viscy_main/VisCy/viscy/data/hcs.py", line 383, in setup
> [rank3]: self._setup_fit(dataset_settings)
> [rank3]: File "/hpc/mydata/soorya.pradeep/viscy_main/VisCy/viscy/data/hcs.py", line 396, in _setup_fit
> [rank3]: train_transform = Compose(
> [rank3]: File "/hpc/mydata/soorya.pradeep/viscy/lib/python3.10/site-packages/monai/transforms/compose.py", line 251, in __init__
> [rank3]: self.set_random_state(seed=get_seed())
> [rank3]: File "/hpc/mydata/soorya.pradeep/viscy/lib/python3.10/site-packages/monai/transforms/compose.py", line 263, in set_random_state
> [rank3]: _transform.set_random_state(seed=self.R.randint(MAX_SEED, dtype="uint32"))
> [rank3]: File "/hpc/mydata/soorya.pradeep/viscy/lib/python3.10/site-packages/monai/transforms/transform.py", line 207, in set_random_state
> [rank3]: _seed = _seed % MAX_SEED
> [rank3]: OverflowError: Python integer 4294967296 out of bounds for uint32