hristo-vrigazov / dnn.cool Goto Github PK

  File "/home/ml/anaconda3/envs/jobs/lib/python3.7/site-packages/catalyst/core/runner.py", line 929, in run_experiment
    self._run_event("on_exception")
  File "/home/ml/anaconda3/envs/jobs/lib/python3.7/site-packages/catalyst/core/runner.py", line 724, in _run_event
    getattr(callback, event)(self)
  File "/home/ml/anaconda3/envs/jobs/lib/python3.7/site-packages/catalyst/callbacks/exception.py", line 34, in on_exception
    raise exception
  File "/home/ml/anaconda3/envs/jobs/lib/python3.7/site-packages/catalyst/core/runner.py", line 917, in run_experiment
    self._run_stage(stage)
  File "/home/ml/anaconda3/envs/jobs/lib/python3.7/site-packages/catalyst/core/runner.py", line 885, in _run_stage
    self._run_epoch(stage=stage, epoch=self.epoch)
  File "/home/ml/anaconda3/envs/jobs/lib/python3.7/site-packages/catalyst/core/runner.py", line 864, in _run_epoch
    self._run_loader(loader)
  File "/home/ml/anaconda3/envs/jobs/lib/python3.7/site-packages/catalyst/core/runner.py", line 801, in _run_loader
    self._run_batch(batch)
  File "/home/ml/anaconda3/envs/jobs/lib/python3.7/site-packages/dnn_cool/runner.py", line 337, in _run_batch
    self._handle_batch(batch=batch)
  File "/home/ml/anaconda3/envs/jobs/lib/python3.7/site-packages/catalyst/runners/supervised.py", line 144, in _handle_batch
    self.output = self.forward(batch)
  File "/home/ml/anaconda3/envs/jobs/lib/python3.7/site-packages/catalyst/runners/supervised.py", line 131, in forward
    output = self._process_input(batch, **kwargs)
  File "/home/ml/anaconda3/envs/jobs/lib/python3.7/site-packages/catalyst/runners/supervised.py", line 94, in _process_input_str
    output = self.model(batch[self.input_key], **kwargs)
  File "/home/ml/anaconda3/envs/jobs/lib/python3.7/site-packages/torch/nn/modules/module.py", line 727, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/home/ml/anaconda3/envs/jobs/lib/python3.7/site-packages/torch/nn/parallel/data_parallel.py", line 162, in forward
    return self.gather(outputs, self.output_device)
  File "/home/ml/anaconda3/envs/jobs/lib/python3.7/site-packages/dnn_cool/catalyst_utils.py", line 351, in gather
    r_per_sample_losses=self.r_per_sample_losses)
  File "/home/ml/anaconda3/envs/jobs/lib/python3.7/site-packages/dnn_cool/catalyst_utils.py", line 419, in reduce_on_device
    loss = criterion(outputs, targets)
  File "/home/ml/anaconda3/envs/jobs/lib/python3.7/site-packages/torch/nn/modules/module.py", line 727, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/home/ml/anaconda3/envs/jobs/lib/python3.7/site-packages/dnn_cool/losses.py", line 249, in forward
    flow_result = self.flow(self, LossFlowData(outputs, targets), LossItems(loss_items))
  File "/datastore/fast/nikolay/tmp/pycharm_project_834/multitask/project.py", line 103, in full_flow
    out += flow.title(x.titles, x.pooled)
  File "/home/ml/anaconda3/envs/jobs/lib/python3.7/site-packages/torch/nn/modules/module.py", line 727, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/home/ml/anaconda3/envs/jobs/lib/python3.7/site-packages/dnn_cool/losses.py", line 160, in forward
    return self.postprocess_results(self.compute_with_precondition(loss_flow_data))
  File "/home/ml/anaconda3/envs/jobs/lib/python3.7/site-packages/dnn_cool/losses.py", line 164, in compute_with_precondition
    result_from_device_reducing = self._device_reducing_cache(key, loss_flow_data.outputs)
  File "/home/ml/anaconda3/envs/jobs/lib/python3.7/site-packages/dnn_cool/losses.py", line 97, in __call__
    result_from_device_reducing = self._compute_device_reduced(key, outputs)
  File "/home/ml/anaconda3/envs/jobs/lib/python3.7/site-packages/dnn_cool/losses.py", line 105, in _compute_device_reduced
    already_reduced_per_device = _already_reduced_per_device(outputs)
  File "/home/ml/anaconda3/envs/jobs/lib/python3.7/site-packages/dnn_cool/losses.py", line 81, in _already_reduced_per_device
    keys = list(outputs.keys())
TypeError: 'LossFlowData' object is not callable
python-BaseException
1/30 * Epoch (train):   0% 0/64 [18:30<?, ?it/s]

Process finished with exit code 1