Hi, I try to evaluate the model every 5 training times, but I encounter this problem. Could you help me ?
The code I changes:
# Start training
for epoch in range(start_epoch, args.epochs):
adjust_lr(epoch)
trainer.train(epoch, source_train_loader, source_triplet_train_loader, target_train_loader, optimizer)
save_checkpoint({
'state_dict': model.module.state_dict(),
'epoch': epoch + 1,
}, fpath=osp.join(args.logs_dir, 'checkpoint.pth.tar'))
print('\n * Finished epoch {:3d} \n'.
format(epoch))
if epoch % 5 == 0:
# Final test
print('Test with best model:')
evaluator = Evaluator(model)
evaluator.evaluate(query_loader, gallery_loader, dataset.query,
dataset.gallery, args.output_feature, args.rerank)
# Final test
print('Test with best model:')
evaluator = Evaluator(model)
evaluator.evaluate(query_loader, gallery_loader, dataset.query, dataset.gallery, args.output_feature, args.rerank)
Epoch: [0][20/129] Time 0.759 (1.177) Data 0.000 (0.055) Loss 5.916 (6.386) Prec_c 10.94% (5.20%) Prec_t 4.55% (1.93%)
Epoch: [0][40/129] Time 0.761 (0.969) Data 0.000 (0.031) Loss 5.152 (5.923) Prec_c 17.97% (10.86%) Prec_t 6.25% (4.30%)
Epoch: [0][60/129] Time 0.751 (0.898) Data 0.000 (0.022) Loss 3.891 (5.385) Prec_c 27.34% (16.58%) Prec_t 12.50% (7.51%)
Epoch: [0][80/129] Time 0.757 (0.864) Data 0.000 (0.018) Loss 3.114 (4.856) Prec_c 44.53% (22.59%) Prec_t 23.86% (11.14%)
Epoch: [0][100/129] Time 0.764 (0.856) Data 0.000 (0.016) Loss 2.398 (4.423) Prec_c 54.69% (27.72%) Prec_t 26.14% (13.48%)
Epoch: [0][120/129] Time 0.764 (0.841) Data 0.000 (0.014) Loss 2.374 (4.068) Prec_c 51.56% (32.34%) Prec_t 41.48% (15.98%)
* Finished epoch 0
Test with best model:
Exception ignored in: <function _DataLoaderIter.__del__ at 0x7f32d70c7f28>
Traceback (most recent call last):
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 717, in __del__
self._shutdown_workers()
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 713, in _shutdown_workers
w.join()
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/multiprocessing/process.py", line 138, in join
assert self._parent_pid == os.getpid(), 'can only join a child process'
AssertionError: can only join a child process
Exception ignored in: <function _DataLoaderIter.__del__ at 0x7f32d70c7f28>
Traceback (most recent call last):
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 717, in __del__
self._shutdown_workers()
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 713, in _shutdown_workers
Exception ignored in: <function _DataLoaderIter.__del__ at 0x7f32d70c7f28>
Traceback (most recent call last):
Exception ignored in: File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 717, in __del__
<function _DataLoaderIter.__del__ at 0x7f32d70c7f28>
Traceback (most recent call last):
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 717, in __del__
w.join()
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/multiprocessing/process.py", line 138, in join
assert self._parent_pid == os.getpid(), 'can only join a child process'
AssertionError: can only join a child process
self._shutdown_workers()
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 713, in _shutdown_workers
self._shutdown_workers()
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 713, in _shutdown_workers
w.join()
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/multiprocessing/process.py", line 138, in join
w.join()
assert self._parent_pid == os.getpid(), 'can only join a child process'
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/multiprocessing/process.py", line 138, in join
AssertionError: can only join a child process
assert self._parent_pid == os.getpid(), 'can only join a child process'
AssertionError: can only join a child process
Exception ignored in: <function _DataLoaderIter.__del__ at 0x7f32d70c7f28>
Traceback (most recent call last):
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 717, in __del__
self._shutdown_workers()
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 713, in _shutdown_workers
w.join()
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/multiprocessing/process.py", line 138, in join
assert self._parent_pid == os.getpid(), 'can only join a child process'
AssertionError: can only join a child process
Exception ignored in: <function _DataLoaderIter.__del__ at 0x7f32d70c7f28>
Traceback (most recent call last):
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 717, in __del__
self._shutdown_workers()
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 713, in _shutdown_workers
w.join()
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/multiprocessing/process.py", line 138, in join
assert self._parent_pid == os.getpid(), 'can only join a child process'
AssertionError: can only join a child process
Exception ignored in: <function _DataLoaderIter.__del__ at 0x7f32d70c7f28>
Traceback (most recent call last):
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 717, in __del__
self._shutdown_workers()Exception ignored in:
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 713, in _shutdown_workers
<function _DataLoaderIter.__del__ at 0x7f32d70c7f28>
Traceback (most recent call last):
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 717, in __del__
self._shutdown_workers()
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 713, in _shutdown_workers
w.join()
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/multiprocessing/process.py", line 138, in join
w.join()
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/multiprocessing/process.py", line 138, in join
assert self._parent_pid == os.getpid(), 'can only join a child process'
AssertionError: can only join a child process
assert self._parent_pid == os.getpid(), 'can only join a child process'
AssertionError: can only join a child process
Exception ignored in: <function _DataLoaderIter.__del__ at 0x7f32d70c7f28>
Traceback (most recent call last):
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 717, in __del__
self._shutdown_workers()
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 713, in _shutdown_workers
w.join()
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/multiprocessing/process.py", line 138, in join
assert self._parent_pid == os.getpid(), 'can only join a child process'
AssertionError: can only join a child process
Exception ignored in: Exception ignored in: <function _DataLoaderIter.__del__ at 0x7f32d70c7f28>
Traceback (most recent call last):
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 717, in __del__
Exception ignored in: <function _DataLoaderIter.__del__ at 0x7f32d70c7f28>
Traceback (most recent call last):
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 717, in __del__
<function _DataLoaderIter.__del__ at 0x7f32d70c7f28>
Traceback (most recent call last):
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 717, in __del__
self._shutdown_workers()
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 713, in _shutdown_workers
self._shutdown_workers()
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 713, in _shutdown_workers
Exception ignored in: w.join()
<function _DataLoaderIter.__del__ at 0x7f32d70c7f28> File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/multiprocessing/process.py", line 138, in join
Traceback (most recent call last):
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 717, in __del__
assert self._parent_pid == os.getpid(), 'can only join a child process'
AssertionError: can only join a child process
w.join()
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/multiprocessing/process.py", line 138, in join
self._shutdown_workers()
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 713, in _shutdown_workers
assert self._parent_pid == os.getpid(), 'can only join a child process'
AssertionError: can only join a child processself._shutdown_workers()
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 713, in _shutdown_workers
w.join()
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/multiprocessing/process.py", line 138, in join
assert self._parent_pid == os.getpid(), 'can only join a child process'
AssertionError: can only join a child process
w.join()
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/multiprocessing/process.py", line 138, in join
assert self._parent_pid == os.getpid(), 'can only join a child process'
AssertionError: can only join a child process
Exception ignored in: <function _DataLoaderIter.__del__ at 0x7f32d70c7f28>
Traceback (most recent call last):
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 717, in __del__
self._shutdown_workers()
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 713, in _shutdown_workers
w.join()
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/multiprocessing/process.py", line 138, in join
assert self._parent_pid == os.getpid(), 'can only join a child process'
AssertionError: can only join a child process
Exception ignored in: <function _DataLoaderIter.__del__ at 0x7f32d70c7f28>
Traceback (most recent call last):
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 717, in __del__
self._shutdown_workers()
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 713, in _shutdown_workers
w.join()
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/multiprocessing/process.py", line 138, in join
assert self._parent_pid == os.getpid(), 'can only join a child process'
AssertionError: can only join a child process
Exception ignored in: <function _DataLoaderIter.__del__ at 0x7f32d70c7f28>
Traceback (most recent call last):
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 717, in __del__
self._shutdown_workers()
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 713, in _shutdown_workers
w.join()
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/multiprocessing/process.py", line 138, in join
assert self._parent_pid == os.getpid(), 'can only join a child process'
AssertionError: can only join a child process
Traceback (most recent call last):
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/multiprocessing/queues.py", line 242, in _feed
send_bytes(obj)
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/multiprocessing/connection.py", line 200, in send_bytes
self._send_bytes(m[offset:offset + size])
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/multiprocessing/connection.py", line 404, in _send_bytes
self._send(header + buf)
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/multiprocessing/connection.py", line 368, in _send
n = write(self._handle, buf)
BrokenPipeError: [Errno 32] Broken pipe
Traceback (most recent call last):
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/multiprocessing/queues.py", line 232, in _feed
close()
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/multiprocessing/connection.py", line 177, in close
self._close()
Traceback (most recent call last):
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/multiprocessing/connection.py", line 361, in _close
_close(self._handle)
OSError: [Errno 9] Bad file descriptor
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/multiprocessing/queues.py", line 242, in _feed
send_bytes(obj)
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/multiprocessing/connection.py", line 200, in send_bytes
self._send_bytes(m[offset:offset + size])
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/multiprocessing/connection.py", line 404, in _send_bytes
self._send(header + buf)
File "/home/lsc/.conda/envs/lsc_pytorch/lib/python3.7/multiprocessing/connection.py", line 368, in _send
n = write(self._handle, buf)
BrokenPipeError: [Errno 32] Broken pipe
Mean AP: 15.7%
CMC Scores
top-1 38.5%
top-5 56.5%
top-10 65.3%
top-20 73.5%
Epoch: [1][20/129] Time 0.737 (0.793) Data 0.000 (0.053) Loss 1.328 (1.349) Prec_c 72.66% (72.70%) Prec_t 25.00% (30.97%)
Epoch: [1][40/129] Time 0.735 (0.766) Data 0.000 (0.030) Loss 1.178 (1.307) Prec_c 79.69% (73.95%) Prec_t 33.52% (29.82%)
Epoch: [1][60/129] Time 0.737 (0.759) Data 0.000 (0.022) Loss 1.165 (1.260) Prec_c 77.34% (75.46%) Prec_t 32.39% (30.54%)
Epoch: [1][80/129] Time 0.750 (0.756) Data 0.000 (0.018) Loss 1.206 (1.229) Prec_c 77.34% (76.09%) Prec_t 30.11% (31.13%)
Epoch: [1][100/129] Time 0.775 (0.771) Data 0.000 (0.016) Loss 0.919 (1.191) Prec_c 81.25% (76.95%) Prec_t 36.93% (31.81%)
Epoch: [1][120/129] Time 0.772 (0.770) Data 0.000 (0.014) Loss 0.950 (1.157) Prec_c 81.25% (77.62%) Prec_t 39.77% (33.23%)
* Finished epoch 1
I am really confused.
The version of my pytorch is : 1.0.0