Finetuned by my own llama2
Traceback (most recent call last):
File "/workspace/Chat-UniVi/ChatUniVi/train/train_mem.py", line 13, in
train()
File "/workspace/Chat-UniVi/ChatUniVi/train/train.py", line 1065, in train
trainer.train()
File "/root/miniconda3/envs/chatunivi/lib/python3.10/site-packages/transformers/trainer.py", line 1539, in train
return inner_training_loop(
File "/root/miniconda3/envs/chatunivi/lib/python3.10/site-packages/transformers/trainer.py", line 1809, in _inner_training_loop
tr_loss_step = self.training_step(model, inputs)
File "/root/miniconda3/envs/chatunivi/lib/python3.10/site-packages/transformers/trainer.py", line 2665, in training_step
self.accelerator.backward(loss)
File "/root/miniconda3/envs/chatunivi/lib/python3.10/site-packages/accelerate/accelerator.py", line 1847, in backward
self.deepspeed_engine_wrapped.backward(loss, **kwargs)
File "/root/miniconda3/envs/chatunivi/lib/python3.10/site-packages/accelerate/utils/deepspeed.py", line 167, in backward
self.engine.backward(loss, **kwargs)
File "/root/miniconda3/envs/chatunivi/lib/python3.10/site-packages/deepspeed/utils/nvtx.py", line 15, in wrapped_fn
ret_val = func(*args, **kwargs)
File "/root/miniconda3/envs/chatunivi/lib/python3.10/site-packages/deepspeed/runtime/engine.py", line 1861, in backward
self.optimizer.backward(loss, retain_graph=retain_graph)
File "/root/miniconda3/envs/chatunivi/lib/python3.10/site-packages/deepspeed/utils/nvtx.py", line 15, in wrapped_fn
ret_val = func(*args, **kwargs)
File "/root/miniconda3/envs/chatunivi/lib/python3.10/site-packages/deepspeed/runtime/zero/stage3.py", line 1995, in backward
self._get_param_coordinator(training=True).reset_step()
File "/root/miniconda3/envs/chatunivi/lib/python3.10/site-packages/deepspeed/runtime/zero/partitioned_param_coordinator.py", line 191, in reset_step
assert_ints_same_as_other_ranks([m.id for m in self.__submodule_order])
File "/root/miniconda3/envs/chatunivi/lib/python3.10/site-packages/deepspeed/utils/nvtx.py", line 15, in wrapped_fn
ret_val = func(*args, **kwargs)
File "/root/miniconda3/envs/chatunivi/lib/python3.10/site-packages/deepspeed/runtime/zero/utils.py", line 86, in assert_ints_same_as_other_ranks
raise RuntimeError(f"disagreement between rank0 and rank{dist.get_rank()}: "
**RuntimeError: disagreement between rank0 and rank7: rank0: [**0, 565, 566, 567, 568, 569, 570, 571, 572, 574, 580, 575, 578, 576, 577, 579, 585, 581, 583, 582, 584, 586, 592, 587, 590, 588, 589, 591, 597, 593, 595, 594, 596, 598, 604, 599, 602, 600, 601, 603, 609, 605, 607, 606, 608, 610, 616, 611, 614, 612, 613, 615, 621, 617, 619, 618, 620, 622, 628, 623, 626, 624, 625, 627, 633, 629, 631, 630, 632, 634, 640, 635, 638, 636, 637, 639, 645, 641, 643, 642, 644, 646, 652, 647, 650, 648, 649, 651, 657, 653, 655, 654, 656, 658, 664, 659, 662, 660, 661, 663, 669, 665, 667, 666, 668, 670, 676, 671, 674, 672, 673, 675, 681, 677, 679, 678, 680, 682, 688, 683, 686, 684, 685, 687, 693, 689, 691, 690, 692, 694, 700, 695, 698, 696, 697, 699, 705, 701, 703, 702, 704, 706, 712, 707, 710, 708, 709, 711, 717, 713, 715, 714, 716, 718, 724, 719, 722, 720, 721, 723, 729, 725, 727, 726, 728, 730, 736, 731, 734, 732, 733, 735, 741, 737, 739, 738, 740, 742, 748, 743, 746, 744, 745, 747, 753, 749, 751, 750, 752, 754, 760, 755, 758, 756, 757, 759, 765, 761, 763, 762, 764, 766, 772, 767, 770, 768, 769, 771, 777, 773, 775, 774, 776, 778, 784, 779, 782, 780, 781, 783, 789, 785, 787, 786, 788, 790, 796, 791, 794, 792, 793, 795, 801, 797, 799, 798, 800, 802, 808, 803, 806, 804, 805, 807, 813, 809, 811, 810, 812, 814, 820, 815, 818, 816, 817, 819, 825, 821, 823, 822, 824, 826, 832, 827, 830, 828, 829, 831, 837, 833, 835, 834, 836, 838, 844, 839, 842, 840, 841, 843, 849, 845, 847, 846, 848, 850, 856, 851, 854, 852, 853, 855, 861, 857, 859, 858, 860, 862, 870, 864, 865, 866, 867, 868, 869, 864, 865, 866, 867, 868, 869, 864, 865, 866, 867, 868, 869, 864, 865, 866, 867, 868, 869, 864, 865, 866, 867, 868, 869, 863, 2, 2, 864, 865, 866, 867, 868, 869, 863, 2, 2, 864, 865, 866, 867, 868, 869, 863, 2, 2, 864, 865, 866, 867, 868, 869, 863, 2, 2, 870, 864, 865, 866, 867, 868, 869, 864, 865, 866, 867, 868, 869, 864, 865, 866, 867, 868, 869, 864, 865, 866, 867, 868, 869, 864, 865, 866, 867, 868, 869, 863, 2, 2, 864, 865, 866, 867, 868, 869, 863, 2, 2, 864, 865, 866, 867, 868, 869, 863, 2, 2, 864, 865, 866, 867, 868, 869, 863, 2, 2, 1, 4, 16, 5, 6, 7, 8, 10, 9, 17, 11, 15, 18, 30, 19, 20, 21, 22, 24, 23, 31, 25, 29, 32, 44, 33, 34, 35, 36, 38, 37, 45, 39, 43, 46, 58, 47, 48, 49, 50, 52, 51, 59, 53, 57, 60, 72, 61, 62, 63, 64, 66, 65, 73, 67, 71, 74, 86, 75, 76, 77, 78, 80, 79, 87, 81, 85, 88, 100, 89, 90, 91, 92, 94, 93, 101, 95, 99, 102, 114, 103, 104, 105, 106, 108, 107, 115, 109, 113, 116, 128, 117, 118, 119, 120, 122, 121, 129, 123, 127, 130, 142, 131, 132, 133, 134, 136, 135, 143, 137, 141, 144, 156, 145, 146, 147, 148, 150, 149, 15
![image](https://private-user-images.githubusercontent.com/128944251/287494662-0f401ab1-9b1c-4713-8d9b-0b130491b972.png?jwt=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJnaXRodWIuY29tIiwiYXVkIjoicmF3LmdpdGh1YnVzZXJjb250ZW50LmNvbSIsImtleSI6ImtleTUiLCJleHAiOjE3MjMyNzg2NTAsIm5iZiI6MTcyMzI3ODM1MCwicGF0aCI6Ii8xMjg5NDQyNTEvMjg3NDk0NjYyLTBmNDAxYWIxLTliMWMtNDcxMy04ZDliLTBiMTMwNDkxYjk3Mi5wbmc_WC1BbXotQWxnb3JpdGhtPUFXUzQtSE1BQy1TSEEyNTYmWC1BbXotQ3JlZGVudGlhbD1BS0lBVkNPRFlMU0E1M1BRSzRaQSUyRjIwMjQwODEwJTJGdXMtZWFzdC0xJTJGczMlMkZhd3M0X3JlcXVlc3QmWC1BbXotRGF0ZT0yMDI0MDgxMFQwODI1NTBaJlgtQW16LUV4cGlyZXM9MzAwJlgtQW16LVNpZ25hdHVyZT0xNmJkMjFkZmY2Y2JjZmEwNjAzODJmOGQ0MjZmNzU2MzNlYzVlNDI1NGJhZmU1MjczMGQzNzFkMmE1M2MwN2U0JlgtQW16LVNpZ25lZEhlYWRlcnM9aG9zdCZhY3Rvcl9pZD0wJmtleV9pZD0wJnJlcG9faWQ9MCJ9.JH0Lb6cK_0AOZvryLl4-F3YKvoKpa2f_EwHwl6WsI80)