ScheMoE
The development of this code refers to tutel.
torch==1.9.1
git clone https://github.com/Fragile-azalea/ScheMoE.git
cd ScheMoE
pip install -e .
# Single Machine:
python3 -m torch.distributed.run --nproc_per_node=4 -m schemoe.examples.pre_test --batch_size=16
# Distribute:
# pls refers to schemoe/examples/run_mpi.sh
- g++==7.5.0
- cuda==10.2
- gpu==2080Ti