请检查是否正确加载我们修改的transformers。如果选择使用更高版本的torch和transformers，请将我们修改的部分（主要为llama_modeling.py）适配到最新版本的transformers即可。

from loramoe.

代码修改：1. 将模型换成了llama3 2. 修改了device

报错情况：Traceback (most recent call last):

File "/home/xsong/SiameseModelTrain/LoRAMoE/run_loramoe.py", line 569, in
main()
File "/home/xsong/SiameseModelTrain/LoRAMoE/run_loramoe.py", line 345, in main
tokenizer = LlamaTokenizer.from_pretrained(model_args.tokenizer_name_or_path, **tokenizer_kwargs)
File "/home/xsong/SiameseModelTrain/LoRAMoE/transformers/tokenization_utils_base.py", line 1825, in from_pretrained
return cls._from_pretrained(
File "/home/xsong/SiameseModelTrain/LoRAMoE/transformers/tokenization_utils_base.py", line 1988, in _from_pretrained
tokenizer = cls(*init_inputs, **init_kwargs)
File "/home/xsong/SiameseModelTrain/LoRAMoE/transformers/models/llama/tokenization_llama.py", line 96, in init
self.sp_model.Load(vocab_file)
File "/home/xsong/anaconda3/envs/newloramoe/lib/python3.10/site-packages/sentencepiece/init.py", line 961, in Load
return self.LoadFromFile(model_file)
File "/home/xsong/anaconda3/envs/newloramoe/lib/python3.10/site-packages/sentencepiece/init.py", line 316, in LoadFromFile
return _sentencepiece.SentencePieceProcessor_LoadFromFile(self, arg)
TypeError: not a string
ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 1415452) of binary: /home/xsong/anaconda3/envs/newloramoe/bin/python
Traceback (most recent call last):
File "/home/xsong/anaconda3/envs/newloramoe/bin/torchrun", line 33, in
sys.exit(load_entry_point('torch==2.0.1', 'console_scripts', 'torchrun')())
File "/home/xsong/anaconda3/envs/newloramoe/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/init.py", line 346, in wrapper
return f(*args, **kwargs)
File "/home/xsong/anaconda3/envs/newloramoe/lib/python3.10/site-packages/torch/distributed/run.py", line 794, in main
run(args)
File "/home/xsong/anaconda3/envs/newloramoe/lib/python3.10/site-packages/torch/distributed/run.py", line 785, in run
elastic_launch(
File "/home/xsong/anaconda3/envs/newloramoe/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 134, in call
return launch_agent(self._config, self._entrypoint, list(args))
File "/home/xsong/anaconda3/envs/newloramoe/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 250, in launch_agent
raise ChildFailedError(
torch.distributed.elastic.multiprocessing.errors.ChildFailedError:

run_loramoe.py FAILED

报错情况：Traceback (most recent call last):
File "/home/xsong/SiameseModelTrain/LoRAMoE/run_loramoe.py", line 569, in
main()
File "/home/xsong/SiameseModelTrain/LoRAMoE/run_loramoe.py", line 345, in main
tokenizer = LlamaTokenizer.from_pretrained(model_args.tokenizer_name_or_path, **tokenizer_kwargs)
File "/home/xsong/SiameseModelTrain/LoRAMoE/transformers/tokenization_utils_base.py", line 1825, in from_pretrained
return cls._from_pretrained(
File "/home/xsong/SiameseModelTrain/LoRAMoE/transformers/tokenization_utils_base.py", line 1988, in _from_pretrained
tokenizer = cls(*init_inputs, **init_kwargs)
File "/home/xsong/SiameseModelTrain/LoRAMoE/transformers/models/llama/tokenization_llama.py", line 96, in init
self.sp_model.Load(vocab_file)
File "/home/xsong/anaconda3/envs/newloramoe/lib/python3.10/site-packages/sentencepiece/init.py", line 961, in Load
return self.LoadFromFile(model_file)
File "/home/xsong/anaconda3/envs/newloramoe/lib/python3.10/site-packages/sentencepiece/init.py", line 316, in LoadFromFile
return _sentencepiece.SentencePieceProcessor_LoadFromFile(self, arg)
TypeError: not a string
ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 1415452) of binary: /home/xsong/anaconda3/envs/newloramoe/bin/python
Traceback (most recent call last):
File "/home/xsong/anaconda3/envs/newloramoe/bin/torchrun", line 33, in
sys.exit(load_entry_point('torch==2.0.1', 'console_scripts', 'torchrun')())
File "/home/xsong/anaconda3/envs/newloramoe/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/init.py", line 346, in wrapper
return f(*args, **kwargs)
File "/home/xsong/anaconda3/envs/newloramoe/lib/python3.10/site-packages/torch/distributed/run.py", line 794, in main
run(args)
File "/home/xsong/anaconda3/envs/newloramoe/lib/python3.10/site-packages/torch/distributed/run.py", line 785, in run
elastic_launch(
File "/home/xsong/anaconda3/envs/newloramoe/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 134, in call
return launch_agent(self._config, self._entrypoint, list(args))
File "/home/xsong/anaconda3/envs/newloramoe/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 250, in launch_agent
raise ChildFailedError(
torch.distributed.elastic.multiprocessing.errors.ChildFailedError:
run_loramoe.py FAILED

问题解决了嘛

from loramoe.