# 请在此处粘贴依赖情况

# 请在此处粘贴运行日志

# 用了6张a10（24G）进行预训练，block开到512，没有加lm_head，embedding层，开启了zero2 offload依旧报错

# torch.cuda.OutOfMemoryError: CUDA out of memory. Tried to allocate 130.00 MiB (GPU 0; 22.20 GiB total capacity; 20.60 GiB already allocated; 126.12 MiB free; 20.98 GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF
  0%|                                                                                                                                                                                  | 0/3 [00:00<?, ?it/s

# python llama/Chinese-LLaMA-Alpaca-2/scripts/merge_llama2_with_chinese_lora_low_mem.py --base_model llama/llama-2-7b/ --lora_model chinese-llama-2-lora-7b --output_type huggingface --output_dir llama-2-7b-combined
================================================================================
Base model: llama/llama-2-7b/
LoRA model: chinese-llama-2-lora-7b
Loading chinese-llama-2-lora-7b
You are using the legacy behaviour of the <class 'transformers.models.llama.tokenization_llama.LlamaTokenizer'>. This means that tokens that come after special tokens will not be properly handled. We recommend you to read the related pull request available at https://github.com/huggingface/transformers/pull/24565
Saving tokenizer
Done.
Check output dir: llama-2-7b-combined


# ls -al llama-2-7b-combined/     ### 沒有看到最終模型檔案
total 844
drwxr-xr-x  2 root root   4096 Aug  4 15:13 .
drwxrwxr-x 11 1000 1000   4096 Aug  4 15:06 ..
-rw-r--r--  1 root root    435 Aug  4 15:13 special_tokens_map.json
-rw-r--r--  1 root root 844403 Aug  4 15:13 tokenizer.model
-rw-r--r--  1 root root    766 Aug  4 15:13 tokenizer_config.json

# ls -al  llama/llama-2-7b/                  ### 基模型在這裡
total 13161080
drwxr-xr-x 2 root root        4096 Aug  4 14:15 .
drwxr-xr-x 6 root root        4096 Aug  4 14:23 ..
-rw-r--r-- 1 root root         100 Jul 14 07:00 checklist.chk
-rw-r--r-- 1 root root 13476925163 Jul 14 07:00 consolidated.00.pth
-rw-r--r-- 1 root root         102 Jul 14 07:00 params.json

# ls -al chinese-llama-2-lora-7b    ### LoRA模型在這裡
total 1197992
drwxr-xr-x  3 root root       4096 Aug  4 14:53 .
drwxrwxr-x 11 1000 1000       4096 Aug  4 15:06 ..
drwxr-xr-x  8 root root       4096 Aug  4 14:38 .git
-rw-r--r--  1 root root       1519 Aug  4 14:38 .gitattributes
-rw-r--r--  1 root root       1945 Aug  4 14:38 README.md
-rw-rw-r--  1 root root        471 Jul 27 12:52 adapter_config.json
-rw-rw-r--  1 root root 1225856253 Jul 27 12:41 adapter_model.bin
-rw-rw-r--  1 root root        435 Jul 27 12:41 special_tokens_map.json
-rw-rw-r--  1 root root     844403 Jul 27 12:41 tokenizer.model
-rw-rw-r--  1 root root        748 Jul 27 12:41 tokenizer_config.json

# pip list | grep -E 'transformers|peft|torch'
ctransformers            0.2.5
peft                     0.3.0.dev0
pytorch-quantization     2.1.2
sentence-transformers    2.2.2
torch                    2.0.1
torch-tensorrt           1.5.0.dev0
torchdata                0.7.0a0
torchtext                0.16.0a0
torchvision              0.16.0a0
transformers             4.31.0

# 请在此处粘贴运行日志

 pip list | grep -E 'transformers|peft|torch'
peft                     0.3.0.dev0
torch                    2.0.1
transformers             4.31.0

# 请在此处粘贴运行代码（如没有可删除该代码块）

# 请在此处粘贴依赖情况

# 请在此处粘贴运行日志

SFT代码
lr=2e-4
lora_rank=64
lora_alpha=128
lora_trainable="q_proj,v_proj,k_proj,o_proj,gate_proj,down_proj,up_proj"
modules_to_save="embed_tokens,lm_head"
lora_dropout=0.05

pretrained_model=chinese-alpaca-2-7b
chinese_tokenizer_path=chinese-alpaca-2-7b
dataset_dir=
per_device_train_batch_size=64
per_device_eval_batch_size=64
gradient_accumulation_steps=8
output_dir=output_dir/chinese-alpaca-2-7b-datav3_v2-sft-lr${lr}-rank${lora_rank}-alpha${lora_alpha}-dropout${lora_dropout}
validation_file=val.json

deepspeed_config_file=ds_zero2_no_offload.json

torchrun --nnodes 1 --nproc_per_node 1 run_clm_sft_with_peft.py \
    --deepspeed ${deepspeed_config_file} \
    --model_name_or_path ${pretrained_model} \
    --tokenizer_name_or_path ${chinese_tokenizer_path} \
    --dataset_dir ${dataset_dir} \
    --validation_split_percentage 0.001 \
    --per_device_train_batch_size ${per_device_train_batch_size} \
    --per_device_eval_batch_size ${per_device_eval_batch_size} \
    --do_train \
    --do_eval \
    --seed $RANDOM \
    --fp16 \
    --flash_attn \
    --num_train_epochs 3 \
    --lr_scheduler_type cosine \
    --learning_rate ${lr} \
    --warmup_ratio 0.03 \
    --weight_decay 0 \
    --logging_strategy steps \
    --logging_steps 10 \
    --save_strategy steps \
    --save_total_limit 3 \
    --evaluation_strategy steps \
    --eval_steps 50 \
    --save_steps 10 \
    --gradient_accumulation_steps ${gradient_accumulation_steps} \
    --preprocessing_num_workers 8 \
    --max_seq_length 1024 \
    --output_dir ${output_dir} \
    --overwrite_output_dir \
    --ddp_timeout 30000 \
    --logging_first_step True \
    --lora_rank ${lora_rank} \
    --lora_alpha ${lora_alpha} \
    --trainable ${lora_trainable} \
    --modules_to_save ${modules_to_save} \
    --lora_dropout ${lora_dropout} \
    --torch_dtype float16 \
    --validation_file ${validation_file} \
    --gradient_checkpointing \
    --ddp_find_unused_parameters False


CUDA_VISIBLE_DEVICES=1 python gradio_demo.py --base_model chinese-alpaca-2-7b --lora_model ../training/output_dir/chinese-alpaca-2-7b-datav3_v2-sft-l
r2e-4-rank64-alpha128-dropout0.05/checkpoint-10/sft_lora_model/

len(history): 1
history:  [['你好', None]]
Input length: 36
/home/daliqiji/miniconda3/envs/chllmalp2/lib/python3.10/site-packages/transformers/generation/utils.py:1219: UserWarning: You have modified the pretrained model configuration to control generation. This is a deprecated strategy to control generation and will be removed soon, in a future version. Please use a generation configuration file (see https://huggingface.co/docs/transformers/main_classes/text_generation)
  warnings.warn(
Traceback (most recent call last):
  File "/home/daliqiji/project/llm/Chinese-LLaMA-Alpaca-2/scripts/inference/gradio_demo.py", line 258, in gentask
    ret = self.mfunc(callback=_callback, **self.kwargs)
  File "/home/daliqiji/project/llm/Chinese-LLaMA-Alpaca-2/scripts/inference/gradio_demo.py", line 419, in generate_with_callback
    model.generate(**kwargs)
  File "/home/daliqiji/miniconda3/envs/chllmalp2/lib/python3.10/site-packages/peft/peft_model.py", line 581, in generate
    outputs = self.base_model.generate(**kwargs)
  File "/home/daliqiji/miniconda3/envs/chllmalp2/lib/python3.10/site-packages/torch/autograd/grad_mode.py", line 27, in decorate_context
    return func(*args, **kwargs)
  File "/home/daliqiji/miniconda3/envs/chllmalp2/lib/python3.10/site-packages/transformers/generation/utils.py", line 1485, in generate
    return self.sample(
  File "/home/daliqiji/miniconda3/envs/chllmalp2/lib/python3.10/site-packages/transformers/generation/utils.py", line 2524, in sample
    outputs = self(
  File "/home/daliqiji/miniconda3/envs/chllmalp2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl
    return forward_call(*input, **kwargs)
  File "/home/daliqiji/miniconda3/envs/chllmalp2/lib/python3.10/site-packages/accelerate/hooks.py", line 165, in new_forward
    output = old_forward(*args, **kwargs)
  File "/home/daliqiji/miniconda3/envs/chllmalp2/lib/python3.10/site-packages/transformers/models/llama/modeling_llama.py", line 687, in forward
    outputs = self.model(
  File "/home/daliqiji/miniconda3/envs/chllmalp2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl
    return forward_call(*input, **kwargs)
  File "/home/daliqiji/miniconda3/envs/chllmalp2/lib/python3.10/site-packages/transformers/models/llama/modeling_llama.py", line 577, in forward
    layer_outputs = decoder_layer(
  File "/home/daliqiji/miniconda3/envs/chllmalp2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl
    return forward_call(*input, **kwargs)
  File "/home/daliqiji/miniconda3/envs/chllmalp2/lib/python3.10/site-packages/transformers/models/llama/modeling_llama.py", line 292, in forward
    hidden_states, self_attn_weights, present_key_value = self.self_attn(
  File "/home/daliqiji/miniconda3/envs/chllmalp2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl
    return forward_call(*input, **kwargs)
  File "/home/daliqiji/project/llm/Chinese-LLaMA-Alpaca-2/scripts/attn_and_long_ctx_patches.py", line 44, in xformers_forward
    query_states = self.q_proj(hidden_states).view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
  File "/home/daliqiji/miniconda3/envs/chllmalp2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl
    return forward_call(*input, **kwargs)
  File "/home/daliqiji/miniconda3/envs/chllmalp2/lib/python3.10/site-packages/peft/tuners/lora.py", line 358, in forward
    result += self.lora_B(self.lora_A(self.lora_dropout(x))) * self.scaling
  File "/home/daliqiji/miniconda3/envs/chllmalp2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl
    return forward_call(*input, **kwargs)
  File "/home/daliqiji/miniconda3/envs/chllmalp2/lib/python3.10/site-packages/torch/nn/modules/linear.py", line 114, in forward
    return F.linear(input, self.weight, self.bias)
RuntimeError: expected scalar type Half but found Float

# 请在此处粘贴依赖情况

# 请在此处粘贴运行日志

wandb disabled
lr=1e-4
lora_rank=64
lora_alpha=128
lora_trainable="q_proj,v_proj,k_proj,o_proj,gate_proj,down_proj,up_proj"
modules_to_save="embed_tokens,lm_head"
lora_dropout=0.05

dir_date=$(date +%m%d)
train_version=v0.1

NUM_NODES=1
SEQ_LEN=2048
GC_SCALE=4
SKYPILOT_NUM_GPUS_PER_NODE=4
PER_DEVICE_BATCH_SIZE=$((2048 * $GC_SCALE / $SEQ_LEN))
GRADIENT_ACCUMULATION_STEPS=$((128 * 512 / $SEQ_LEN / $PER_DEVICE_BATCH_SIZE / $NUM_NODES / $SKYPILOT_NUM_GPUS_PER_NODE))

pretrained_model=/data/output/merged-chinese-alpaca-2-7b-lora-hf
chinese_tokenizer_path=/data/chinese-alpaca-2-lora-7b
peft_model=/data/chinese-alpaca-2-lora-7b
dataset_dir=/data/train-data/
# data cache 需要每次清理一下，否则加上之前的数据
data_cache=/data/cache/${dir_date}-${train_version}
rm -rf ${data_cache}*
per_device_train_batch_size=${PER_DEVICE_BATCH_SIZE}
per_device_eval_batch_size=${PER_DEVICE_BATCH_SIZE}
training_steps=1500
gradient_accumulation_steps=${GRADIENT_ACCUMULATION_STEPS}
output_dir=/data/output/fine-tunning-chinese-alpaca-2-7b-lora-${dir_date}-${train_version}
block_size=1024
max_seq_length=1024
deepspeed_config_file=ds_zero2_no_offload.json
validation_file=/data/train-data/fine_tunning.json
run_clm_sft_with_peft=run_clm_sft_with_peft.py

torchrun --nnodes 1 --nproc_per_node ${SKYPILOT_NUM_GPUS_PER_NODE} ${run_clm_sft_with_peft} \
    --deepspeed ${deepspeed_config_file} \
    --model_name_or_path ${pretrained_model} \
    --tokenizer_name_or_path ${chinese_tokenizer_path} \
    --peft_path ${chinese_tokenizer_path} \
    --dataset_dir ${dataset_dir} \
    --validation_split_percentage 0.001 \
    --per_device_train_batch_size ${per_device_train_batch_size} \
    --per_device_eval_batch_size ${per_device_eval_batch_size} \
    --do_train \
    --do_eval \
    --seed $RANDOM \
    --fp16 \
    --num_train_epochs 2 \
    --max_steps ${training_steps} \
    --lr_scheduler_type cosine \
    --learning_rate ${lr} \
    --warmup_ratio 0.03 \
    --weight_decay 0 \
    --logging_strategy steps \
    --logging_steps 10 \
    --save_strategy steps \
    --save_total_limit 3 \
    --evaluation_strategy steps \
    --eval_steps 250 \
    --save_steps 500 \
    --gradient_accumulation_steps ${gradient_accumulation_steps} \
    --preprocessing_num_workers 8 \
    --max_seq_length ${max_seq_length} \
    --output_dir ${output_dir} \
    --overwrite_output_dir \
    --ddp_timeout 30000 \
    --logging_first_step True \
    --gradient_checkpointing \
    --torch_dtype float16 \
    --ddp_find_unused_parameters False \
    --peft_path ${peft_model} \
    --gradient_checkpointing \
    --validation_file ${validation_file} \
    --flash_attn

peft                     0.3.0.dev0
torch                    2.0.1
transformers             4.31.0

Dataset json downloaded and prepared to /data/train-data/fine_tunning/json/default-a5e3f5abb73f5f82/0.0.0/8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96. Subsequent calls will reuse this data.
100%|█████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 925.08it/s]
08/03/2023 23:16:27 - WARNING - root - building dataset...
08/03/2023 23:16:27 - WARNING - root - building dataset...
Traceback (most recent call last):
  File "/data/python/Chinese-LLaMA-Alpaca/scripts/training/build_dataset.py", line 68, in build_instruction_dataset
    processed_dataset = datasets.load_from_disk(cache_path)
  File "/root/anaconda3/envs/alpaca/lib/python3.10/site-packages/datasets/load.py", line 1906, in load_from_disk
    raise FileNotFoundError(
FileNotFoundError: Directory /data/train-data/fine_tunning is neither a `Dataset` directory nor a `DatasetDict` directory.

During handling of the above exception, another exception occurred:

# 请在此处粘贴运行代码（如没有可删除该代码块）

# 请在此处粘贴依赖情况

# 请在此处粘贴运行代码（如没有可删除该代码块）

# 请在此处粘贴依赖情况

# 请在此处粘贴运行日志

# 请在此处粘贴运行代码（如没有可删除该代码块）

# 请在此处粘贴依赖情况

# 请在此处粘贴运行日志

# 请在此处粘贴依赖情况

# 请在此处粘贴运行日志

# 请在此处粘贴运行代码（如没有可删除该代码块）

# 请在此处粘贴依赖情况

# 请在此处粘贴运行日志

# 请在此处粘贴依赖情况

# 请在此处粘贴运行日志

# 训练代码：跟仓库里run_pt那个基本一样
#!/usr/bin/env python
# coding=utf-8
# Copyright 2020 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Fine-tuning the library models for causal language modeling (GPT, GPT-2, CTRL, ...) on a text file or a dataset.

Here is the full list of checkpoints on the hub that can be fine-tuned by this script:
https://huggingface.co/models?filter=text-generation
"""
# You can also adapt this script on your own causal language modeling task. Pointers for this are left as comments.

import logging
import numpy as np
import math
import os
import sys
from dataclasses import dataclass, field
from itertools import chain
from typing import Optional, List, Dict, Any, Mapping
from pathlib import Path
import datasets
import torch
from datasets import load_dataset, concatenate_datasets

import transformers
from transformers import (
    CONFIG_MAPPING,
    MODEL_FOR_CAUSAL_LM_MAPPING,
    AutoConfig,
    AutoModelForCausalLM,
    LlamaForCausalLM,
    LlamaTokenizer,
    AutoTokenizer,
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    is_torch_tpu_available,
    set_seed,
)
from transformers.testing_utils import CaptureLogger
from transformers.trainer_utils import get_last_checkpoint
from transformers.utils import send_example_telemetry
from transformers.utils.versions import require_version

from sklearn.metrics import accuracy_score
from peft import LoraConfig, TaskType, get_peft_model, PeftModel, get_peft_model_state_dict
from transformers.trainer_utils import PREFIX_CHECKPOINT_DIR


class SavePeftModelCallback(transformers.TrainerCallback):
    def save_model(self, args, state, kwargs):
        if state.best_model_checkpoint is not None:
            checkpoint_folder = os.path.join(state.best_model_checkpoint, "pt_lora_model")
        else:
            checkpoint_folder = os.path.join(args.output_dir, f"{PREFIX_CHECKPOINT_DIR}-{state.global_step}")

        peft_model_path = os.path.join(checkpoint_folder, "pt_lora_model")
        kwargs["model"].save_pretrained(peft_model_path)
        kwargs["tokenizer"].save_pretrained(peft_model_path)

    def on_save(self, args, state, control, **kwargs):
        self.save_model(args, state, kwargs)
        return control

    def on_train_end(self, args, state, control, **kwargs):
        peft_model_path = os.path.join(args.output_dir, "pt_lora_model")
        kwargs["model"].save_pretrained(peft_model_path)
        kwargs["tokenizer"].save_pretrained(peft_model_path)


def accuracy(predictions, references, normalize=True, sample_weight=None):
        return {
            "accuracy": float(
                accuracy_score(references, predictions, normalize=normalize, sample_weight=sample_weight)
            )
        }


def compute_metrics(eval_preds):
    preds, labels = eval_preds
    # preds have the same shape as the labels, after the argmax(-1) has been calculated
    # by preprocess_logits_for_metrics but we need to shift the labels
    labels = labels[:, 1:].reshape(-1)
    preds = preds[:, :-1].reshape(-1)
    return accuracy(predictions=preds, references=labels)


def preprocess_logits_for_metrics(logits, labels):
    if isinstance(logits, tuple):
        # Depending on the model and config, logits may contain extra tensors,
        # like past_key_values, but logits always come first
        logits = logits[0]
    return logits.argmax(dim=-1)


def fault_tolerance_data_collator(features: List) -> Dict[str, Any]:
    if not isinstance(features[0], Mapping):
        features = [vars(f) for f in features]
    first = features[0]
    batch = {}

    # Special handling for labels.
    # Ensure that tensor is created with the correct type
    # (it should be automatically the case, but let's make sure of it.)
    if "label" in first and first["label"] is not None:
        label = first["label"].item() if isinstance(first["label"], torch.Tensor) else first["label"]
        dtype = torch.long if isinstance(label, int) else torch.float
        batch["labels"] = torch.tensor([f["label"] for f in features], dtype=dtype)
    elif "label_ids" in first and first["label_ids"] is not None:
        if isinstance(first["label_ids"], torch.Tensor):
            batch["labels"] = torch.stack([f["label_ids"] for f in features])
        else:
            dtype = torch.long if isinstance(first["label_ids"][0], int) else torch.float
            batch["labels"] = torch.tensor([f["label_ids"] for f in features], dtype=dtype)

    # Handling of all other possible keys.
    # Again, we will use the first element to figure out which key/values are not None for this model.

    try:
        for k, v in first.items():
            if k not in ("label", "label_ids") and v is not None and not isinstance(v, str):
                if isinstance(v, torch.Tensor):
                    batch[k] = torch.stack([f[k] for f in features])
                elif isinstance(v, np.ndarray):
                    batch[k] = torch.tensor(np.stack([f[k] for f in features]))
                else:
                    batch[k] = torch.tensor([f[k] for f in features])
    except ValueError: # quick fix by simply take the first example
        for k, v in first.items():
            if k not in ("label", "label_ids") and v is not None and not isinstance(v, str):
                if isinstance(v, torch.Tensor):
                    batch[k] = torch.stack([features[0][k]] * len(features))
                elif isinstance(v, np.ndarray):
                    batch[k] = torch.tensor(np.stack([features[0][k]] * len(features)))
                else:
                    batch[k] = torch.tensor([features[0][k]] * len(features))

    return batch


MODEL_CONFIG_CLASSES = list(MODEL_FOR_CAUSAL_LM_MAPPING.keys())
MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)


@dataclass
class ModelArguments:
    """
    Arguments pertaining to which model/config/tokenizer we are going to fine-tune, or train from scratch.
    """

    model_name_or_path: Optional[str] = field(
        default=None,
        metadata={
            "help": (
                "The model checkpoint for weights initialization.Don't set if you want to train a model from scratch."
            )
        },
    )
    tokenizer_name_or_path: Optional[str] = field(
        default=None,
        metadata={
            "help": (
                "The tokenizer for weights initialization.Don't set if you want to train a model from scratch."
            )
        },
    )
    model_type: Optional[str] = field(
        default=None,
        metadata={"help": "If training from scratch, pass a model type from the list: " + ", ".join(MODEL_TYPES)},
    )
    config_overrides: Optional[str] = field(
        default=None,
        metadata={
            "help": (
                "Override some existing default config settings when a model is trained from scratch. Example: "
                "n_embd=10,resid_pdrop=0.2,scale_attn_weights=false,summary_type=cls_index"
            )
        },
    )
    config_name: Optional[str] = field(
        default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"}
    )
    tokenizer_name: Optional[str] = field(
        default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
    )
    cache_dir: Optional[str] = field(
        default=None,
        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
    )
    use_fast_tokenizer: bool = field(
        default=True,
        metadata={"help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."},
    )
    model_revision: str = field(
        default="main",
        metadata={"help": "The specific model version to use (can be a branch name, tag name or commit id)."},
    )
    use_auth_token: bool = field(
        default=False,
        metadata={
            "help": (
                "Will use the token generated when running `huggingface-cli login` (necessary to use this script "
                "with private models)."
            )
        },
    )
    torch_dtype: Optional[str] = field(
        default=None,
        metadata={
            "help": (
                "Override the default `torch.dtype` and load the model under this dtype. If `auto` is passed, the "
                "dtype will be automatically derived from the model's weights."
            ),
            "choices": ["auto", "bfloat16", "float16", "float32"],
        },
    )

    def __post_init__(self):
        if self.config_overrides is not None and (self.config_name is not None or self.model_name_or_path is not None):
            raise ValueError(
                "--config_overrides can't be used in combination with --config_name or --model_name_or_path"
            )


@dataclass
class DataTrainingArguments:
    """
    Arguments pertaining to what data we are going to input our model for training and eval.
    """

    dataset_dir: Optional[str] = field(
        default=None, metadata={"help": "The name of the dataset to use (via the datasets library)."}
    )
    dataset_config_name: Optional[str] = field(
        default=None, metadata={"help": "The configuration name of the dataset to use (via the datasets library)."}
    )
    train_file: Optional[str] = field(default=None, metadata={"help": "The input training data file (a text file)."})
    validation_file: Optional[str] = field(
        default=None,
        metadata={"help": "An optional input evaluation data file to evaluate the perplexity on (a text file)."},
    )
    max_train_samples: Optional[int] = field(
        default=None,
        metadata={
            "help": (
                "For debugging purposes or quicker training, truncate the number of training examples to this "
                "value if set."
            )
        },
    )
    max_eval_samples: Optional[int] = field(
        default=None,
        metadata={
            "help": (
                "For debugging purposes or quicker training, truncate the number of evaluation examples to this "
                "value if set."
            )
        },
    )
    streaming: bool = field(default=False, metadata={"help": "Enable streaming mode"})
    block_size: Optional[int] = field(
        default=None,
        metadata={
            "help": (
                "Optional input sequence length after tokenization. "
                "The training dataset will be truncated in block of this size for training. "
                "Default to the model max input length for single sentence inputs (take into account special tokens)."
            )
        },
    )
    overwrite_cache: bool = field(
        default=False, metadata={"help": "Overwrite the cached training and evaluation sets"}
    )
    validation_split_percentage: Optional[float] = field(
        default=0.05,
        metadata={
            "help": "The percentage of the train set used as validation set in case there's no validation split"
        },
    )
    preprocessing_num_workers: Optional[int] = field(
        default=None,
        metadata={"help": "The number of processes to use for the preprocessing."},
    )
    keep_linebreaks: bool = field(
        default=True, metadata={"help": "Whether to keep line breaks when using TXT files or not."}
    )
    data_cache_dir: Optional[str] = field(default="./", metadata={"help": "The datasets processed stored"})

    def __post_init__(self):
        if self.streaming:
            require_version("datasets>=2.0.0", "The streaming feature requires `datasets>=2.0.0`")


@dataclass
class MyTrainingArguments(TrainingArguments):
    trainable : Optional[str] = field(default="q_proj,v_proj")
    lora_rank : Optional[int] = field(default=8)
    lora_dropout : Optional[float] = field(default=0.1)
    lora_alpha : Optional[float] = field(default=32.)
    modules_to_save : Optional[str] = field(default=None)
    debug_mode : Optional[bool] = field(default=False)
    peft_path : Optional[str] = field(default=None)
    flash_attn : Optional[bool] = field(default=False)
    train_peft : Optional[bool] = field(default=True)


logger = logging.getLogger(__name__)


def main():

    parser = HfArgumentParser((ModelArguments, DataTrainingArguments, MyTrainingArguments))
    if len(sys.argv) == 2 and sys.argv[1].endswith(".json"):
        # If we pass only one argument to the script and it's the path to a json file,
        # let's parse it to get our arguments.
        model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]))
    else:
        model_args, data_args, training_args = parser.parse_args_into_dataclasses()
    if training_args.flash_attn:
        from flash_attn_patch import replace_llama_attn_with_flash_attn
        replace_llama_attn_with_flash_attn()

    # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The
    # information sent is the one passed as arguments along with your Python/PyTorch versions.
    send_example_telemetry("run_clm", model_args, data_args)

    # Setup logging
    logging.basicConfig(format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",datefmt="%m/%d/%Y %H:%M:%S",
        level=logging.INFO,  # if training_args.local_rank in [-1, 0] else logging.WARN,
        handlers=[logging.StreamHandler(sys.stdout)],)

    if training_args.should_log:
        # The default of training_args.log_level is passive, so we set log level at info here to have that default.
        transformers.utils.logging.set_verbosity_info()

    log_level = training_args.get_process_log_level()
    logger.setLevel(log_level)
    datasets.utils.logging.set_verbosity(log_level)
    transformers.utils.logging.set_verbosity(log_level)
    transformers.utils.logging.enable_default_handler()
    transformers.utils.logging.enable_explicit_format()
    # transformers.tokenization_utils.logging.set_verbosity_warning()

    # Log on each process the small summary:
    logger.warning(
        f"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}"
        + f"distributed training: {bool(training_args.local_rank != -1)}, 16-bits training: {training_args.fp16}"
    )

    # Detecting last checkpoint.
    last_checkpoint = None
    if os.path.isdir(training_args.output_dir) and training_args.do_train and not training_args.overwrite_output_dir:
        last_checkpoint = get_last_checkpoint(training_args.output_dir)
        if last_checkpoint is None and len(os.listdir(training_args.output_dir)) > 0:
            raise ValueError(
                f"Output directory ({training_args.output_dir}) already exists and is not empty. "
                "Use --overwrite_output_dir to overcome."
            )
        elif last_checkpoint is not None and training_args.resume_from_checkpoint is None:
            logger.info(
                f"Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change "
                "the `--output_dir` or add `--overwrite_output_dir` to train from scratch."
            )

    # Set seed before initializing model.
    set_seed(training_args.seed)

    config_kwargs = {
        "cache_dir": model_args.cache_dir,
        "revision": model_args.model_revision,
        "use_auth_token": True if model_args.use_auth_token else None,
    }
    if model_args.config_name:
        config = AutoConfig.from_pretrained(model_args.config_name, **config_kwargs)
    elif model_args.model_name_or_path:
        config = AutoConfig.from_pretrained(model_args.model_name_or_path, **config_kwargs)
    else:
        config = CONFIG_MAPPING[model_args.model_type]()
        logger.warning("You are instantiating a new config instance from scratch.")
        if model_args.config_overrides is not None:
            logger.info(f"Overriding config: {model_args.config_overrides}")
            config.update_from_string(model_args.config_overrides)
            logger.info(f"New config: {config}")

    tokenizer_kwargs = {
        "cache_dir": model_args.cache_dir,
        "use_fast": model_args.use_fast_tokenizer,
        "revision": model_args.model_revision,
        "use_auth_token": True if model_args.use_auth_token else None,
    }
    if model_args.tokenizer_name:
        tokenizer = AutoTokenizer.from_pretrained(model_args.tokenizer_name, **tokenizer_kwargs)
    elif model_args.tokenizer_name_or_path:
        tokenizer = LlamaTokenizer.from_pretrained(model_args.tokenizer_name_or_path, **tokenizer_kwargs)
    else:
        raise ValueError(
            "You are instantiating a new tokenizer from scratch. This is not supported by this script."
            "You can do it from another script, save it, and load it from here, using --tokenizer_name."
        )

    # Preprocessing the datasets.
    # First we tokenize all the texts.
    # since this will be pickled to avoid _LazyModule error in Hasher force logger loading before tokenize_function
    tok_logger = transformers.utils.logging.get_logger("transformers.tokenization_utils_base")

    def tokenize_function(examples):
        with CaptureLogger(tok_logger) as cl:
            output = tokenizer(examples["text"])
        # clm input could be much much longer than block_size
        if "Token indices sequence length is longer than the" in cl.out:
            tok_logger.warning(
                "^^^^^^^^^^^^^^^^ Please ignore the warning above - this long input will be chunked into smaller bits"
                " before being passed to the model."
            )
        return output
    if data_args.block_size is None:
        block_size = tokenizer.model_max_length
        if block_size > 1024:
            logger.warning(
                "The chosen tokenizer supports a `model_max_length` that is longer than the default `block_size` value"
                " of 1024. If you would like to use a longer `block_size` up to `tokenizer.model_max_length` you can"
                " override this default with `--block_size xxx`."
            )
            block_size = 1024
    else:
        if data_args.block_size > tokenizer.model_max_length:
            logger.warning(
                f"The block_size passed ({data_args.block_size}) is larger than the maximum length for the model"
                f"({tokenizer.model_max_length}). Using block_size={tokenizer.model_max_length}."
            )
        block_size = min(data_args.block_size, tokenizer.model_max_length)

    # Main data processing function that will concatenate all texts from our dataset and generate chunks of block_size.
    def group_texts(examples):
        # Concatenate all texts.
        concatenated_examples = {k: list(chain(*examples[k])) for k in examples.keys()}
        total_length = len(concatenated_examples[list(examples.keys())[0]])
        # We drop the small remainder, we could add padding if the model supported it instead of this drop, you can
        # customize this part to your needs.
        if total_length >= block_size:
            total_length = (total_length // block_size) * block_size
        # Split by chunks of max_len.
        result = {
            k: [t[i : i + block_size] for i in range(0, total_length, block_size)]
            for k, t in concatenated_examples.items()
        }
        result["labels"] = result["input_ids"].copy()
        return result
    with training_args.main_process_first(desc="dataset map tokenization and grouping"):
        lm_datasets = []
        path = Path(data_args.dataset_dir)
        files = [file.name for file in path.glob("*.txt")]
        if training_args.debug_mode is True:
            files = [files[0]]
        for idx, file in enumerate(files):
            data_file = os.path.join(path, file)
            filename = ''.join(file.split(".")[:-1])
            cache_path = os.path.join(data_args.data_cache_dir, filename)
            os.makedirs(cache_path, exist_ok=True)
            try:
                processed_dataset = datasets.load_from_disk(cache_path, keep_in_memory=False)
                logger.info(f'training datasets-{filename} has been loaded from disk')
            except Exception:
                cache_dir = os.path.join(data_args.data_cache_dir, filename+"_text")
                os.makedirs(cache_dir, exist_ok=True)
                raw_dataset = load_dataset("text", data_files=data_file, cache_dir=cache_dir, keep_in_memory=False)
                logger.info(f"{file} has been loaded")
                tokenized_dataset = raw_dataset.map(
                    tokenize_function,
                    batched=True,
                    num_proc=data_args.preprocessing_num_workers,
                    remove_columns="text",
                    load_from_cache_file=True,
                    keep_in_memory=False,
                    cache_file_names = {k: os.path.join(cache_dir, 'tokenized.arrow') for k in raw_dataset},
                    desc="Running tokenizer on dataset",
                )
                grouped_datasets = tokenized_dataset.map(
                    group_texts,
                    batched=True,
                    num_proc=data_args.preprocessing_num_workers,
                    load_from_cache_file=True,
                    keep_in_memory=False,
                    cache_file_names = {k: os.path.join(cache_dir, 'grouped.arrow') for k in tokenized_dataset},
                    desc=f"Grouping texts in chunks of {block_size}",
                )
                processed_dataset = grouped_datasets
                processed_dataset.save_to_disk(cache_path)
            if idx == 0:
                lm_datasets = processed_dataset['train']
            else:
                assert lm_datasets.features.type == processed_dataset["train"].features.type
                lm_datasets = concatenate_datasets([lm_datasets, processed_dataset["train"]])

        lm_datasets = lm_datasets.train_test_split(test_size = data_args.validation_split_percentage)

    if training_args.do_train:
        train_dataset = lm_datasets['train']
        if data_args.max_train_samples is not None:
            max_train_samples = min(len(train_dataset), data_args.max_train_samples)
            train_dataset = train_dataset.select(range(max_train_samples))
        logger.info(f"Num train_samples  {len(train_dataset)}")
        logger.info("Training example:")
        logger.info(tokenizer.decode(train_dataset[0]['input_ids']))
    if training_args.do_eval:
        eval_dataset = lm_datasets["test"]
        if data_args.max_eval_samples is not None:
            max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples)
            eval_dataset = eval_dataset.select(range(max_eval_samples))
        logger.info(f"Num eval_samples  {len(eval_dataset)}")
        logger.info("Evaluation example:")
        logger.info(tokenizer.decode(eval_dataset[0]['input_ids']))
    if model_args.model_name_or_path:
        torch_dtype = (
            model_args.torch_dtype
            if model_args.torch_dtype in ["auto", None]
            else getattr(torch, model_args.torch_dtype)
        )
        model = LlamaForCausalLM.from_pretrained(
            model_args.model_name_or_path,
            from_tf=bool(".ckpt" in model_args.model_name_or_path),
            config=config,
            cache_dir=model_args.cache_dir,
            revision=model_args.model_revision,
            use_auth_token=True if model_args.use_auth_token else None,
            torch_dtype=torch_dtype,
            low_cpu_mem_usage=True,
            load_in_8bit=True,
            device_map='auto'
        )
    else:
        model = AutoModelForCausalLM.from_config(config)
        n_params = sum({p.data_ptr(): p.numel() for p in model.parameters()}.values())
        logger.info(f"Training new model from scratch - Total size={n_params/2**20:.2f}M params")

    model_vocab_size = model.get_output_embeddings().weight.size(0)
    tokenizer_vocab_size = len(tokenizer)
    logger.info(f"Model vocab size: {model_vocab_size}")
    logger.info(f"Tokenizer vocab size: {tokenizer_vocab_size}")
    if tokenizer_vocab_size != 55296:
        raise ValueError(f"The vocab size of tokenizer is {tokenizer_vocab_size}, not 55296. Please use Chinese-LLaMA-2 tokenizer.")
    if model_vocab_size != tokenizer_vocab_size:
        logger.info(f"Rezize model vocab size to {tokenizer_vocab_size}")
        model.resize_token_embeddings(len(tokenizer))
    
    if training_args.train_peft:
        logger.info("Train Peft Model!")
        if training_args.peft_path is not None:
            logger.info("Peft from pre-trained model")
            model = PeftModel.from_pretrained(model, training_args.peft_path)
        else:
            logger.info("Init new peft model")
            target_modules = training_args.trainable.split(',')
            modules_to_save = training_args.modules_to_save
            if modules_to_save is not None:
                modules_to_save = modules_to_save.split(',')
            lora_rank = training_args.lora_rank
            lora_dropout = training_args.lora_dropout
            lora_alpha = training_args.lora_alpha
            logger.info(f"target_modules: {target_modules}")
            logger.info(f"lora_rank: {lora_rank}")
            peft_config = LoraConfig(
                task_type=TaskType.CAUSAL_LM,
                target_modules=target_modules,
                inference_mode=False,
                r=lora_rank, lora_alpha=lora_alpha,
                lora_dropout=lora_dropout,
                modules_to_save=modules_to_save)
            model = get_peft_model(model, peft_config)
        model.print_trainable_parameters()
        old_state_dict = model.state_dict
        model.state_dict = (
            lambda self, *_, **__: get_peft_model_state_dict(self, old_state_dict())
        ).__get__(model, type(model))
    else:
        logger.info("Tranin Full Model!")

    # Initialize our Trainer
    trainer = Trainer(
        model=model,
        args=training_args,
        train_dataset=train_dataset if training_args.do_train else None,
        eval_dataset=eval_dataset if training_args.do_eval else None,
        tokenizer=tokenizer,
        data_collator=fault_tolerance_data_collator,
        compute_metrics=compute_metrics if training_args.do_eval and not is_torch_tpu_available() else None,
        preprocess_logits_for_metrics=preprocess_logits_for_metrics
        if training_args.do_eval and not is_torch_tpu_available()
        else None,
    )
    trainer.add_callback(SavePeftModelCallback)
    # Training
    if training_args.do_train:
        checkpoint = None
        if training_args.resume_from_checkpoint is not None:
            checkpoint = training_args.resume_from_checkpoint
        elif last_checkpoint is not None:
            checkpoint = last_checkpoint
        
        with torch.autocast("cuda"): 
            train_result = trainer.train(resume_from_checkpoint=checkpoint)

        metrics = train_result.metrics

        max_train_samples = (
            data_args.max_train_samples if data_args.max_train_samples is not None else len(train_dataset)
        )
        metrics["train_samples"] = min(max_train_samples, len(train_dataset))

        trainer.log_metrics("train", metrics)
        trainer.save_metrics("train", metrics)
        trainer.save_state()

    # Evaluation
    if training_args.do_eval:
        logger.info("*** Evaluate ***")
        
        with torch.autocast("cuda"): 
            metrics = trainer.evaluate()

        max_eval_samples = data_args.max_eval_samples if data_args.max_eval_samples is not None else len(eval_dataset)
        metrics["eval_samples"] = min(max_eval_samples, len(eval_dataset))
        try:
            perplexity = math.exp(metrics["eval_loss"])
        except OverflowError:
            perplexity = float("inf")
        metrics["perplexity"] = perplexity

        trainer.log_metrics("eval", metrics)
        trainer.save_metrics("eval", metrics)


if __name__ == "__main__":
    main()

lr=2e-4
lora_rank=64
lora_alpha=128
lora_trainable="q_proj,v_proj,k_proj,o_proj,gate_proj,down_proj,up_proj"
modules_to_save="embed_tokens,lm_head"
lora_dropout=0.05

pretrained_model=/code/xx/LLM_mine/model/LLama2/llama2_chinese
chinese_tokenizer_path=/code/xx/LLM_mine/model/LLama2/llama2_chinese
dataset_dir=/code/xx/LLM_mine/data/wudao_test
data_cache=/code/xx/LLM_mine/scripts/pretraining/pretrain_output/data_cache_wudao
per_device_train_batch_size=1
per_device_eval_batch_size=1
gradient_accumulation_steps=1
output_dir=/code/xx/LLM_mine/scripts/pretraining/pretrain_output/pretrain_llama2_final

deepspeed_config_file=/code/xx/LLM_mine/scripts/pretraining/ds_zero2_no_offload.json

torchrun --nnodes 2 --nproc_per_node 5 --rdzv_id=100 --rdzv_backend=c10d --rdzv_endpoint=${MASTER_ADDR}:29400 /code/xiongxiong/LLM_mine/scripts/pretraining/run_pt_llama2_flashattn.py \
    --deepspeed ${deepspeed_config_file} \
    --model_name_or_path ${pretrained_model} \
    --tokenizer_name_or_path ${chinese_tokenizer_path} \
    --dataset_dir ${dataset_dir} \
    --data_cache_dir ${data_cache} \
    --validation_split_percentage 0.001 \
    --per_device_train_batch_size ${per_device_train_batch_size} \
    --per_device_eval_batch_size ${per_device_eval_batch_size} \
    --do_train \
    --seed 666 \
    --fp16 \
    --num_train_epochs 1 \
    --lr_scheduler_type cosine \
    --learning_rate ${lr} \
    --warmup_ratio 0.05 \
    --weight_decay 0.01 \
    --logging_strategy steps \
    --logging_steps 10 \
    --save_strategy steps \
    --save_total_limit 3 \
    --save_steps 50 \
    --gradient_accumulation_steps ${gradient_accumulation_steps} \
    --preprocessing_num_workers 8 \
    --block_size 1024 \
    --output_dir ${output_dir} \
    --overwrite_output_dir \
    --ddp_timeout 30000 \
    --logging_first_step True \
    --lora_rank ${lora_rank} \
    --lora_alpha ${lora_alpha} \
    --trainable ${lora_trainable} \
    --modules_to_save ${modules_to_save} \
    --lora_dropout ${lora_dropout} \
    --torch_dtype float16 \
    --gradient_checkpointing \
    --ddp_find_unused_parameters False \
    --train_peft True \
    --flash_attn False

 RUN pip install git+https://github.com/huggingface/peft.git@13e53fc
 RUN pip install transformers==4.31.0
 RUN pip install sentencepiece==0.1.97
 RUN pip install bitsandbytes==0.39.1
 RUN pip install xformers
 RUN MAX_JOBS=2 pip install flash-attn --no-build-isolation -i https://mirrors.aliyun.com/pypi/simple

[INFO|trainer.py:1686] 2023-08-03 17:19:49,528 >> ***** Running training *****

[INFO|trainer.py:1687] 2023-08-03 17:19:49,528 >>   Num examples = 1,553,003

[INFO|trainer.py:1688] 2023-08-03 17:19:49,528 >>   Num Epochs = 1

[INFO|trainer.py:1689] 2023-08-03 17:19:49,528 >>   Instantaneous batch size per device = 1

[INFO|trainer.py:1692] 2023-08-03 17:19:49,528 >>   Total train batch size (w. parallel, distributed & accumulation) = 32

[INFO|trainer.py:1693] 2023-08-03 17:19:49,528 >>   Gradient Accumulation steps = 4

[INFO|trainer.py:1694] 2023-08-03 17:19:49,528 >>   Total optimization steps = 48,531

[INFO|trainer.py:1695] 2023-08-03 17:19:49,535 >>   Number of trainable parameters = 1,734,344,704

[INFO|integrations.py:716] 2023-08-03 17:19:49,545 >> Automatic Weights & Biases logging enabled, to disable set os.environ["WANDB_DISABLED"] = "true"

wandb: Currently logged in as: doublebear315. Use `wandb login --relogin` to force relogin

wandb: Tracking run with wandb version 0.15.3

wandb: Run data is saved locally in /wandb/run-20230803_171951-cglufcfi

wandb: Run `wandb offline` to turn off syncing.

wandb: Syncing run efficient-breeze-127

wandb: ⭐️ View project at https://wandb.ai/doublebear315/huggingface

wandb: 🚀 View run at https://wandb.ai/doublebear315/huggingface/runs/cglufcfi


  0%|          | 0/48531 [00:00> `use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...

Traceback (most recent call last):

  File "/code/xiongxiong/LLM_mine/scripts/pretraining/run_pt_llama2_flashattn.py", line 646, in 

    main()

  File "/code/xiongxiong/LLM_mine/scripts/pretraining/run_pt_llama2_flashattn.py", line 613, in main

    train_result = trainer.train(resume_from_checkpoint=checkpoint)

  File "/opt/conda/lib/python3.10/site-packages/transformers/trainer.py", line 1539, in train

    return inner_training_loop(

  File "/opt/conda/lib/python3.10/site-packages/transformers/trainer.py", line 1809, in _inner_training_loop

    tr_loss_step = self.training_step(model, inputs)

  File "/opt/conda/lib/python3.10/site-packages/transformers/trainer.py", line 2654, in training_step

    loss = self.compute_loss(model, inputs)

  File "/opt/conda/lib/python3.10/site-packages/transformers/trainer.py", line 2679, in compute_loss

    outputs = model(**inputs)

  File "/opt/conda/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl

    return forward_call(*args, **kwargs)

  File "/opt/conda/lib/python3.10/site-packages/deepspeed/utils/nvtx.py", line 15, in wrapped_fn

    ret_val = func(*args, **kwargs)

  File "/opt/conda/lib/python3.10/site-packages/deepspeed/runtime/engine.py", line 1735, in forward

    loss = self.module(*inputs, **kwargs)

  File "/opt/conda/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl

    return forward_call(*args, **kwargs)

  File "/opt/conda/lib/python3.10/site-packages/peft/peft_model.py", line 529, in forward

    return self.base_model(

  File "/opt/conda/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl

    return forward_call(*args, **kwargs)

  File "/opt/conda/lib/python3.10/site-packages/accelerate/hooks.py", line 165, in new_forward

    output = old_forward(*args, **kwargs)

  File "/opt/conda/lib/python3.10/site-packages/transformers/models/llama/modeling_llama.py", line 806, in forward

    outputs = self.model(

  File "/opt/conda/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl

    return forward_call(*args, **kwargs)

  File "/opt/conda/lib/python3.10/site-packages/accelerate/hooks.py", line 165, in new_forward

    output = old_forward(*args, **kwargs)

  File "/opt/conda/lib/python3.10/site-packages/transformers/models/llama/modeling_llama.py", line 685, in forward

    layer_outputs = torch.utils.checkpoint.checkpoint(

  File "/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py", line 249, in checkpoint

    return CheckpointFunction.apply(function, preserve, *args)

  File "/opt/conda/lib/python3.10/site-packages/torch/autograd/function.py", line 506, in apply

    return super().apply(*args, **kwargs)  # type: ignore[misc]

  File "/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py", line 107, in forward

    outputs = run_function(*args)

  File "/opt/conda/lib/python3.10/site-packages/transformers/models/llama/modeling_llama.py", line 681, in custom_forward

    return module(*inputs, output_attentions, None)

  File "/opt/conda/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl

    return forward_call(*args, **kwargs)

  File "/opt/conda/lib/python3.10/site-packages/accelerate/hooks.py", line 165, in new_forward

    output = old_forward(*args, **kwargs)

  File "/opt/conda/lib/python3.10/site-packages/transformers/models/llama/modeling_llama.py", line 408, in forward

    hidden_states, self_attn_weights, present_key_value = self.self_attn(

  File "/opt/conda/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl

    return forward_call(*args, **kwargs)

  File "/opt/conda/lib/python3.10/site-packages/accelerate/hooks.py", line 165, in new_forward

    output = old_forward(*args, **kwargs)

  File "/code/xiongxiong/LLM_mine/scripts/pretraining/flash_attn_patch.py", line 39, in forward

    .view(bsz, q_len, self.num_heads, self.head_dim)

RuntimeError: shape '[1, 1024, 64, 128]' is invalid for input of size 1048576

wandb: Waiting for W&B process to finish... (failed 1). Press Control-C to abort syncing.

wandb: - 0.015 MB of 0.015 MB uploaded (0.000 MB deduped)
wandb: \ 0.015 MB of 0.040 MB uploaded (0.000 MB deduped)
wandb: | 0.040 MB of 0.040 MB uploaded (0.000 MB deduped)
wandb: / 0.040 MB of 0.040 MB uploaded (0.000 MB deduped)
wandb: 🚀 View run efficient-breeze-127 at: https://wandb.ai/doublebear315/huggingface/runs/cglufcfi

wandb: Synced 6 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)

wandb: Find logs at: 

ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 1157) of binary: /opt/conda/bin/python

Traceback (most recent call last):

  File "/opt/conda/bin/torchrun", line 8, in 

    sys.exit(main())

  File "/opt/conda/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper

    return f(*args, **kwargs)

  File "/opt/conda/lib/python3.10/site-packages/torch/distributed/run.py", line 794, in main

    run(args)

  File "/opt/conda/lib/python3.10/site-packages/torch/distributed/run.py", line 785, in run

    elastic_launch(

  File "/opt/conda/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 134, in __call__

    return launch_agent(self._config, self._entrypoint, list(args))

  File "/opt/conda/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 250, in launch_agent

    raise ChildFailedError(

torch.distributed.elastic.multiprocessing.errors.ChildFailedError: 

============================================================

/code/xiongxiong/LLM_mine/scripts/pretraining/run_pt_llama2_flashattn.py FAILED

------------------------------------------------------------

Failures:

  

------------------------------------------------------------

Root Cause (first observed failure):

[0]:

  time      : 2023-08-03_17:20:13

  host      : pd-xiongxiong-p-tra-asbqtmoctdau-torchjob-master-0

  rank      : 0 (local_rank: 0)

  exitcode  : 1 (pid: 1157)

  error_file: 

  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html

============================================================

# 请在此处粘贴运行代码（如没有可删除该代码块）

# 请在此处粘贴依赖情况

# 请在此处粘贴运行日志

# 请在此处粘贴依赖情况

# 请在此处粘贴运行日志

CUDA_VISIBLE_DEVICES=0,1 deepspeed --master_port 61001 train_full.py \
    --data_path ./data/train_data2.json \
    --model_name_or_path checkpoints/llama2-7B \
    --per_device_train_batch_size 1 --output_dir out/fschat_7b_full \
    --fp16 --num_train_epochs 3 \
    --per_device_eval_batch_size 1 \
    --gradient_accumulation_steps 84 \
    --learning_rate 2e-5 --weight_decay 0. \
    --warmup_ratio 0.03 --lr_scheduler_type "cosine" \
    --model_max_length 1344 \
    --logging_steps 10```

Exception: Current loss scale already at minimum - cannot decrease scale anymore. Exiting run.
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮
│                                                                                                  │
│    98                                                                                            │
│    99                                                                                            │
│   100 if __name__ == "__main__":                                                                 │
│ ❱ 101 │   train()                                                                                │
│   102                                                                                            │
│                                                                                                  │
│                                                                                                  │
│    92 │   if list(pathlib.Path(training_args.output_dir).glob("checkpoint-*")):                  │
│    93 │   │   trainer.train(resume_from_checkpoint=True)                                         │
│    94 │   else:                                                                                  │
│ ❱  95 │   │   trainer.train()                                                                    │
│    96 │   trainer.save_state()                                                                   │
│    97 │   safe_save_model_for_hf_trainer(trainer=trainer, output_dir=training_args.output_dir)   │
│    98                                                                                            │
│                                                                                                  │
│ /opt/conda/lib/python3.10/site-packages/transformers/trainer.py:1645 in train                    │
│                                                                                                  │
│   1642 │   │   inner_training_loop = find_executable_batch_size(                                 │
│   1643 │   │   │   self._inner_training_loop, self._train_batch_size, args.auto_find_batch_size  │
│   1644 │   │   )                                                                                 │
│ ❱ 1645 │   │   return inner_training_loop(                                                       │
│   1646 │   │   │   args=args,                                                                    │
│   1647 │   │   │   resume_from_checkpoint=resume_from_checkpoint,                                │
│   1648 │   │   │   trial=trial,                                                                  │
│                                                                                                  │
│ /opt/conda/lib/python3.10/site-packages/transformers/trainer.py:1938 in _inner_training_loop     │
│                                                                                                  │
│   1935 │   │   │   │   │   self.control = self.callback_handler.on_step_begin(args, self.state,  │
│   1936 │   │   │   │                                                                             │
│   1937 │   │   │   │   with self.accelerator.accumulate(model):                                  │
│ ❱ 1938 │   │   │   │   │   tr_loss_step = self.training_step(model, inputs)                      │
│   1939 │   │   │   │                                                                             │
│   1940 │   │   │   │   if (                                                                      │
│   1941 │   │   │   │   │   args.logging_nan_inf_filter                                           │
│                                                                                                  │
│ /opt/conda/lib/python3.10/site-packages/transformers/trainer.py:2770 in training_step            │
│                                                                                                  │
│   2767 │   │   │   with amp.scale_loss(loss, self.optimizer) as scaled_loss:                     │
│   2768 │   │   │   │   scaled_loss.backward()                                                    │
│   2769 │   │   else:                                                                             │
│ ❱ 2770 │   │   │   self.accelerator.backward(loss)                                               │
│   2771 │   │                                                                                     │
│   2772 │   │   return loss.detach() / self.args.gradient_accumulation_steps                      │
│   2773                                                                                           │
│                                                                                                  │
│ /opt/conda/lib/python3.10/site-packages/accelerate/accelerator.py:1815 in backward               │
│                                                                                                  │
│   1812 │   │   │   # deepspeed handles loss scaling by gradient_accumulation_steps in its `back  │
│   1813 │   │   │   loss = loss / self.gradient_accumulation_steps                                │
│   1814 │   │   if self.distributed_type == DistributedType.DEEPSPEED:                            │
│ ❱ 1815 │   │   │   self.deepspeed_engine_wrapped.backward(loss, **kwargs)                        │
│   1816 │   │   elif self.distributed_type == DistributedType.MEGATRON_LM:                        │
│   1817 │   │   │   return                                                                        │
│   1818 │   │   elif self.scaler is not None:                                                     │
│                                                                                                  │
│ /opt/conda/lib/python3.10/site-packages/accelerate/utils/deepspeed.py:176 in backward            │
│                                                                                                  │
│   173 │   │   # - zero grad                                                                      │
│   174 │   │   # - checking overflow                                                              │
│   175 │   │   # - lr_scheduler step (only if engine.lr_scheduler is not None)                    │
│ ❱ 176 │   │   self.engine.step()                                                                 │
│   177 │   │   # and this plugin overrides the above calls with no-ops when Accelerate runs und   │
│   178 │   │   # Deepspeed, but allows normal functionality for non-Deepspeed cases thus enabli   │
│   179 │   │   # training loop that works transparently under many training regimes.              │
│                                                                                                  │
│ /opt/conda/lib/python3.10/site-packages/deepspeed/runtime/engine.py:2184 in step                 │
│                                                                                                  │
│   2181 │   │   │   │   │   and self.quantizer.any_precision_switch()):                           │
│   2182 │   │   │   │   self._take_model_step(lr_kwargs, self.block_eigenvalue)                   │
│   2183 │   │   │   else:                                                                         │
│ ❱ 2184 │   │   │   │   self._take_model_step(lr_kwargs)                                          │
│   2185 │   │   │                                                                                 │
│   2186 │   │   │   report_progress = self.global_rank == 0 if self.global_rank else True         │
│   2187                                                                                           │
│                                                                                                  │
│ /opt/conda/lib/python3.10/site-packages/deepspeed/runtime/engine.py:2086 in _take_model_step     │
│                                                                                                  │
│   2083 │   │   │   │   clip_grad_norm_(parameters=master_params,                                 │
│   2084 │   │   │   │   │   │   │   │   max_norm=self.gradient_clipping(),                        │
│   2085 │   │   │   │   │   │   │   │   mpu=self.mpu)                                             │
│ ❱ 2086 │   │   self.optimizer.step()                                                             │
│   2087 │   │                                                                                     │
│   2088 │   │   if hasattr(self.optimizer, '_global_grad_norm'):                                  │
│   2089 │   │   │   self._global_grad_norm = self.optimizer._global_grad_norm                     │
│                                                                                                  │
│ /opt/conda/lib/python3.10/site-packages/deepspeed/runtime/zero/stage_1_and_2.py:1778 in step     │
│                                                                                                  │
│   1775 │   │   timer_names = [OPTIMIZER_ALLGATHER, OPTIMIZER_GRADIENTS, OPTIMIZER_STEP]          │
│   1776 │   │                                                                                     │
│   1777 │   │   prev_scale = self.loss_scale                                                      │
│ ❱ 1778 │   │   self._update_scale(self.overflow)                                                 │
│   1779 │   │   if self.overflow:                                                                 │
│   1780 │   │   │   see_memory_usage('After overflow before clearing gradients')                  │
│   1781 │   │   │   self.zero_grad(set_to_none=True)                                              │
│                                                                                                  │
│ /opt/conda/lib/python3.10/site-packages/deepspeed/runtime/zero/stage_1_and_2.py:2028 in          │
│ _update_scale                                                                                    │
│                                                                                                  │
│   2025 │   │   self._check_overflow(partition_gradients)                                         │
│   2026 │                                                                                         │
│   2027 │   def _update_scale(self, has_overflow=False):                                          │
│ ❱ 2028 │   │   self.loss_scaler.update_scale(has_overflow)                                       │
│   2029 │                                                                                         │
│   2030 │   # Promote state so it can be retrieved or set via "fp16_optimizer_instance.state"     │
│   2031 │   def _get_state(self):                                                                 │
│                                                                                                  │
│ /opt/conda/lib/python3.10/site-packages/deepspeed/runtime/fp16/loss_scaler.py:164 in             │
│ update_scale                                                                                     │
│                                                                                                  │
│   161 │   │   │   # self.cur_scale /= self.scale_factor                                          │
│   162 │   │   │   if self.delayed_shift == 1 or self.cur_hysteresis == 1:                        │
│   163 │   │   │   │   if (self.cur_scale == self.min_scale) and self.raise_error_at_min_scale:   │
│ ❱ 164 │   │   │   │   │   raise Exception(                                                       │
│   165 │   │   │   │   │   │   "Current loss scale already at minimum - cannot decrease scale a   │
│   166 │   │   │   │   │   )                                                                      │
│   167 │   │   │   │   else:                                                                      │
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
Exception: Current loss scale already at minimum - cannot decrease scale anymore. Exiting run.

# 请在此处粘贴依赖情况

# 请在此处粘贴运行日志

# 请在此处粘贴运行代码（如没有可删除该代码块）

# 请在此处粘贴依赖情况

# 请在此处粘贴运行日志

# 请在此处粘贴运行代码（如没有可删除该代码块）

# 请在此处粘贴依赖情况

# 请在此处粘贴运行日志

# 请在此处粘贴依赖情况

# 请在此处粘贴运行日志

peft                     0.4.0
torch                    2.0.1
transformers             4.31.0

[WARNING|logging.py:295] 2023-08-04 11:20:55,373 >> `use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...

[INFO|trainer.py:3081] 2023-08-04 11:44:30,657 >> ***** Running Evaluation *****
[INFO|trainer.py:3083] 2023-08-04 11:44:30,657 >>   Num examples = 299
[INFO|trainer.py:3086] 2023-08-04 11:44:30,658 >>   Batch size = 16
Traceback (most recent call last):
  File "/mnt/bn/fulei-v6-hl-nas-mlx/mlx/workspace/llm/llm_train/Chinese-LLaMA-Alpaca-2/scripts/training/run_clm_sft_with_peft.py", line 507, in <module>
    main()
  File "/mnt/bn/fulei-v6-hl-nas-mlx/mlx/workspace/llm/llm_train/Chinese-LLaMA-Alpaca-2/scripts/training/run_clm_sft_with_peft.py", line 480, in main
    train_result = trainer.train(resume_from_checkpoint=checkpoint)
  File "/root/miniforge3/envs/flash-atten2/lib/python3.10/site-packages/transformers/trainer.py", line 1539, in train
    return inner_training_loop(
  File "/root/miniforge3/envs/flash-atten2/lib/python3.10/site-packages/transformers/trainer.py", line 1901, in _inner_training_loop
    self._maybe_log_save_evaluate(tr_loss, model, trial, epoch, ignore_keys_for_eval)
  File "/root/miniforge3/envs/flash-atten2/lib/python3.10/site-packages/transformers/trainer.py", line 2226, in _maybe_log_save_evaluate
    metrics = self.evaluate(ignore_keys=ignore_keys_for_eval)
  File "/root/miniforge3/envs/flash-atten2/lib/python3.10/site-packages/transformers/trainer.py", line 2934, in evaluate
    output = eval_loop(
  File "/root/miniforge3/envs/flash-atten2/lib/python3.10/site-packages/transformers/trainer.py", line 3123, in evaluation_loop
    loss, logits, labels = self.prediction_step(model, inputs, prediction_loss_only, ignore_keys=ignore_keys)
  File "/root/miniforge3/envs/flash-atten2/lib/python3.10/site-packages/transformers/trainer.py", line 3337, in prediction_step
    loss, outputs = self.compute_loss(model, inputs, return_outputs=True)
  File "/root/miniforge3/envs/flash-atten2/lib/python3.10/site-packages/transformers/trainer.py", line 2679, in compute_loss
    outputs = model(**inputs)
  File "/root/miniforge3/envs/flash-atten2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1538, in _call_impl
    result = forward_call(*args, **kwargs)
  File "/root/miniforge3/envs/flash-atten2/lib/python3.10/site-packages/peft/peft_model.py", line 922, in forward
    return self.base_model(
  File "/root/miniforge3/envs/flash-atten2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
    return forward_call(*args, **kwargs)
  File "/root/miniforge3/envs/flash-atten2/lib/python3.10/site-packages/transformers/models/llama/modeling_llama.py", line 806, in forward
    outputs = self.model(
  File "/root/miniforge3/envs/flash-atten2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
    return forward_call(*args, **kwargs)
  File "/root/miniforge3/envs/flash-atten2/lib/python3.10/site-packages/transformers/models/llama/modeling_llama.py", line 693, in forward
    layer_outputs = decoder_layer(
  File "/root/miniforge3/envs/flash-atten2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
    return forward_call(*args, **kwargs)
  File "/root/miniforge3/envs/flash-atten2/lib/python3.10/site-packages/transformers/models/llama/modeling_llama.py", line 408, in forward
    hidden_states, self_attn_weights, present_key_value = self.self_attn(
  File "/root/miniforge3/envs/flash-atten2/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
    return forward_call(*args, **kwargs)
  File "/mnt/bn/fulei-v6-hl-nas-mlx/mlx/workspace/llm/llm_train/Chinese-LLaMA-Alpaca-2/scripts/training/flash_attn_patch.py", line 59, in forward
    assert not use_cache, "use_cache is not supported"
AssertionError: use_cache is not supported

 pip list | grep -E 'transformers|peft|torch'
peft                     0.4.0
torch                    2.0.1
transformers             4.31.0

[INFO|trainer.py:1686] 2023-08-01 08:21:52,370 >> ***** Running training *****
[INFO|trainer.py:1687] 2023-08-01 08:21:52,370 >>   Num examples = 8,328
[INFO|trainer.py:1688] 2023-08-01 08:21:52,371 >>   Num Epochs = 1
[INFO|trainer.py:1689] 2023-08-01 08:21:52,371 >>   Instantaneous batch size per device = 1
[INFO|trainer.py:1692] 2023-08-01 08:21:52,371 >>   Total train batch size (w. parallel, distributed & accumulation) = 2
[INFO|trainer.py:1693] 2023-08-01 08:21:52,371 >>   Gradient Accumulation steps = 2
[INFO|trainer.py:1694] 2023-08-01 08:21:52,371 >>   Total optimization steps = 4,164
[INFO|trainer.py:1695] 2023-08-01 08:21:52,372 >>   Number of trainable parameters = 159,907,840
  0%|                                                                                          | 0/4164 [00:00<?, ?it/s]Traceback (most recent call last):
  File "/home/maguoheng/Chinese-LLaMA-Alpaca-2/scripts/training/run_clm_pt_with_peft.py", line 636, in <module>
    main()
  File "/home/maguoheng/Chinese-LLaMA-Alpaca-2/scripts/training/run_clm_pt_with_peft.py", line 604, in main
    train_result = trainer.train(resume_from_checkpoint=checkpoint)
  File "/home/maguoheng/anaconda3/envs/chinese-llama2-alpaca/lib/python3.9/site-packages/transformers/trainer.py", line 1539, in train
    return inner_training_loop(
  File "/home/maguoheng/anaconda3/envs/chinese-llama2-alpaca/lib/python3.9/site-packages/transformers/trainer.py", line 1809, in _inner_training_loop
    tr_loss_step = self.training_step(model, inputs)
  File "/home/maguoheng/anaconda3/envs/chinese-llama2-alpaca/lib/python3.9/site-packages/transformers/trainer.py", line 2654, in training_step
    loss = self.compute_loss(model, inputs)
  File "/home/maguoheng/anaconda3/envs/chinese-llama2-alpaca/lib/python3.9/site-packages/transformers/trainer.py", line 2679, in compute_loss
    outputs = model(**inputs)
  File "/home/maguoheng/anaconda3/envs/chinese-llama2-alpaca/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
    return forward_call(*args, **kwargs)
  File "/home/maguoheng/anaconda3/envs/chinese-llama2-alpaca/lib/python3.9/site-packages/deepspeed/utils/nvtx.py", line 15, in wrapped_fn
    ret_val = func(*args, **kwargs)
  File "/home/maguoheng/anaconda3/envs/chinese-llama2-alpaca/lib/python3.9/site-packages/deepspeed/runtime/engine.py", line 1769, in forward
    loss = self.module(*inputs, **kwargs)
  File "/home/maguoheng/anaconda3/envs/chinese-llama2-alpaca/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
    return forward_call(*args, **kwargs)
  File "/home/maguoheng/anaconda3/envs/chinese-llama2-alpaca/lib/python3.9/site-packages/peft/peft_model.py", line 922, in forward
    return self.base_model(
  File "/home/maguoheng/anaconda3/envs/chinese-llama2-alpaca/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
    return forward_call(*args, **kwargs)
  File "/home/maguoheng/anaconda3/envs/chinese-llama2-alpaca/lib/python3.9/site-packages/transformers/models/llama/modeling_llama.py", line 806, in forward
    outputs = self.model(
  File "/home/maguoheng/anaconda3/envs/chinese-llama2-alpaca/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
    return forward_call(*args, **kwargs)
  File "/home/maguoheng/anaconda3/envs/chinese-llama2-alpaca/lib/python3.9/site-packages/transformers/models/llama/modeling_llama.py", line 693, in forward
    layer_outputs = decoder_layer(
  File "/home/maguoheng/anaconda3/envs/chinese-llama2-alpaca/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
    return forward_call(*args, **kwargs)
  File "/home/maguoheng/anaconda3/envs/chinese-llama2-alpaca/lib/python3.9/site-packages/transformers/models/llama/modeling_llama.py", line 408, in forward
    hidden_states, self_attn_weights, present_key_value = self.self_attn(
  File "/home/maguoheng/anaconda3/envs/chinese-llama2-alpaca/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
    return forward_call(*args, **kwargs)
  File "/home/maguoheng/anaconda3/envs/chinese-llama2-alpaca/lib/python3.9/site-packages/transformers/models/llama/modeling_llama.py", line 330, in forward
    attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim)
torch.cuda.OutOfMemoryError: CUDA out of memory. Tried to allocate 64.00 MiB (GPU 0; 23.99 GiB total capacity; 22.70 GiB already allocated; 0 bytes free; 22.88 GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF
  0%|                                                                                          | 0/4164 [00:01<?, ?it/s]
ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 28969) of binary: /home/maguoheng/anaconda3/envs/chinese-llama2-alpaca/bin/python
Traceback (most recent call last):
  File "/home/maguoheng/anaconda3/envs/chinese-llama2-alpaca/bin/torchrun", line 8, in <module>
    sys.exit(main())
  File "/home/maguoheng/anaconda3/envs/chinese-llama2-alpaca/lib/python3.9/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper
    return f(*args, **kwargs)
  File "/home/maguoheng/anaconda3/envs/chinese-llama2-alpaca/lib/python3.9/site-packages/torch/distributed/run.py", line 794, in main
    run(args)
  File "/home/maguoheng/anaconda3/envs/chinese-llama2-alpaca/lib/python3.9/site-packages/torch/distributed/run.py", line 785, in run
    elastic_launch(
  File "/home/maguoheng/anaconda3/envs/chinese-llama2-alpaca/lib/python3.9/site-packages/torch/distributed/launcher/api.py", line 134, in __call__
    return launch_agent(self._config, self._entrypoint, list(args))
  File "/home/maguoheng/anaconda3/envs/chinese-llama2-alpaca/lib/python3.9/site-packages/torch/distributed/launcher/api.py", line 250, in launch_agent
    raise ChildFailedError(
torch.distributed.elastic.multiprocessing.errors.ChildFailedError:
============================================================
run_clm_pt_with_peft.py FAILED
------------------------------------------------------------
Failures:
  <NO_OTHER_FAILURES>
------------------------------------------------------------
Root Cause (first observed failure):
[0]:
  time      : 2023-08-01_08:21:58
  host      : MICROSO-AR91S8Q.
  rank      : 0 (local_rank: 0)
  exitcode  : 1 (pid: 28969)
  error_file: <N/A>
  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
============================================================

1.在预训练完成后，模型在oobabooga进行推理的时候，模型会出现 自问自答的情况。是模型训练，文本导致的还是，oobabooga设置导致的。
2.在某次合并后出现正常操作。  但是在回复的时候经常会出现说一半的情况。

已经训练合并完成  模型训练用的  llama2-alpaca 进行训练，  合并 模型 model 指向 llama2-alpaca 模型进行合并。

# 请在此处粘贴运行日志

# 请在此处粘贴运行代码（如没有可删除该代码块）

python gradio_demo.py --base_model C:\Users\aa\Downloads\ziqingyang-chinese-alpaca-2-7b --lora_model C:\Users\aa\Downloads\ziqingyang-chinese-alpaca-2-lora-7b --tokenizer_path C:\Users\aa\Downloads\ziqingyang-chinese-alpaca-2-lora-7b --load_in_8bit

# 请在此处粘贴依赖情况

# 请在此处粘贴运行代码（如没有可删除该代码块）

# 请在此处粘贴依赖情况

# 请在此处粘贴运行日志

SYSTEM_PROMPT = """You are a helpful assistant. 你是一个乐于助人的助手。"""

PROMPT_TEMPLATE = (
    "[INST] <<SYS>>\n"
    "{system_prompt}\n"
    "<</SYS>>\n\n"
    "{instruction} [/INST]"
)

full_prompt = PROMPT_TEMPLATE.format_map({"instruction": your_instruction, "system_prompt": SYSTEM_PROMPT})

########参数部分########
lr=1e-4
lora_rank=64
lora_alpha=128
lora_trainable="q_proj,v_proj,k_proj,o_proj,gate_proj,down_proj,up_proj"
modules_to_save="embed_tokens,lm_head"
lora_dropout=0.05

pretrained_model=/root/.cache/huggingface/chinese-alpaca-2-7b-hf
chinese_tokenizer_path=/root/.cache/huggingface/chinese-alpaca-2-7b-hf
dataset_dir=/root/.cache/huggingface/data/merge.json
per_device_train_batch_size=1
per_device_eval_batch_size=1
gradient_accumulation_steps=1
output_dir=output_dir
validation_file=/root/.cache/huggingface/data/merge.json
max_seq_length=1024

deepspeed_config_file=ds_zero2_no_offload.json

########启动命令########
torchrun --nnodes 1 --nproc_per_node 4 run_clm_sft_with_peft.py \
    --deepspeed ${deepspeed_config_file} \
    --model_name_or_path ${pretrained_model} \
    --tokenizer_name_or_path ${chinese_tokenizer_path} \
    --dataset_dir ${dataset_dir} \
    --validation_split_percentage 0.001 \
    --per_device_train_batch_size ${per_device_train_batch_size} \
    --per_device_eval_batch_size ${per_device_eval_batch_size} \
    --do_train \
    --do_eval \
    --seed $RANDOM \
    --fp16 \
    --num_train_epochs 2 \
    --lr_scheduler_type cosine \
    --learning_rate ${lr} \
    --warmup_ratio 0.03 \
    --weight_decay 0 \
    --logging_strategy steps \
    --logging_steps 10 \
    --save_strategy steps \
    --save_total_limit 3 \
    --evaluation_strategy steps \
    --eval_steps 250 \
    --save_steps 500 \
    --gradient_accumulation_steps ${gradient_accumulation_steps} \
    --preprocessing_num_workers 8 \
    --max_seq_length ${max_seq_length} \
    --output_dir ${output_dir} \
    --overwrite_output_dir \
    --ddp_timeout 30000 \
    --logging_first_step True \
    --lora_rank ${lora_rank} \
    --lora_alpha ${lora_alpha} \
    --trainable ${lora_trainable} \
    --modules_to_save ${modules_to_save} \
    --lora_dropout ${lora_dropout} \
    --torch_dtype float16 \
    --validation_file ${validation_file} \
    --gradient_checkpointing \
    --ddp_find_unused_parameters False

peft                     0.3.0.dev0
torch                    2.0.1
transformers             4.31.0

(llama2) root@eb03b13bd90d:~/Chinese-LLaMA-Alpaca-2/scripts/training# bash run_sft.sh
WARNING:torch.distributed.run:
*****************************************
Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
*****************************************
[2023-08-03 08:13:01,087] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2023-08-03 08:13:01,106] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)
[2023-08-03 08:13:01,202] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented
[2023-08-03 08:13:01,202] [INFO] [comm.py:616:init_distributed] cdb=None
[2023-08-03 08:13:01,202] [INFO] [comm.py:643:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl
[2023-08-03 08:13:01,219] [WARNING] [comm.py:152:init_deepspeed_backend] NCCL backend in DeepSpeed not yet implemented
[2023-08-03 08:13:01,219] [INFO] [comm.py:616:init_distributed] cdb=None
08/03/2023 08:13:01 - WARNING - __main__ - Process rank: 1, device: cuda:1, n_gpu: 1distributed training: True, 16-bits training: True
[WARNING|logging.py:295] 2023-08-03 08:13:01,415 >> You are using the legacy behaviour of the <class 'transformers.models.llama.tokenization_llama.LlamaTokenizer'>. This means that tokens that come after special tokens will not be properly handled. We recommend you to read the related pull request available at https://github.com/huggingface/transformers/pull/24565
08/03/2023 08:13:01 - WARNING - __main__ - Process rank: 0, device: cuda:0, n_gpu: 1distributed training: True, 16-bits training: True
[INFO|configuration_utils.py:710] 2023-08-03 08:13:01,818 >> loading configuration file /root/.cache/huggingface/chinese-alpaca-2-7b-hf/config.json
[INFO|configuration_utils.py:768] 2023-08-03 08:13:01,818 >> Model config LlamaConfig {
  "_name_or_path": "/root/.cache/huggingface/chinese-alpaca-2-7b-hf",
  "architectures": [
    "LlamaForCausalLM"
  ],
  "bos_token_id": 1,
  "eos_token_id": 2,
  "hidden_act": "silu",
  "hidden_size": 4096,
  "initializer_range": 0.02,
  "intermediate_size": 11008,
  "max_length": 4096,
  "max_position_embeddings": 4096,
  "model_type": "llama",
  "num_attention_heads": 32,
  "num_hidden_layers": 32,
  "num_key_value_heads": 32,
  "pad_token_id": 0,
  "pretraining_tp": 1,
  "rms_norm_eps": 1e-05,
  "rope_scaling": null,
  "tie_word_embeddings": false,
  "torch_dtype": "float16",
  "transformers_version": "4.31.0",
  "use_cache": true,
  "vocab_size": 55296
}

[INFO|tokenization_utils_base.py:1837] 2023-08-03 08:13:01,818 >> loading file tokenizer.model
[INFO|tokenization_utils_base.py:1837] 2023-08-03 08:13:01,818 >> loading file added_tokens.json
[INFO|tokenization_utils_base.py:1837] 2023-08-03 08:13:01,818 >> loading file special_tokens_map.json
[INFO|tokenization_utils_base.py:1837] 2023-08-03 08:13:01,818 >> loading file tokenizer_config.json
[WARNING|logging.py:295] 2023-08-03 08:13:01,819 >> You are using the legacy behaviour of the <class 'transformers.models.llama.tokenization_llama.LlamaTokenizer'>. This means that tokens that come after special tokens will not be properly handled. We recommend you to read the related pull request available at https://github.com/huggingface/transformers/pull/24565
08/03/2023 08:13:01 - INFO - __main__ - Training files: 
08/03/2023 08:13:01 - WARNING - root - building dataset...
ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: -7) local_rank: 0 (pid: 1280) of binary: /root/miniconda3/envs/llama2/bin/python
Traceback (most recent call last):
  File "/root/miniconda3/envs/llama2/bin/torchrun", line 8, in <module>
    sys.exit(main())
  File "/root/miniconda3/envs/llama2/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper
    return f(*args, **kwargs)
  File "/root/miniconda3/envs/llama2/lib/python3.10/site-packages/torch/distributed/run.py", line 794, in main
    run(args)
  File "/root/miniconda3/envs/llama2/lib/python3.10/site-packages/torch/distributed/run.py", line 785, in run
    elastic_launch(
  File "/root/miniconda3/envs/llama2/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 134, in __call__
    return launch_agent(self._config, self._entrypoint, list(args))
  File "/root/miniconda3/envs/llama2/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 250, in launch_agent
    raise ChildFailedError(
torch.distributed.elastic.multiprocessing.errors.ChildFailedError: 
====================================================
run_clm_sft_with_peft.py FAILED
----------------------------------------------------
Failures:
[1]:
  time      : 2023-08-03_08:13:04
  host      : eb03b13bd90d
  rank      : 1 (local_rank: 1)
  exitcode  : -7 (pid: 1281)
  error_file: <N/A>
  traceback : Signal 7 (SIGBUS) received by PID 1281
----------------------------------------------------
Root Cause (first observed failure):
[0]:
  time      : 2023-08-03_08:13:04
  host      : eb03b13bd90d
  rank      : 0 (local_rank: 0)
  exitcode  : -7 (pid: 1280)
  error_file: <N/A>
  traceback : Signal 7 (SIGBUS) received by PID 1280
====================================================

# 请在此处粘贴依赖情况

{
  "architectures": [
    "LlamaForCausalLM"
  ],
  "bos_token_id": 1,
  "eos_token_id": 2,
  "hidden_act": "silu",
  "hidden_size": 4096,
  "initializer_range": 0.02,
  "intermediate_size": 11008,
  "max_position_embeddings": 4096,
  "model_type": "llama",
  "num_attention_heads": 32,
  "num_hidden_layers": 32,
  "num_key_value_heads": 32,
  "pad_token_id": 0,
  "pretraining_tp": 1,
  "rms_norm_eps": 1e-05,
  "rope_scaling": null,
  "tie_word_embeddings": false,
  "torch_dtype": "float16",
  "transformers_version": "4.31.0",
  "use_cache": true,
  "vocab_size": 55296
}
{
  "_from_model_config": true,
  "bos_token_id": 1,
  "eos_token_id": 2,
  "pad_token_id": 0,
  "transformers_version": "4.31.0"
}

{
  "architectures": [
    "LlamaForCausalLM"
  ],
  "bos_token_id": 1,
  "eos_token_id": 2,
  "hidden_act": "silu",
  "hidden_size": 4096,
  "initializer_range": 0.02,
  "intermediate_size": 11008,
  "max_position_embeddings": 4096,
  "model_type": "llama",
  "num_attention_heads": 32,
  "num_hidden_layers": 32,
  "num_key_value_heads": 32,
  "pad_token_id": 0,
  "pretraining_tp": 1,
  "rms_norm_eps": 1e-05,
  "rope_scaling": null,
  "tie_word_embeddings": false,
  "torch_dtype": "float16",
  "transformers_version": "4.31.0",
  "use_cache": true,
  "vocab_size": 55296
}
{
"_from_model_config": true,
"bos_token_id": 1,
"eos_token_id": 2,
"pad_token_id": 0,
"transformers_version": "4.31.0"
}