diff --git a/dev/eval/export_hf.py b/dev/eval/export_hf.py new file mode 100644 index 0000000..d9eb4a8 --- /dev/null +++ b/dev/eval/export_hf.py @@ -0,0 +1,171 @@ +""" +Script to convert GPT2 models from llm.c binary format to Hugging Face + +It can optinally upload to your account on Hugging Face if you have the CLI: + pip install -U "huggingface_hub[cli]" + huggingface-cli login + +Export to a local HF model: + python export_hf.py --input input_file.bin --output model_name + +Export to a local HF model and also push to your account on Hugging Face: + python export_hf.py --input input_file.bin --output model_name --push true +""" + +import numpy as np +import torch +import argparse, sys +from transformers import GPT2Config, GPT2Tokenizer, GPT2LMHeadModel + +# ----------------------------------------------------------------------------- +# Tensor functions for both bfloat16 (from int16) and normal float32 +# Both return float32 tensors + +def tensor_bf16(data_int16, transpose=False): + if transpose: + data_int16 = data_int16.transpose(1,0) + return torch.tensor(data_int16).view(torch.bfloat16).to(torch.float32) + +def tensor_fp32(data_float32, transpose=False): + if transpose: + data_float32 = data_float32.transpose(1,0) + return torch.tensor(data_float32).view(torch.float32) + +# ----------------------------------------------------------------------------- +# Main conversion function + +def convert(filepath, output, push_to_hub=False, out_dtype="bfloat16"): + print(f"Converting model {filepath} to {output} in {out_dtype} format and pushing to Hugging Face: {push_to_hub}") + + f = open(filepath, 'rb') + # Read in our header, checking the magic number and version + # version 3 = fp32, padded vocab + # version 5 = bf16, padded vocab + model_header = np.frombuffer(f.read(256*4), dtype=np.int32) + if model_header[0] != 20240326: + print("ERROR: magic number mismatch in the data .bin file!") + exit(1) + version = model_header[1] + if not version in [3, 5]: + print("Bad version in model file") + exit(1) + + # Load in our model parameters + maxT = model_header[2].item() # max sequence length + V = model_header[3].item() # vocab size + L = model_header[4].item() # num layers + H = model_header[5].item() # num heads + C = model_header[6].item() # channels + Vp = model_header[7].item() # padded vocab size + + print(f"{version=}, {maxT=}, {V=}, {Vp=}, {L=}, {H=}, {C=}") + + # Define the shapes of our parameters + shapes = { + 'wte': (Vp, C), + 'wpe': (maxT, C), + 'ln1w': (L, C), + 'ln1b': (L, C), + 'qkvw': (L, 3 * C, C), + 'qkvb': (L, 3 * C), + 'attprojw': (L, C, C), + 'attprojb': (L, C), + 'ln2w': (L, C), + 'ln2b': (L, C), + 'fcw': (L, 4 * C, C), + 'fcb': (L, 4 * C), + 'fcprojw': (L, C, 4 * C), + 'fcprojb': (L, C), + 'lnfw': (C,), + 'lnfb': (C,), + } + + # Load in our weights given our parameter shapes + dtype = np.float32 if version == 3 else np.int16 + w = {} + for key, shape in shapes.items(): + num_elements = np.prod(shape) + data = np.frombuffer(f.read(num_elements * np.dtype(dtype).itemsize), dtype=dtype) + w[key] = data.reshape(shape) + # The binary file saves the padded vocab - drop the padding back to GPT2 size + if shape[0] == Vp: + w[key] = w[key].reshape(shape)[:(V-Vp), :] + # Ensure the file is fully read and then close + assert f.read() == b'' + f.close() + + # Map to our model dict, the tensors at this stage are always fp32 + mk_tensor = { + 3 : tensor_fp32, + 5 : tensor_bf16, + }[version] + model_dict = {} + model_dict['transformer.wte.weight'] = mk_tensor(w['wte']) + model_dict['transformer.wpe.weight'] = mk_tensor(w['wpe']) + model_dict['lm_head.weight'] = model_dict['transformer.wte.weight'] # Tie weights + for i in range(L): + model_dict[f'transformer.h.{i}.ln_1.weight'] = mk_tensor(w['ln1w'][i]) + model_dict[f'transformer.h.{i}.ln_1.bias'] = mk_tensor(w['ln1b'][i]) + model_dict[f'transformer.h.{i}.attn.c_attn.weight'] = mk_tensor(w['qkvw'][i], True) + model_dict[f'transformer.h.{i}.attn.c_attn.bias'] = mk_tensor(w['qkvb'][i]) + model_dict[f'transformer.h.{i}.attn.c_proj.weight'] = mk_tensor(w['attprojw'][i], True) + model_dict[f'transformer.h.{i}.attn.c_proj.bias'] = mk_tensor(w['attprojb'][i]) + model_dict[f'transformer.h.{i}.ln_2.weight'] = mk_tensor(w['ln2w'][i]) + model_dict[f'transformer.h.{i}.ln_2.bias'] = mk_tensor(w['ln2b'][i]) + model_dict[f'transformer.h.{i}.mlp.c_fc.weight'] = mk_tensor(w['fcw'][i], True) + model_dict[f'transformer.h.{i}.mlp.c_fc.bias'] = mk_tensor(w['fcb'][i]) + model_dict[f'transformer.h.{i}.mlp.c_proj.weight'] = mk_tensor(w['fcprojw'][i], True) + model_dict[f'transformer.h.{i}.mlp.c_proj.bias'] = mk_tensor(w['fcprojb'][i]) + model_dict['transformer.ln_f.weight'] = mk_tensor(w['lnfw']) + model_dict['transformer.ln_f.bias'] = mk_tensor(w['lnfb']) + + # Create a GPT-2 model instance, in the requested dtype + config = GPT2Config(vocab_size = V, + n_positions = maxT, + n_ctx = maxT, + n_embd = C, + n_layer = L, + n_head = H) + model = GPT2LMHeadModel(config) + if out_dtype == "bfloat16": + model = model.to(torch.bfloat16) + + # Set the model dict and save + model.load_state_dict(model_dict) + model.save_pretrained(output, max_shard_size="5GB", safe_serialization=True) + + # Copy over a standard gpt2 tokenizer + tokenizer = GPT2Tokenizer.from_pretrained("gpt2") + tokenizer.save_pretrained(output) + + if push_to_hub: + print(f"Uploading {output} to Hugging Face") + model.push_to_hub(output) + tokenizer.push_to_hub(output) + +def spin(output): + print("Taking the exported model for a spin...") + print('-'*80) + from transformers import AutoModelForCausalLM, AutoTokenizer + tokenizer = AutoTokenizer.from_pretrained(output) + model = AutoModelForCausalLM.from_pretrained(output) + tokens = tokenizer.encode("During photosynthesis in green plants", return_tensors="pt") + output = model.generate(tokens, max_new_tokens=64, repetition_penalty=1.3) + samples = tokenizer.batch_decode(output) + for sample in samples: + print('-'*30) + print(sample) + +# ----------------------------------------------------------------------------- + +if __name__== '__main__': + parser=argparse.ArgumentParser() + parser.add_argument("--input", "-i", help="The name of the llm.c model.bin file", type=str, required=True) + parser.add_argument("--output","-o", help="The Hugging Face output model directory", type=str, required=True) + parser.add_argument("--dtype", "-d", help="Output as either float32 or bfloat16 (default)", type=str, default="bfloat16") + parser.add_argument("--push", "-p", help="Push the model to your Hugging Face account", type=bool, default=False) + parser.add_argument("--spin", "-s", help="Take the model for a spin at the end?", type=bool, default=True) + args = parser.parse_args() + convert(args.input, args.output, args.push, args.dtype) + if args.spin: + spin(args.output) diff --git a/dev/eval/run_eval.sh b/dev/eval/run_eval.sh new file mode 100755 index 0000000..d1e28f6 --- /dev/null +++ b/dev/eval/run_eval.sh @@ -0,0 +1,52 @@ +# https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard +# (See About tab -> REPRODUCIBILITY) + +# This script is intended to be run from the parent/root directory of llm.c repo. + +# Clone the evaluation harness: + +# git clone https://github.com/EleutherAI/lm-evaluation-harness/ +# cd lm-evaluation-harness +# git checkout b281b0921b636bc36ad05c0b0b0763bd6dd43463 +# pip install -e . + +# Then return to the parent directory and run this script + +# cd .. +# ./dev/eval/run_eval.sh [model_name] [result_name] + +# where model_name is either a HF model such as openai-community/gpt2 or a local path such as ./gpt2-124M-run1 +# and result_name is the name of the folder under lm-evaluation-harness/results to store the evaluations + +# Since the evals can take a couple of hours to run, depending on the model size, you may wish to +# run within a "screen" session or by using nohup to run the script: + +# nohup ./dev/eval/run_eval.sh [model_name] [result_name] > run.txt 2> err.txt & + +if [ -z "$1" ]; then + echo "Error: missing HuggingFace model name or path to local model" + echo "./run_eval.sh hf_account/model_name my_result" + exit 1 +fi +if [ -z "$2" ]; then + echo "Error: missing output name for results" + echo "./run_eval.sh hf_account/model_name my_result" + exit 1 +fi + +export MODEL="$(realpath -s "$1")" +export RESULT="$2" +echo "Evaluating model $MODEL" +echo "Saving results to ./lm-evaluation-harness/results/$RESULT" + +cd lm-evaluation-harness + +python main.py --model hf-causal-experimental --model_args pretrained=$MODEL,use_accelerate=True,trust_remote_code=True --tasks truthfulqa_mc --batch_size 1 --no_cache --write_out --output_path results/$RESULT/truthfulqa_0shot.json --device cuda +python main.py --model hf-causal-experimental --model_args pretrained=$MODEL,use_accelerate=True,trust_remote_code=True --tasks winogrande --batch_size 1 --no_cache --write_out --output_path results/$RESULT/winogrande_5shot.json --device cuda --num_fewshot 5 +python main.py --model hf-causal-experimental --model_args pretrained=$MODEL,use_accelerate=True,trust_remote_code=True --tasks arc_challenge --batch_size 1 --no_cache --write_out --output_path results/$RESULT/arc_challenge_25shot.json --device cuda --num_fewshot 25 +python main.py --model hf-causal-experimental --model_args pretrained=$MODEL,use_accelerate=True,trust_remote_code=True --tasks hellaswag --batch_size 1 --no_cache --write_out --output_path results/$RESULT/hellaswag_10shot.json --device cuda --num_fewshot 10 +python main.py --model hf-causal-experimental --model_args pretrained=$MODEL,use_accelerate=True,trust_remote_code=True --tasks gsm8k --batch_size 1 --no_cache --write_out --output_path results/$RESULT/gsm8k_5shot.json --device cuda --num_fewshot 5 +python main.py --model hf-causal-experimental --model_args pretrained=$MODEL,use_accelerate=True,trust_remote_code=True --tasks hendrycksTest-abstract_algebra,hendrycksTest-anatomy,hendrycksTest-astronomy,hendrycksTest-business_ethics,hendrycksTest-clinical_knowledge,hendrycksTest-college_biology,hendrycksTest-college_chemistry,hendrycksTest-college_computer_science,hendrycksTest-college_mathematics,hendrycksTest-college_medicine,hendrycksTest-college_physics,hendrycksTest-computer_security,hendrycksTest-conceptual_physics,hendrycksTest-econometrics,hendrycksTest-electrical_engineering,hendrycksTest-elementary_mathematics,hendrycksTest-formal_logic,hendrycksTest-global_facts,hendrycksTest-high_school_biology,hendrycksTest-high_school_chemistry,hendrycksTest-high_school_computer_science,hendrycksTest-high_school_european_history,hendrycksTest-high_school_geography,hendrycksTest-high_school_government_and_politics,hendrycksTest-high_school_macroeconomics,hendrycksTest-high_school_mathematics,hendrycksTest-high_school_microeconomics,hendrycksTest-high_school_physics,hendrycksTest-high_school_psychology,hendrycksTest-high_school_statistics,hendrycksTest-high_school_us_history,hendrycksTest-high_school_world_history,hendrycksTest-human_aging,hendrycksTest-human_sexuality,hendrycksTest-international_law,hendrycksTest-jurisprudence,hendrycksTest-logical_fallacies,hendrycksTest-machine_learning,hendrycksTest-management,hendrycksTest-marketing,hendrycksTest-medical_genetics,hendrycksTest-miscellaneous,hendrycksTest-moral_disputes,hendrycksTest-moral_scenarios,hendrycksTest-nutrition,hendrycksTest-philosophy,hendrycksTest-prehistory,hendrycksTest-professional_accounting,hendrycksTest-professional_law,hendrycksTest-professional_medicine,hendrycksTest-professional_psychology,hendrycksTest-public_relations,hendrycksTest-security_studies,hendrycksTest-sociology,hendrycksTest-us_foreign_policy,hendrycksTest-virology,hendrycksTest-world_religions --batch_size 1 --no_cache --write_out --output_path results/$RESULT/mmlu_5shot.json --device cuda --num_fewshot 5 + +cd .. +python dev/eval/summarize_eval.py lm-evaluation-harness/results/$RESULT diff --git a/dev/eval/summarize_eval.py b/dev/eval/summarize_eval.py new file mode 100644 index 0000000..44471eb --- /dev/null +++ b/dev/eval/summarize_eval.py @@ -0,0 +1,27 @@ +import json, sys + +RESULT = sys.argv[1] +print("-"*40) + +key = {"arc_challenge_25shot.json": "acc_norm", + "gsm8k_5shot.json": "acc", + "hellaswag_10shot.json": "acc_norm", + "mmlu_5shot.json": "acc", + "truthfulqa_0shot.json": "mc2", + "winogrande_5shot.json": "acc" + } + +total = 0 +for test in ["arc_challenge_25shot.json", "gsm8k_5shot.json", "hellaswag_10shot.json", "mmlu_5shot.json", "truthfulqa_0shot.json", "winogrande_5shot.json"]: + data = json.loads(open("./%s/%s"%(RESULT, test)).read()) + r_count = 0 + r_total = 0 + for test_name in data['results']: + r_count += 1 + r_total += data['results'][test_name][key[test]] + score = (r_total*100)/r_count + print(f"{test:<30} : {score:.4f}") + total += score +average = total / 6.0 +print("-"*40) +print(f"Average Score : {average:.4f}")