mirror of
https://github.com/karpathy/llm.c.git
synced 2026-07-28 20:35:09 -04:00
add scripts to export to HF and run Eleuther evals
This commit is contained in:
Andrej Karpathy
parent
0e69e3ae46
commit
d7ea4da298
3 changed files with 250 additions and 0 deletions
171
dev/eval/export_hf.py
Normal file
171
dev/eval/export_hf.py
Normal file
|
|
@ -0,0 +1,171 @@
|
|||
"""
|
||||
Script to convert GPT2 models from llm.c binary format to Hugging Face
|
||||
|
||||
It can optinally upload to your account on Hugging Face if you have the CLI:
|
||||
pip install -U "huggingface_hub[cli]"
|
||||
huggingface-cli login
|
||||
|
||||
Export to a local HF model:
|
||||
python export_hf.py --input input_file.bin --output model_name
|
||||
|
||||
Export to a local HF model and also push to your account on Hugging Face:
|
||||
python export_hf.py --input input_file.bin --output model_name --push true
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import argparse, sys
|
||||
from transformers import GPT2Config, GPT2Tokenizer, GPT2LMHeadModel
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Tensor functions for both bfloat16 (from int16) and normal float32
|
||||
# Both return float32 tensors
|
||||
|
||||
def tensor_bf16(data_int16, transpose=False):
|
||||
if transpose:
|
||||
data_int16 = data_int16.transpose(1,0)
|
||||
return torch.tensor(data_int16).view(torch.bfloat16).to(torch.float32)
|
||||
|
||||
def tensor_fp32(data_float32, transpose=False):
|
||||
if transpose:
|
||||
data_float32 = data_float32.transpose(1,0)
|
||||
return torch.tensor(data_float32).view(torch.float32)
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Main conversion function
|
||||
|
||||
def convert(filepath, output, push_to_hub=False, out_dtype="bfloat16"):
|
||||
print(f"Converting model {filepath} to {output} in {out_dtype} format and pushing to Hugging Face: {push_to_hub}")
|
||||
|
||||
f = open(filepath, 'rb')
|
||||
# Read in our header, checking the magic number and version
|
||||
# version 3 = fp32, padded vocab
|
||||
# version 5 = bf16, padded vocab
|
||||
model_header = np.frombuffer(f.read(256*4), dtype=np.int32)
|
||||
if model_header[0] != 20240326:
|
||||
print("ERROR: magic number mismatch in the data .bin file!")
|
||||
exit(1)
|
||||
version = model_header[1]
|
||||
if not version in [3, 5]:
|
||||
print("Bad version in model file")
|
||||
exit(1)
|
||||
|
||||
# Load in our model parameters
|
||||
maxT = model_header[2].item() # max sequence length
|
||||
V = model_header[3].item() # vocab size
|
||||
L = model_header[4].item() # num layers
|
||||
H = model_header[5].item() # num heads
|
||||
C = model_header[6].item() # channels
|
||||
Vp = model_header[7].item() # padded vocab size
|
||||
|
||||
print(f"{version=}, {maxT=}, {V=}, {Vp=}, {L=}, {H=}, {C=}")
|
||||
|
||||
# Define the shapes of our parameters
|
||||
shapes = {
|
||||
'wte': (Vp, C),
|
||||
'wpe': (maxT, C),
|
||||
'ln1w': (L, C),
|
||||
'ln1b': (L, C),
|
||||
'qkvw': (L, 3 * C, C),
|
||||
'qkvb': (L, 3 * C),
|
||||
'attprojw': (L, C, C),
|
||||
'attprojb': (L, C),
|
||||
'ln2w': (L, C),
|
||||
'ln2b': (L, C),
|
||||
'fcw': (L, 4 * C, C),
|
||||
'fcb': (L, 4 * C),
|
||||
'fcprojw': (L, C, 4 * C),
|
||||
'fcprojb': (L, C),
|
||||
'lnfw': (C,),
|
||||
'lnfb': (C,),
|
||||
}
|
||||
|
||||
# Load in our weights given our parameter shapes
|
||||
dtype = np.float32 if version == 3 else np.int16
|
||||
w = {}
|
||||
for key, shape in shapes.items():
|
||||
num_elements = np.prod(shape)
|
||||
data = np.frombuffer(f.read(num_elements * np.dtype(dtype).itemsize), dtype=dtype)
|
||||
w[key] = data.reshape(shape)
|
||||
# The binary file saves the padded vocab - drop the padding back to GPT2 size
|
||||
if shape[0] == Vp:
|
||||
w[key] = w[key].reshape(shape)[:(V-Vp), :]
|
||||
# Ensure the file is fully read and then close
|
||||
assert f.read() == b''
|
||||
f.close()
|
||||
|
||||
# Map to our model dict, the tensors at this stage are always fp32
|
||||
mk_tensor = {
|
||||
3 : tensor_fp32,
|
||||
5 : tensor_bf16,
|
||||
}[version]
|
||||
model_dict = {}
|
||||
model_dict['transformer.wte.weight'] = mk_tensor(w['wte'])
|
||||
model_dict['transformer.wpe.weight'] = mk_tensor(w['wpe'])
|
||||
model_dict['lm_head.weight'] = model_dict['transformer.wte.weight'] # Tie weights
|
||||
for i in range(L):
|
||||
model_dict[f'transformer.h.{i}.ln_1.weight'] = mk_tensor(w['ln1w'][i])
|
||||
model_dict[f'transformer.h.{i}.ln_1.bias'] = mk_tensor(w['ln1b'][i])
|
||||
model_dict[f'transformer.h.{i}.attn.c_attn.weight'] = mk_tensor(w['qkvw'][i], True)
|
||||
model_dict[f'transformer.h.{i}.attn.c_attn.bias'] = mk_tensor(w['qkvb'][i])
|
||||
model_dict[f'transformer.h.{i}.attn.c_proj.weight'] = mk_tensor(w['attprojw'][i], True)
|
||||
model_dict[f'transformer.h.{i}.attn.c_proj.bias'] = mk_tensor(w['attprojb'][i])
|
||||
model_dict[f'transformer.h.{i}.ln_2.weight'] = mk_tensor(w['ln2w'][i])
|
||||
model_dict[f'transformer.h.{i}.ln_2.bias'] = mk_tensor(w['ln2b'][i])
|
||||
model_dict[f'transformer.h.{i}.mlp.c_fc.weight'] = mk_tensor(w['fcw'][i], True)
|
||||
model_dict[f'transformer.h.{i}.mlp.c_fc.bias'] = mk_tensor(w['fcb'][i])
|
||||
model_dict[f'transformer.h.{i}.mlp.c_proj.weight'] = mk_tensor(w['fcprojw'][i], True)
|
||||
model_dict[f'transformer.h.{i}.mlp.c_proj.bias'] = mk_tensor(w['fcprojb'][i])
|
||||
model_dict['transformer.ln_f.weight'] = mk_tensor(w['lnfw'])
|
||||
model_dict['transformer.ln_f.bias'] = mk_tensor(w['lnfb'])
|
||||
|
||||
# Create a GPT-2 model instance, in the requested dtype
|
||||
config = GPT2Config(vocab_size = V,
|
||||
n_positions = maxT,
|
||||
n_ctx = maxT,
|
||||
n_embd = C,
|
||||
n_layer = L,
|
||||
n_head = H)
|
||||
model = GPT2LMHeadModel(config)
|
||||
if out_dtype == "bfloat16":
|
||||
model = model.to(torch.bfloat16)
|
||||
|
||||
# Set the model dict and save
|
||||
model.load_state_dict(model_dict)
|
||||
model.save_pretrained(output, max_shard_size="5GB", safe_serialization=True)
|
||||
|
||||
# Copy over a standard gpt2 tokenizer
|
||||
tokenizer = GPT2Tokenizer.from_pretrained("gpt2")
|
||||
tokenizer.save_pretrained(output)
|
||||
|
||||
if push_to_hub:
|
||||
print(f"Uploading {output} to Hugging Face")
|
||||
model.push_to_hub(output)
|
||||
tokenizer.push_to_hub(output)
|
||||
|
||||
def spin(output):
|
||||
print("Taking the exported model for a spin...")
|
||||
print('-'*80)
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
tokenizer = AutoTokenizer.from_pretrained(output)
|
||||
model = AutoModelForCausalLM.from_pretrained(output)
|
||||
tokens = tokenizer.encode("During photosynthesis in green plants", return_tensors="pt")
|
||||
output = model.generate(tokens, max_new_tokens=64, repetition_penalty=1.3)
|
||||
samples = tokenizer.batch_decode(output)
|
||||
for sample in samples:
|
||||
print('-'*30)
|
||||
print(sample)
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
if __name__== '__main__':
|
||||
parser=argparse.ArgumentParser()
|
||||
parser.add_argument("--input", "-i", help="The name of the llm.c model.bin file", type=str, required=True)
|
||||
parser.add_argument("--output","-o", help="The Hugging Face output model directory", type=str, required=True)
|
||||
parser.add_argument("--dtype", "-d", help="Output as either float32 or bfloat16 (default)", type=str, default="bfloat16")
|
||||
parser.add_argument("--push", "-p", help="Push the model to your Hugging Face account", type=bool, default=False)
|
||||
parser.add_argument("--spin", "-s", help="Take the model for a spin at the end?", type=bool, default=True)
|
||||
args = parser.parse_args()
|
||||
convert(args.input, args.output, args.push, args.dtype)
|
||||
if args.spin:
|
||||
spin(args.output)
|
||||
52
dev/eval/run_eval.sh
Executable file
52
dev/eval/run_eval.sh
Executable file
|
|
@ -0,0 +1,52 @@
|
|||
# https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard
|
||||
# (See About tab -> REPRODUCIBILITY)
|
||||
|
||||
# This script is intended to be run from the parent/root directory of llm.c repo.
|
||||
|
||||
# Clone the evaluation harness:
|
||||
|
||||
# git clone https://github.com/EleutherAI/lm-evaluation-harness/
|
||||
# cd lm-evaluation-harness
|
||||
# git checkout b281b0921b636bc36ad05c0b0b0763bd6dd43463
|
||||
# pip install -e .
|
||||
|
||||
# Then return to the parent directory and run this script
|
||||
|
||||
# cd ..
|
||||
# ./dev/eval/run_eval.sh [model_name] [result_name]
|
||||
|
||||
# where model_name is either a HF model such as openai-community/gpt2 or a local path such as ./gpt2-124M-run1
|
||||
# and result_name is the name of the folder under lm-evaluation-harness/results to store the evaluations
|
||||
|
||||
# Since the evals can take a couple of hours to run, depending on the model size, you may wish to
|
||||
# run within a "screen" session or by using nohup to run the script:
|
||||
|
||||
# nohup ./dev/eval/run_eval.sh [model_name] [result_name] > run.txt 2> err.txt &
|
||||
|
||||
if [ -z "$1" ]; then
|
||||
echo "Error: missing HuggingFace model name or path to local model"
|
||||
echo "./run_eval.sh hf_account/model_name my_result"
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "$2" ]; then
|
||||
echo "Error: missing output name for results"
|
||||
echo "./run_eval.sh hf_account/model_name my_result"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
export MODEL="$(realpath -s "$1")"
|
||||
export RESULT="$2"
|
||||
echo "Evaluating model $MODEL"
|
||||
echo "Saving results to ./lm-evaluation-harness/results/$RESULT"
|
||||
|
||||
cd lm-evaluation-harness
|
||||
|
||||
python main.py --model hf-causal-experimental --model_args pretrained=$MODEL,use_accelerate=True,trust_remote_code=True --tasks truthfulqa_mc --batch_size 1 --no_cache --write_out --output_path results/$RESULT/truthfulqa_0shot.json --device cuda
|
||||
python main.py --model hf-causal-experimental --model_args pretrained=$MODEL,use_accelerate=True,trust_remote_code=True --tasks winogrande --batch_size 1 --no_cache --write_out --output_path results/$RESULT/winogrande_5shot.json --device cuda --num_fewshot 5
|
||||
python main.py --model hf-causal-experimental --model_args pretrained=$MODEL,use_accelerate=True,trust_remote_code=True --tasks arc_challenge --batch_size 1 --no_cache --write_out --output_path results/$RESULT/arc_challenge_25shot.json --device cuda --num_fewshot 25
|
||||
python main.py --model hf-causal-experimental --model_args pretrained=$MODEL,use_accelerate=True,trust_remote_code=True --tasks hellaswag --batch_size 1 --no_cache --write_out --output_path results/$RESULT/hellaswag_10shot.json --device cuda --num_fewshot 10
|
||||
python main.py --model hf-causal-experimental --model_args pretrained=$MODEL,use_accelerate=True,trust_remote_code=True --tasks gsm8k --batch_size 1 --no_cache --write_out --output_path results/$RESULT/gsm8k_5shot.json --device cuda --num_fewshot 5
|
||||
python main.py --model hf-causal-experimental --model_args pretrained=$MODEL,use_accelerate=True,trust_remote_code=True --tasks hendrycksTest-abstract_algebra,hendrycksTest-anatomy,hendrycksTest-astronomy,hendrycksTest-business_ethics,hendrycksTest-clinical_knowledge,hendrycksTest-college_biology,hendrycksTest-college_chemistry,hendrycksTest-college_computer_science,hendrycksTest-college_mathematics,hendrycksTest-college_medicine,hendrycksTest-college_physics,hendrycksTest-computer_security,hendrycksTest-conceptual_physics,hendrycksTest-econometrics,hendrycksTest-electrical_engineering,hendrycksTest-elementary_mathematics,hendrycksTest-formal_logic,hendrycksTest-global_facts,hendrycksTest-high_school_biology,hendrycksTest-high_school_chemistry,hendrycksTest-high_school_computer_science,hendrycksTest-high_school_european_history,hendrycksTest-high_school_geography,hendrycksTest-high_school_government_and_politics,hendrycksTest-high_school_macroeconomics,hendrycksTest-high_school_mathematics,hendrycksTest-high_school_microeconomics,hendrycksTest-high_school_physics,hendrycksTest-high_school_psychology,hendrycksTest-high_school_statistics,hendrycksTest-high_school_us_history,hendrycksTest-high_school_world_history,hendrycksTest-human_aging,hendrycksTest-human_sexuality,hendrycksTest-international_law,hendrycksTest-jurisprudence,hendrycksTest-logical_fallacies,hendrycksTest-machine_learning,hendrycksTest-management,hendrycksTest-marketing,hendrycksTest-medical_genetics,hendrycksTest-miscellaneous,hendrycksTest-moral_disputes,hendrycksTest-moral_scenarios,hendrycksTest-nutrition,hendrycksTest-philosophy,hendrycksTest-prehistory,hendrycksTest-professional_accounting,hendrycksTest-professional_law,hendrycksTest-professional_medicine,hendrycksTest-professional_psychology,hendrycksTest-public_relations,hendrycksTest-security_studies,hendrycksTest-sociology,hendrycksTest-us_foreign_policy,hendrycksTest-virology,hendrycksTest-world_religions --batch_size 1 --no_cache --write_out --output_path results/$RESULT/mmlu_5shot.json --device cuda --num_fewshot 5
|
||||
|
||||
cd ..
|
||||
python dev/eval/summarize_eval.py lm-evaluation-harness/results/$RESULT
|
||||
27
dev/eval/summarize_eval.py
Normal file
27
dev/eval/summarize_eval.py
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
import json, sys
|
||||
|
||||
RESULT = sys.argv[1]
|
||||
print("-"*40)
|
||||
|
||||
key = {"arc_challenge_25shot.json": "acc_norm",
|
||||
"gsm8k_5shot.json": "acc",
|
||||
"hellaswag_10shot.json": "acc_norm",
|
||||
"mmlu_5shot.json": "acc",
|
||||
"truthfulqa_0shot.json": "mc2",
|
||||
"winogrande_5shot.json": "acc"
|
||||
}
|
||||
|
||||
total = 0
|
||||
for test in ["arc_challenge_25shot.json", "gsm8k_5shot.json", "hellaswag_10shot.json", "mmlu_5shot.json", "truthfulqa_0shot.json", "winogrande_5shot.json"]:
|
||||
data = json.loads(open("./%s/%s"%(RESULT, test)).read())
|
||||
r_count = 0
|
||||
r_total = 0
|
||||
for test_name in data['results']:
|
||||
r_count += 1
|
||||
r_total += data['results'][test_name][key[test]]
|
||||
score = (r_total*100)/r_count
|
||||
print(f"{test:<30} : {score:.4f}")
|
||||
total += score
|
||||
average = total / 6.0
|
||||
print("-"*40)
|
||||
print(f"Average Score : {average:.4f}")
|
||||
Loading…
Add table
Add a link
Reference in a new issue