aparler/c-llm
1
0
Fork 0
mirror of https://github.com/karpathy/llm.c.git synced 2026-07-27 20:25:09 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

make hellaswag optional eval yay

Browse source
This commit is contained in:
Andrej Karpathy 2024-05-22 22:42:37 +00:00
parent 67239d9b8f
commit da59861d38
2 changed files with 14 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

5
dev/data/hellaswag.py
View file

@ -101,16 +101,11 @@ def render_example(example):
def iterate_examples(split):
# there are 10,042 examples in total in val
n = 0
download(split)
with open(os.path.join(DATA_CACHE_DIR, f"hellaswag_{split}.jsonl"), "r") as f:
for line in f:
example = json.loads(line)
n += 1
yield example
# DEBUGGING, TODO REMOVE
if n >= 101:
break
@torch.no_grad()
def evaluate(model_type, device):

16
train_gpt2.cu
View file

@ -2694,7 +2694,12 @@ int main(int argc, char *argv[]) {
// build an EvalLoader for HellaSwag
EvalLoader eval_loader;
const char* hellaswag_path = "dev/data/hellaswag/hellaswag_val.bin";
evalloader_init(&eval_loader, hellaswag_path, B, T, multi_gpu_config.process_rank, multi_gpu_config.num_processes);
const char hellaswag_available = access(hellaswag_path, F_OK) == 0;
if (hellaswag_available) {
evalloader_init(&eval_loader, hellaswag_path, B, T, multi_gpu_config.process_rank, multi_gpu_config.num_processes);
}
printf0("| hellaswag available | %-50s |\n", hellaswag_available ? "yes" : "no");
printf0("+-----------------------+----------------------------------------------------+\n");
// pretty print in a table the multi-gpu configuration as well
set_zero_configs(&multi_gpu_config, zero_stage, model.num_parameters);
@ -2702,6 +2707,11 @@ int main(int argc, char *argv[]) {
printf0("| zero_stage | %-50d |\n", multi_gpu_config.zero_stage);
printf0("+-----------------------+----------------------------------------------------+\n");
// prints outside of pretty table to here and below
if (!hellaswag_available) {
printf0("HellaSwag eval not found at %s, skipping its evaluation\n", hellaswag_path);
printf0("You can run `python dev/data/hellaswag.py` to export and use it.\n");
}
// more prints related to allocations from gpt2_build_from_checkpoint down here to not mess up our table above
printf0("num_parameters: %zu => bytes: %zu\n", model.num_parameters, model.num_parameters_bytes);
printf0("allocated %d MiB for model parameters\n", (int)round(model.num_parameters_bytes / (1024 * 1024)));
@ -2755,11 +2765,13 @@ int main(int argc, char *argv[]) {
}
// once in a while estimate HellaSwag accuracy
if (step % val_loss_every == 0 || last_step) {
if (hellaswag_available &&
(step % val_loss_every == 0 || last_step)) {
NvtxRange evaluation_range("evaluation");
float eval_acc_norm = 0.0f;
evalloader_reset(&eval_loader);
for (int i = 0; i < eval_loader.num_batches; i++) {
if (i % 10 == 0) { printf("evaluating HellaSwag: %d/%d\r", i, eval_loader.num_batches); }
evalloader_next_batch(&eval_loader);
gpt2_forward(&model, eval_loader.inputs, eval_loader.targets, B, T);
int correct = evalloader_stat_losses(&eval_loader, model.cpu_losses_fp32);