From 1751af16079a87f8735d1febfe2de71253d6d8d7 Mon Sep 17 00:00:00 2001 From: Andrej Karpathy Date: Thu, 11 Apr 2024 16:04:14 +0000 Subject: [PATCH] inline docs for sizes --- train_gpt2.c | 78 +++++++++++++++++++++++++-------------------------- train_gpt2.cu | 78 +++++++++++++++++++++++++-------------------------- 2 files changed, 78 insertions(+), 78 deletions(-) diff --git a/train_gpt2.c b/train_gpt2.c index a54a7b9..de39856 100644 --- a/train_gpt2.c +++ b/train_gpt2.c @@ -588,22 +588,22 @@ void gpt2_build_from_checkpoint(GPT2 *model, char* checkpoint_path) { printf("channels: %d\n", C); // allocate space for all the parameters and read them in - model->param_sizes[0] = V * C; - model->param_sizes[1] = maxT * C; - model->param_sizes[2] = L * C; - model->param_sizes[3] = L * C; - model->param_sizes[4] = L * (3 * C) * C; - model->param_sizes[5] = L * (3 * C); - model->param_sizes[6] = L * C * C; - model->param_sizes[7] = L * C; - model->param_sizes[8] = L * C; - model->param_sizes[9] = L * C; - model->param_sizes[10] = L * (4 * C) * C; - model->param_sizes[11] = L * (4 * C); - model->param_sizes[12] = L * C * (4 * C); - model->param_sizes[13] = L * C; - model->param_sizes[14] = C; - model->param_sizes[15] = C; + model->param_sizes[0] = V * C; // wte + model->param_sizes[1] = maxT * C; // wpe + model->param_sizes[2] = L * C; // ln1w + model->param_sizes[3] = L * C; // ln1b + model->param_sizes[4] = L * (3 * C) * C; // qkvw + model->param_sizes[5] = L * (3 * C); // qkvb + model->param_sizes[6] = L * C * C; // attprojw + model->param_sizes[7] = L * C; // attprojb + model->param_sizes[8] = L * C; // ln2w + model->param_sizes[9] = L * C; // ln2b + model->param_sizes[10] = L * (4 * C) * C; // fcw + model->param_sizes[11] = L * (4 * C); // fcb + model->param_sizes[12] = L * C * (4 * C); // fcprojw + model->param_sizes[13] = L * C; // fcprojb + model->param_sizes[14] = C; // lnfw + model->param_sizes[15] = C; // lnfb // cound the number of paramaters size_t num_parameters = 0; @@ -652,29 +652,29 @@ void gpt2_forward(GPT2 *model, int* inputs, int* targets, int B, int T) { model->batch_size = B; model->seq_len = T; // and now allocate the space - model->act_sizes[0] = B * T * C; - model->act_sizes[1] = L * B * T * C; - model->act_sizes[2] = L * B * T; - model->act_sizes[3] = L * B * T; - model->act_sizes[4] = L * B * T * 3*C; - model->act_sizes[5] = L * B * T * C; - model->act_sizes[6] = L * B * NH * T * T; - model->act_sizes[7] = L * B * NH * T * T; - model->act_sizes[8] = L * B * T * C; - model->act_sizes[9] = L * B * T * C; - model->act_sizes[10] = L * B * T * C; - model->act_sizes[11] = L * B * T; - model->act_sizes[12] = L * B * T; - model->act_sizes[13] = L * B * T * 4*C; - model->act_sizes[14] = L * B * T * 4*C; - model->act_sizes[15] = L * B * T * C; - model->act_sizes[16] = L * B * T * C; - model->act_sizes[17] = B * T * C; - model->act_sizes[18] = B * T; - model->act_sizes[19] = B * T; - model->act_sizes[20] = B * T * V; - model->act_sizes[21] = B * T * V; - model->act_sizes[22] = B * T; + model->act_sizes[0] = B * T * C; // encoded + model->act_sizes[1] = L * B * T * C; // ln1 + model->act_sizes[2] = L * B * T; // ln1_mean + model->act_sizes[3] = L * B * T; // ln1_rstd + model->act_sizes[4] = L * B * T * 3*C; // qkv + model->act_sizes[5] = L * B * T * C; // atty + model->act_sizes[6] = L * B * NH * T * T; // preatt + model->act_sizes[7] = L * B * NH * T * T; // att + model->act_sizes[8] = L * B * T * C; // attproj + model->act_sizes[9] = L * B * T * C; // residual2 + model->act_sizes[10] = L * B * T * C; // ln2 + model->act_sizes[11] = L * B * T; // ln2_mean + model->act_sizes[12] = L * B * T; // ln2_rstd + model->act_sizes[13] = L * B * T * 4*C; // fch + model->act_sizes[14] = L * B * T * 4*C; // fch_gelu + model->act_sizes[15] = L * B * T * C; // fcproj + model->act_sizes[16] = L * B * T * C; // residual3 + model->act_sizes[17] = B * T * C; // lnf + model->act_sizes[18] = B * T; // lnf_mean + model->act_sizes[19] = B * T; // lnf_rstd + model->act_sizes[20] = B * T * V; // logits + model->act_sizes[21] = B * T * V; // probs + model->act_sizes[22] = B * T; // losses size_t num_activations = 0; for (size_t i = 0; i < NUM_ACTIVATION_TENSORS; i++) { num_activations += model->act_sizes[i]; diff --git a/train_gpt2.cu b/train_gpt2.cu index 0a2cb46..2dcdb68 100644 --- a/train_gpt2.cu +++ b/train_gpt2.cu @@ -656,22 +656,22 @@ void gpt2_build_from_checkpoint(GPT2 *model, char* checkpoint_path) { printf("channels: %d\n", C); // allocate space for all the parameters and read them in - model->param_sizes[0] = V * C; - model->param_sizes[1] = maxT * C; - model->param_sizes[2] = L * C; - model->param_sizes[3] = L * C; - model->param_sizes[4] = L * (3 * C) * C; - model->param_sizes[5] = L * (3 * C); - model->param_sizes[6] = L * C * C; - model->param_sizes[7] = L * C; - model->param_sizes[8] = L * C; - model->param_sizes[9] = L * C; - model->param_sizes[10] = L * (4 * C) * C; - model->param_sizes[11] = L * (4 * C); - model->param_sizes[12] = L * C * (4 * C); - model->param_sizes[13] = L * C; - model->param_sizes[14] = C; - model->param_sizes[15] = C; + model->param_sizes[0] = V * C; // wte + model->param_sizes[1] = maxT * C; // wpe + model->param_sizes[2] = L * C; // ln1w + model->param_sizes[3] = L * C; // ln1b + model->param_sizes[4] = L * (3 * C) * C; // qkvw + model->param_sizes[5] = L * (3 * C); // qkvb + model->param_sizes[6] = L * C * C; // attprojw + model->param_sizes[7] = L * C; // attprojb + model->param_sizes[8] = L * C; // ln2w + model->param_sizes[9] = L * C; // ln2b + model->param_sizes[10] = L * (4 * C) * C; // fcw + model->param_sizes[11] = L * (4 * C); // fcb + model->param_sizes[12] = L * C * (4 * C); // fcprojw + model->param_sizes[13] = L * C; // fcprojb + model->param_sizes[14] = C; // lnfw + model->param_sizes[15] = C; // lnfb // cound the number of paramaters size_t num_parameters = 0; @@ -725,29 +725,29 @@ void gpt2_forward(GPT2 *model, int* inputs, int* targets, int B, int T) { model->batch_size = B; model->seq_len = T; // and now allocate the space - model->act_sizes[0] = B * T * C; - model->act_sizes[1] = L * B * T * C; - model->act_sizes[2] = L * B * T; - model->act_sizes[3] = L * B * T; - model->act_sizes[4] = L * B * T * 3*C; - model->act_sizes[5] = L * B * T * C; - model->act_sizes[6] = L * B * NH * T * T; - model->act_sizes[7] = L * B * NH * T * T; - model->act_sizes[8] = L * B * T * C; - model->act_sizes[9] = L * B * T * C; - model->act_sizes[10] = L * B * T * C; - model->act_sizes[11] = L * B * T; - model->act_sizes[12] = L * B * T; - model->act_sizes[13] = L * B * T * 4*C; - model->act_sizes[14] = L * B * T * 4*C; - model->act_sizes[15] = L * B * T * C; - model->act_sizes[16] = L * B * T * C; - model->act_sizes[17] = B * T * C; - model->act_sizes[18] = B * T; - model->act_sizes[19] = B * T; - model->act_sizes[20] = B * T * V; - model->act_sizes[21] = B * T * V; - model->act_sizes[22] = B * T; + model->act_sizes[0] = B * T * C; // encoded + model->act_sizes[1] = L * B * T * C; // ln1 + model->act_sizes[2] = L * B * T; // ln1_mean + model->act_sizes[3] = L * B * T; // ln1_rstd + model->act_sizes[4] = L * B * T * 3*C; // qkv + model->act_sizes[5] = L * B * T * C; // atty + model->act_sizes[6] = L * B * NH * T * T; // preatt + model->act_sizes[7] = L * B * NH * T * T; // att + model->act_sizes[8] = L * B * T * C; // attproj + model->act_sizes[9] = L * B * T * C; // residual2 + model->act_sizes[10] = L * B * T * C; // ln2 + model->act_sizes[11] = L * B * T; // ln2_mean + model->act_sizes[12] = L * B * T; // ln2_rstd + model->act_sizes[13] = L * B * T * 4*C; // fch + model->act_sizes[14] = L * B * T * 4*C; // fch_gelu + model->act_sizes[15] = L * B * T * C; // fcproj + model->act_sizes[16] = L * B * T * C; // residual3 + model->act_sizes[17] = B * T * C; // lnf + model->act_sizes[18] = B * T; // lnf_mean + model->act_sizes[19] = B * T; // lnf_rstd + model->act_sizes[20] = B * T * V; // logits + model->act_sizes[21] = B * T * V; // probs + model->act_sizes[22] = B * T; // losses model->act_sizes[23] = L * B * T * 3*C; // qkvr model->act_sizes[24] = L * B * T * C; // v_accum size_t num_activations = 0;