aparler/c-llm
1
0
Fork 0
mirror of https://github.com/karpathy/llm.c.git synced 2026-07-28 20:35:09 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

inline docs for sizes

Browse source
This commit is contained in:
Andrej Karpathy 2024-04-11 16:04:14 +00:00
parent bcd0dad6c8
commit 1751af1607
2 changed files with 78 additions and 78 deletions
Download patch file Download diff file Expand all files Collapse all files

78
train_gpt2.c
View file

@ -588,22 +588,22 @@ void gpt2_build_from_checkpoint(GPT2 *model, char* checkpoint_path) {
printf("channels: %d\n", C);
// allocate space for all the parameters and read them in
model->param_sizes[0] = V * C;
model->param_sizes[1] = maxT * C;
model->param_sizes[2] = L * C;
model->param_sizes[3] = L * C;
model->param_sizes[4] = L * (3 * C) * C;
model->param_sizes[5] = L * (3 * C);
model->param_sizes[6] = L * C * C;
model->param_sizes[7] = L * C;
model->param_sizes[8] = L * C;
model->param_sizes[9] = L * C;
model->param_sizes[10] = L * (4 * C) * C;
model->param_sizes[11] = L * (4 * C);
model->param_sizes[12] = L * C * (4 * C);
model->param_sizes[13] = L * C;
model->param_sizes[14] = C;
model->param_sizes[15] = C;
model->param_sizes[0] = V * C; // wte
model->param_sizes[1] = maxT * C; // wpe
model->param_sizes[2] = L * C; // ln1w
model->param_sizes[3] = L * C; // ln1b
model->param_sizes[4] = L * (3 * C) * C; // qkvw
model->param_sizes[5] = L * (3 * C); // qkvb
model->param_sizes[6] = L * C * C; // attprojw
model->param_sizes[7] = L * C; // attprojb
model->param_sizes[8] = L * C; // ln2w
model->param_sizes[9] = L * C; // ln2b
model->param_sizes[10] = L * (4 * C) * C; // fcw
model->param_sizes[11] = L * (4 * C); // fcb
model->param_sizes[12] = L * C * (4 * C); // fcprojw
model->param_sizes[13] = L * C; // fcprojb
model->param_sizes[14] = C; // lnfw
model->param_sizes[15] = C; // lnfb
// cound the number of paramaters
size_t num_parameters = 0;
@ -652,29 +652,29 @@ void gpt2_forward(GPT2 *model, int* inputs, int* targets, int B, int T) {
model->batch_size = B;
model->seq_len = T;
// and now allocate the space
model->act_sizes[0] = B * T * C;
model->act_sizes[1] = L * B * T * C;
model->act_sizes[2] = L * B * T;
model->act_sizes[3] = L * B * T;
model->act_sizes[4] = L * B * T * 3*C;
model->act_sizes[5] = L * B * T * C;
model->act_sizes[6] = L * B * NH * T * T;
model->act_sizes[7] = L * B * NH * T * T;
model->act_sizes[8] = L * B * T * C;
model->act_sizes[9] = L * B * T * C;
model->act_sizes[10] = L * B * T * C;
model->act_sizes[11] = L * B * T;
model->act_sizes[12] = L * B * T;
model->act_sizes[13] = L * B * T * 4*C;
model->act_sizes[14] = L * B * T * 4*C;
model->act_sizes[15] = L * B * T * C;
model->act_sizes[16] = L * B * T * C;
model->act_sizes[17] = B * T * C;
model->act_sizes[18] = B * T;
model->act_sizes[19] = B * T;
model->act_sizes[20] = B * T * V;
model->act_sizes[21] = B * T * V;
model->act_sizes[22] = B * T;
model->act_sizes[0] = B * T * C; // encoded
model->act_sizes[1] = L * B * T * C; // ln1
model->act_sizes[2] = L * B * T; // ln1_mean
model->act_sizes[3] = L * B * T; // ln1_rstd
model->act_sizes[4] = L * B * T * 3*C; // qkv
model->act_sizes[5] = L * B * T * C; // atty
model->act_sizes[6] = L * B * NH * T * T; // preatt
model->act_sizes[7] = L * B * NH * T * T; // att
model->act_sizes[8] = L * B * T * C; // attproj
model->act_sizes[9] = L * B * T * C; // residual2
model->act_sizes[10] = L * B * T * C; // ln2
model->act_sizes[11] = L * B * T; // ln2_mean
model->act_sizes[12] = L * B * T; // ln2_rstd
model->act_sizes[13] = L * B * T * 4*C; // fch
model->act_sizes[14] = L * B * T * 4*C; // fch_gelu
model->act_sizes[15] = L * B * T * C; // fcproj
model->act_sizes[16] = L * B * T * C; // residual3
model->act_sizes[17] = B * T * C; // lnf
model->act_sizes[18] = B * T; // lnf_mean
model->act_sizes[19] = B * T; // lnf_rstd
model->act_sizes[20] = B * T * V; // logits
model->act_sizes[21] = B * T * V; // probs
model->act_sizes[22] = B * T; // losses
size_t num_activations = 0;
for (size_t i = 0; i < NUM_ACTIVATION_TENSORS; i++) {
num_activations += model->act_sizes[i];

78
train_gpt2.cu
View file

@ -656,22 +656,22 @@ void gpt2_build_from_checkpoint(GPT2 *model, char* checkpoint_path) {
printf("channels: %d\n", C);
// allocate space for all the parameters and read them in
model->param_sizes[0] = V * C;
model->param_sizes[1] = maxT * C;
model->param_sizes[2] = L * C;
model->param_sizes[3] = L * C;
model->param_sizes[4] = L * (3 * C) * C;
model->param_sizes[5] = L * (3 * C);
model->param_sizes[6] = L * C * C;
model->param_sizes[7] = L * C;
model->param_sizes[8] = L * C;
model->param_sizes[9] = L * C;
model->param_sizes[10] = L * (4 * C) * C;
model->param_sizes[11] = L * (4 * C);
model->param_sizes[12] = L * C * (4 * C);
model->param_sizes[13] = L * C;
model->param_sizes[14] = C;
model->param_sizes[15] = C;
model->param_sizes[0] = V * C; // wte
model->param_sizes[1] = maxT * C; // wpe
model->param_sizes[2] = L * C; // ln1w
model->param_sizes[3] = L * C; // ln1b
model->param_sizes[4] = L * (3 * C) * C; // qkvw
model->param_sizes[5] = L * (3 * C); // qkvb
model->param_sizes[6] = L * C * C; // attprojw
model->param_sizes[7] = L * C; // attprojb
model->param_sizes[8] = L * C; // ln2w
model->param_sizes[9] = L * C; // ln2b
model->param_sizes[10] = L * (4 * C) * C; // fcw
model->param_sizes[11] = L * (4 * C); // fcb
model->param_sizes[12] = L * C * (4 * C); // fcprojw
model->param_sizes[13] = L * C; // fcprojb
model->param_sizes[14] = C; // lnfw
model->param_sizes[15] = C; // lnfb
// cound the number of paramaters
size_t num_parameters = 0;
@ -725,29 +725,29 @@ void gpt2_forward(GPT2 *model, int* inputs, int* targets, int B, int T) {
model->batch_size = B;
model->seq_len = T;
// and now allocate the space
model->act_sizes[0] = B * T * C;
model->act_sizes[1] = L * B * T * C;
model->act_sizes[2] = L * B * T;
model->act_sizes[3] = L * B * T;
model->act_sizes[4] = L * B * T * 3*C;
model->act_sizes[5] = L * B * T * C;
model->act_sizes[6] = L * B * NH * T * T;
model->act_sizes[7] = L * B * NH * T * T;
model->act_sizes[8] = L * B * T * C;
model->act_sizes[9] = L * B * T * C;
model->act_sizes[10] = L * B * T * C;
model->act_sizes[11] = L * B * T;
model->act_sizes[12] = L * B * T;
model->act_sizes[13] = L * B * T * 4*C;
model->act_sizes[14] = L * B * T * 4*C;
model->act_sizes[15] = L * B * T * C;
model->act_sizes[16] = L * B * T * C;
model->act_sizes[17] = B * T * C;
model->act_sizes[18] = B * T;
model->act_sizes[19] = B * T;
model->act_sizes[20] = B * T * V;
model->act_sizes[21] = B * T * V;
model->act_sizes[22] = B * T;
model->act_sizes[0] = B * T * C; // encoded
model->act_sizes[1] = L * B * T * C; // ln1
model->act_sizes[2] = L * B * T; // ln1_mean
model->act_sizes[3] = L * B * T; // ln1_rstd
model->act_sizes[4] = L * B * T * 3*C; // qkv
model->act_sizes[5] = L * B * T * C; // atty
model->act_sizes[6] = L * B * NH * T * T; // preatt
model->act_sizes[7] = L * B * NH * T * T; // att
model->act_sizes[8] = L * B * T * C; // attproj
model->act_sizes[9] = L * B * T * C; // residual2
model->act_sizes[10] = L * B * T * C; // ln2
model->act_sizes[11] = L * B * T; // ln2_mean
model->act_sizes[12] = L * B * T; // ln2_rstd
model->act_sizes[13] = L * B * T * 4*C; // fch
model->act_sizes[14] = L * B * T * 4*C; // fch_gelu
model->act_sizes[15] = L * B * T * C; // fcproj
model->act_sizes[16] = L * B * T * C; // residual3
model->act_sizes[17] = B * T * C; // lnf
model->act_sizes[18] = B * T; // lnf_mean
model->act_sizes[19] = B * T; // lnf_rstd
model->act_sizes[20] = B * T * V; // logits
model->act_sizes[21] = B * T * V; // probs
model->act_sizes[22] = B * T; // losses
model->act_sizes[23] = L * B * T * 3*C; // qkvr
model->act_sizes[24] = L * B * T * C; // v_accum
size_t num_activations = 0;