aparler/c-llm
1
0
Fork 0
mirror of https://github.com/karpathy/llm.c.git synced 2026-07-20 23:05:08 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #801 from ngc92/ngc92/fix-test

Browse source 
Fix gradient tests
This commit is contained in:
Andrej 2025-05-10 16:24:10 -07:00 • committed by GitHub
commit f1e2ace651
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 15 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

35
test_gpt2.cu
View file

@ -113,8 +113,6 @@ int main(int argc, char *argv[]) {
size_t V = model.config.vocab_size;
size_t Vp = model.config.padded_vocab_size;
size_t maxT = model.config.max_seq_len;
size_t L = model.config.num_layers;
size_t C = model.config.channels;
for (int i = 1; i < argc; i+=2) {
if (i + 1 >= argc) { exit(EXIT_FAILURE); } // must have arg after flag
@ -267,29 +265,26 @@ int main(int argc, char *argv[]) {
// Also, different GPUs may use different matrix multiplication algorithms, so the
// actual errors can be hardware specific.
float grad_thresholds[NUM_PARAMETER_TENSORS] = {5e-1f, 4e-3f, 1e-1f, 3.5e-2f, 2e-2f, 3e-2f, 5e-2f, 5e-2f, 5e-2f, 1.5e-2f, 5e-4f, 8e-3f, 1.5e-3f, 2.5e-3f, 1e-1f, 2e-2f};
float grad_thresholds[NUM_PARAMETER_TENSORS] = {
5e-1f, 4e-3f, 1e-1f, 4e-2f,
5e-2f, 3.5e-2f, 2e-2f, 3e-2f,
5e-2f, 3e-2f, 3e-2f, 3e-2f,
2e-2f, 1e-2f,1e-1f,2e-2f};
#if defined(ENABLE_FP32)
for (int i = 0; i < NUM_PARAMETER_TENSORS; i++) {
grad_thresholds[i] = 1e-6f; // we can be much more precise in FP32
}
#endif
allok = allok & check_tensor(tensors1[0], tensors2[0], V * C, "wte", grad_thresholds[0]);
allok = allok & check_tensor(tensors1[1], tensors2[1], maxT * C, "wpe", grad_thresholds[1]);
allok = allok & check_tensor(tensors1[2], tensors2[2], L * 3*C * C, "qkvw", grad_thresholds[2]);
allok = allok & check_tensor(tensors1[3], tensors2[3], L * 3*C, "qkvb", grad_thresholds[3]);
allok = allok & check_tensor(tensors1[4], tensors2[4], L * C * C, "attprojw", grad_thresholds[4]);
allok = allok & check_tensor(tensors1[5], tensors2[5], L * C, "attprojb", grad_thresholds[5]);
allok = allok & check_tensor(tensors1[6], tensors2[6], L * 4*C * C, "fcw", grad_thresholds[6]);
allok = allok & check_tensor(tensors1[7], tensors2[7], L * 4*C, "fcb", grad_thresholds[7]);
allok = allok & check_tensor(tensors1[8], tensors2[8], L * C * 4*C, "fcprojw", grad_thresholds[8]);
allok = allok & check_tensor(tensors1[9], tensors2[9], L * C, "fcprojb", grad_thresholds[9]);
allok = allok & check_tensor(tensors1[10], tensors2[10], L * C, "ln1w", grad_thresholds[10]);
allok = allok & check_tensor(tensors1[11], tensors2[11], L * C, "ln1b", grad_thresholds[11]);
allok = allok & check_tensor(tensors1[12], tensors2[12], L * C, "ln2w", grad_thresholds[12]);
allok = allok & check_tensor(tensors1[13], tensors2[13], L * C, "ln2b", grad_thresholds[13]);
allok = allok & check_tensor(tensors1[14], tensors2[14], C, "lnfw", grad_thresholds[14]);
allok = allok & check_tensor(tensors1[15], tensors2[15], C, "lnfb", grad_thresholds[15]);
const char* names[NUM_PARAMETER_TENSORS] = {
"wte", "wpe", "ln1w", "ln1b", "qkvw", "qkvb", "attrpojw",
"attprojb", "ln2w", "ln2b", "fcw", "fcb", "fcprojw", "fcprojb",
"lnfw", "lnfb"
};
size_t* count = model.param_elements;
for(int i = 0; i < NUM_PARAMETER_TENSORS; ++i) {
allok = allok & check_tensor(tensors1[i], tensors2[i], count[i], names[i], grad_thresholds[i]);
}
}
float grad_norm = gpt2_calculate_grad_norm(&model, &multi_gpu_config);