aparler/c-llm
1
0
Fork 0
mirror of https://github.com/karpathy/llm.c.git synced 2026-07-27 20:25:09 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

slightly increase wte threshold to make test_gpt2.cu pass (?)

Browse source
This commit is contained in:
ademeure 2024-05-01 15:40:30 +01:00
parent 57c09529d9
commit c778dc0098
1 changed files with 1 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
test_gpt2.cu
View file

@ -265,7 +265,7 @@ int main(int argc, char *argv[]) {
// Also, if code changes and some of these get tripped, it could be ok if it's not by too much,
// because our use of stochastic rounding is adding some non-determinism "pepper noise".
// In that case it's ok to extend the tolerance by a bit, after a manual review.
allok = allok & check_tensor(tensors1[0], tensors2[0], V * C, "wte", 6e-1f);
allok = allok & check_tensor(tensors1[0], tensors2[0], V * C, "wte", 8e-1f);
allok = allok & check_tensor(tensors1[1], tensors2[1], maxT * C, "wpe", 1e-2f);
allok = allok & check_tensor(tensors1[2], tensors2[2], L * 3*C * C, "qkvw", 1.1e-1); // hmm a bit high
allok = allok & check_tensor(tensors1[3], tensors2[3], L * 3*C, "qkvb", 4e-2f);