aparler/c-llm
1
0
Fork 0
mirror of https://github.com/karpathy/llm.c.git synced 2026-07-27 20:25:09 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

bump the threshold for qkvw because flashattention expands the error here a tiny bit

Browse source
This commit is contained in:
Andrej Karpathy 2024-05-11 17:19:41 +00:00
parent 58d9dda497
commit 3dbb0bb89c
1 changed files with 1 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
test_gpt2.cu
View file

@ -247,7 +247,7 @@ int main(int argc, char *argv[]) {
// In that case it's ok to extend the tolerance by a bit, after a manual review.
allok = allok & check_tensor(tensors1[0], tensors2[0], V * C, "wte", 8e-1f);
allok = allok & check_tensor(tensors1[1], tensors2[1], maxT * C, "wpe", 1e-2f);
allok = allok & check_tensor(tensors1[2], tensors2[2], L * 3*C * C, "qkvw", 1.1e-1); // hmm a bit high
allok = allok & check_tensor(tensors1[2], tensors2[2], L * 3*C * C, "qkvw", 1.4e-1); // hmm a bit high
allok = allok & check_tensor(tensors1[3], tensors2[3], L * 3*C, "qkvb", 4e-2f);
allok = allok & check_tensor(tensors1[4], tensors2[4], L * C * C, "attprojw", 3e-2f);
allok = allok & check_tensor(tensors1[5], tensors2[5], L * C, "attprojb", 3e-2f);