aparler/c-llm
1
0
Fork 0
mirror of https://github.com/karpathy/llm.c.git synced 2026-07-26 20:15:08 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' of github.com:karpathy/llm.c

Browse source
This commit is contained in:
Andrej Karpathy 2024-04-18 04:40:09 +00:00
commit 959ea543dc
1 changed files with 2 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

4
dev/cuda/attention_forward.cu
View file

@ -866,13 +866,13 @@ int main(int argc, char **argv) {
if (kernel_num != 2) {
// kernel 2 (knowingly) fails att/preatt because it uses a different algorithm
// that estimates the softmax online and never materializes preatt/att
validate_result(d_att, att, "att", B * T * C, 1e-4f);
validate_result(d_att, att, "att", B * NH * T * T, 1e-4f);
}
if (kernel_num != 2 && kernel_num != 4) {
// kernel 4 (knowingly) fails preatt because it fuses the scale normalization
// into the softmax, so preatt is off by 1.0f / sqrt(HS)
// but att and out (checked below) should match.
validate_result(d_preatt, preatt, "preatt", B * T * C, 1e-4f);
validate_result(d_preatt, preatt, "preatt", B * NH * T * T, 1e-4f);
}
}
printf("All results match. Starting benchmarks.\n\n");