aparler/c-llm
1
0
Fork 0
mirror of https://github.com/karpathy/llm.c.git synced 2026-07-27 20:25:09 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #172 from FeSens/fix/validate_result_attention

Browse source 
fix(attention_forward.cu): Fix the number of elements to check
This commit is contained in:
Andrej 2024-04-17 21:28:31 -07:00 • committed by GitHub
commit aed262d773
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 2 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

4
dev/cuda/attention_forward.cu
View file

@ -866,13 +866,13 @@ int main(int argc, char **argv) {
if (kernel_num != 2) {
// kernel 2 (knowingly) fails att/preatt because it uses a different algorithm
// that estimates the softmax online and never materializes preatt/att
validate_result(d_att, att, "att", B * T * C, 1e-4f);
validate_result(d_att, att, "att", B * NH * T * T, 1e-4f);
}
if (kernel_num != 2 && kernel_num != 4) {
// kernel 4 (knowingly) fails preatt because it fuses the scale normalization
// into the softmax, so preatt is off by 1.0f / sqrt(HS)
// but att and out (checked below) should match.
validate_result(d_preatt, preatt, "preatt", B * T * C, 1e-4f);
validate_result(d_preatt, preatt, "preatt", B * NH * T * T, 1e-4f);
}
}
printf("All results match. Starting benchmarks.\n\n");