From 0c09a375685a21166410556f6415be1e50a3a4f1 Mon Sep 17 00:00:00 2001 From: FeSens Date: Thu, 18 Apr 2024 01:23:05 -0300 Subject: [PATCH] fix(attention_forward.cu): Fix the number of elements to check when verifying attention implementations. This was causing me problems when I was trying to debug a new implementation on fewer tokens. --- dev/cuda/attention_forward.cu | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/cuda/attention_forward.cu b/dev/cuda/attention_forward.cu index b060fa2..7cfc8c8 100644 --- a/dev/cuda/attention_forward.cu +++ b/dev/cuda/attention_forward.cu @@ -866,13 +866,13 @@ int main(int argc, char **argv) { if (kernel_num != 2) { // kernel 2 (knowingly) fails att/preatt because it uses a different algorithm // that estimates the softmax online and never materializes preatt/att - validate_result(d_att, att, "att", B * T * C, 1e-4f); + validate_result(d_att, att, "att", B * NH * T * T, 1e-4f); } if (kernel_num != 2 && kernel_num != 4) { // kernel 4 (knowingly) fails preatt because it fuses the scale normalization // into the softmax, so preatt is off by 1.0f / sqrt(HS) // but att and out (checked below) should match. - validate_result(d_preatt, preatt, "preatt", B * T * C, 1e-4f); + validate_result(d_preatt, preatt, "preatt", B * NH * T * T, 1e-4f); } } printf("All results match. Starting benchmarks.\n\n");