From d680fbad25ee8f7468cb5f032aee9d96ee4cfc2e Mon Sep 17 00:00:00 2001
From: Andrej Karpathy <andrej.karpathy@gmail.com>
Date: Thu, 11 Apr 2024 21:49:14 +0000
Subject: [PATCH] one more comment on attention

---
 train_gpt2.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/train_gpt2.c b/train_gpt2.c
index 54c939f..70e80cd 100644
--- a/train_gpt2.c
+++ b/train_gpt2.c
@@ -221,6 +221,9 @@ void attention_forward(float* out, float* preatt, float* att,
     // preatt, att are (B, NH, T, T). NH = number of heads, T = sequence length
     // that holds the pre-attention and post-attention scores (used in backward)
     // output is (B, T, C)
+    // attention is the only layer that mixes information across time
+    // every other operation is applied at every (b,t) position independently
+    // (and of course, no layer mixes information across batch)
     int C3 = C*3;
     int hs = C / NH; // head size
     float scale = 1.0 / sqrtf(hs);