aparler/c-llm
1
0
Fork 0
mirror of https://github.com/karpathy/llm.c.git synced 2026-07-28 20:35:09 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

make default dloss be 1/BT

Browse source
This commit is contained in:
Andrej Karpathy 2024-04-19 00:27:28 +00:00
parent ea3e2abdcf
commit 8a2cab3431
1 changed files with 2 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

3
dev/cuda/classifier_fused.cu
View file

@ -244,9 +244,10 @@ __global__ void fused_classifier_kernel2(float* dlogits, float* losses, float* p
losses[idx] = -logf(prob);
}
// very sensible default for dlosses is 1/(B*T), which is the uniform loss
float dloss = dlosses != NULL ? dlosses[idx] : 1.0f / (B*T);
// calculate the gradients directly, saves bandwidth from probs during training
// but also supports writing probs for inference-only and debugging
float dloss = dlosses ? dlosses[idx] : 0.f;
const float4* logits_vec4 = reinterpret_cast<const float4*>(logits + idx * P);
for (int i = threadIdx.x; i < (V+3)/4; i += blockDim.x) {
// this is the 2nd read of logits after the one in prepare_softmax2