aparler/c-llm
1
0
Fork 0
mirror of https://github.com/karpathy/llm.c.git synced 2026-07-28 20:35:09 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

revert the adamw allocation to previous. minor

Browse source
This commit is contained in:
Andrej Karpathy 2024-05-17 15:13:36 +00:00
parent b24279c4db
commit c8fa7a8c63
1 changed files with 6 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

11
train_gpt2.cu
View file

@ -2340,11 +2340,12 @@ void gpt2_update(GPT2 *model, float learning_rate, float beta1, float beta2, flo
floatX* grads_memory = (floatX*)model->grads_memory + multi_gpu_config->shard_offset;
if (model->m_memory == NULL) {
size_t alloc_bytes = 2 * num_parameters * sizeof(float);
printf0("allocating %zu MiB for AdamW optimizer state\n", alloc_bytes >> 20);
cudaCheck(cudaMalloc((void**)&model->m_memory, alloc_bytes));
model->v_memory = model->m_memory + num_parameters;
cudaCheck(cudaMemset(model->m_memory, 0, alloc_bytes));
printf0("allocating %zu MiB for AdamW optimizer state m\n", (num_parameters * sizeof(float)) >> 20);
printf0("allocating %zu MiB for AdamW optimizer state v\n", (num_parameters * sizeof(float)) >> 20);
cudaCheck(cudaMalloc((void**)&model->m_memory, num_parameters * sizeof(float)));
cudaCheck(cudaMalloc((void**)&model->v_memory, num_parameters * sizeof(float)));
cudaCheck(cudaMemset(model->m_memory, 0, num_parameters * sizeof(float)));
cudaCheck(cudaMemset(model->v_memory, 0, num_parameters * sizeof(float)));
if (model->use_master_weights == 1) {
printf0("allocating %zu MiB for master copy of params\n", (num_parameters * sizeof(float)) >> 20);
cudaCheck(cudaMalloc((void**)&model->master_weights, num_parameters * sizeof(float)));