aparler/c-llm
1
0
Fork 0
mirror of https://github.com/karpathy/llm.c.git synced 2026-07-27 20:25:09 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

upload good 1558M sh script

Browse source
This commit is contained in:
Andrej Karpathy 2024-07-02 16:49:09 +00:00
parent 39270cc69a
commit c6efd90a86
1 changed files with 14 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

28
scripts/run_gpt2_1558M.sh
View file

@ -1,13 +1,13 @@
# GPT-2 (1558M) repro on FineWeb
# 1558M parameter model on 100B tokens
# => 6 * 1558Me6 * 100e9 = 6.966e20 ~= 1e21 capability model
# => 100,000 steps on 1M tokens/step (1,048,576 to be precise)
# on 8X A100 80GB SXM ($14/hr) steps in ~7s/iter
# => training time 100,000 steps * 7s = 194.4 hours ~= 8.1 days ~= $2721.6
# GPT-2 (1558M) repro on FineWeb-EDU
# 1558M parameter model on 32B tokens
# => 6 * 1558e6 * 32e9 = 6.966e20 ~= 3e20 capability model
# 32,000 steps on ~1M tokens/step (1,048,576 to be precise)
# on 8X H100 80GB SXM ($28/hr) steps in 2.80s/iter
# => training time 32,000 steps * 2.7s => 24 hours ~= 1 day ~= $672
make train_gpt2cu USE_CUDNN=1
out_dir="log_gpt2_1558M"
done_file="$out_dir/DONE_00100000"
done_file="$out_dir/DONE_00032000"
# in case the training stalls or crashes, loop to resume (-y 1)
while true; do
@ -18,24 +18,24 @@ while true; do
break
fi
# run python dev/data/fineweb.py --version 100B to prepro data
# run python dev/data/hellaswag.py to prepro hellaswag eval
mpirun -np 8 ./train_gpt2cu \
-i "dev/data/fineweb100B/fineweb_train_*.bin" \
-j "dev/data/fineweb100B/fineweb_val_*.bin" \
-i "dev/data/edu_fineweb100B/edu_fineweb_train_*.bin" \
-j "dev/data/edu_fineweb100B/edu_fineweb_val_*.bin" \
-o $out_dir \
-v 250 -s 300000 -g 144 \
-v 250 -s 300000 -g 384 \
-h 1 \
-b 16 -t 1024 \
-d 1048576 \
-r 0 \
-z 1 \
-c 0.1 \
-l 0.0002 \
-k "cosine" \
-l 0.0006 \
-q 0.1 \
-u 700 \
-n 2000 \
-x 100000 \
-x 32000 \
-ge 1 \
-y 1 \
-e "d48"