aparler/c-llm
1
0
Fork 0
mirror of https://github.com/karpathy/llm.c.git synced 2026-07-27 20:25:09 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

change default shard size for fineweb10B

Browse source
This commit is contained in:
Andrej Karpathy 2024-05-24 15:56:05 +00:00
parent b0f065fcf8
commit 08fe3eef43
1 changed files with 1 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
dev/data/fineweb.py
View file

@ -29,7 +29,7 @@ from data_common import write_datafile
# ------------------------------------------
parser = argparse.ArgumentParser(description="FineWeb dataset preprocessing")
parser.add_argument("-s", "--shard_size", type=int, default=10**9, help="Size of each shard in tokens")
parser.add_argument("-s", "--shard_size", type=int, default=10**8, help="Size of each shard in tokens")
args = parser.parse_args()
# create the cache directory if it doesn't exist yet