From 08fe3eef43f92c2cc77d66a89c87b41bfc2e1a28 Mon Sep 17 00:00:00 2001 From: Andrej Karpathy Date: Fri, 24 May 2024 15:56:05 +0000 Subject: [PATCH] change default shard size for fineweb10B --- dev/data/fineweb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/data/fineweb.py b/dev/data/fineweb.py index db5a126..891096e 100644 --- a/dev/data/fineweb.py +++ b/dev/data/fineweb.py @@ -29,7 +29,7 @@ from data_common import write_datafile # ------------------------------------------ parser = argparse.ArgumentParser(description="FineWeb dataset preprocessing") -parser.add_argument("-s", "--shard_size", type=int, default=10**9, help="Size of each shard in tokens") +parser.add_argument("-s", "--shard_size", type=int, default=10**8, help="Size of each shard in tokens") args = parser.parse_args() # create the cache directory if it doesn't exist yet