aparler/c-llm
1
0
Fork 0
mirror of https://github.com/karpathy/llm.c.git synced 2026-07-20 23:05:08 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix tokenizer omg

Browse source
This commit is contained in:
Andrej Karpathy 2024-08-13 17:24:15 +00:00
parent 16635d41a2
commit 755458d0e1
3 changed files with 6 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

4
dev/data/fineweb.py
View file

@ -67,8 +67,8 @@ elif args.type =="edu":
def tokenize_llama(doc):
# tokenizes a single document and returns a numpy array of uint32 tokens
tokenizer = AutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3.1-8B")
encode = lambda s: tokenizer.encode(s, add_special_tokens=False, verbose=False)
eot = 128000 # BOS; hardcoding because I don't know how to get it properly TODO
encode = lambda s: tokenizer.encode(s, add_special_tokens=False, verbose=False, split_special_tokens=True)
eot = tokenizer.encode('')[0] # by default the tokenizer adds the EOT token (128000)
tokens = [eot] # the special <|endoftext|> token delimits all documents
tokens.extend(encode(doc["text"]))
tokens_np = np.array(tokens)

4
dev/data/tinyshakespeare.py
View file

@ -51,8 +51,8 @@ def tokenize(model_desc):
eot = enc._special_tokens['<|endoftext|>'] # end of text token
elif model_desc == "llama-3":
tokenizer = AutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3.1-8B")
encode = lambda s: tokenizer.encode(s, add_special_tokens=False, verbose=False)
eot = 128000 # BOS; hardcoding because I don't know how to get it properly TODO
encode = lambda s: tokenizer.encode(s, add_special_tokens=False, verbose=False, split_special_tokens=True)
eot = tokenizer.encode('')[0] # by default the tokenizer adds the EOT token (128000)
else:
raise ValueError(f"unknown model descriptor {model_desc}")
data_filename = os.path.join(DATA_CACHE_DIR, "tiny_shakespeare.txt")

4
dev/data/tinystories.py
View file

@ -77,8 +77,8 @@ def process_shard(shard_index, shard_filename, model_desc):
eot = enc._special_tokens['<|endoftext|>'] # end of text token
elif model_desc == "llama-3":
tokenizer = AutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3.1-8B")
encode = lambda s: tokenizer.encode(s, add_special_tokens=False, verbose=False)
eot = 128000 # BOS; hardcoding because I don't know how to get it properly TODO
encode = lambda s: tokenizer.encode(s, add_special_tokens=False, verbose=False, split_special_tokens=True)
eot = tokenizer.encode('')[0] # by default the tokenizer adds the EOT token (128000)
else:
raise ValueError(f"unknown model descriptor {model_desc}")