aparler/c-llm
1
0
Fork 0
mirror of https://github.com/karpathy/llm.c.git synced 2026-07-28 20:35:09 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

more defensive

Browse source
This commit is contained in:
Andrej Karpathy 2024-04-29 20:25:29 +00:00
parent b4c346ac2a
commit 938f8f78bb
1 changed files with 1 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

1
tokenizer.h
View file

@ -59,6 +59,7 @@ void tokenizer_init(Tokenizer *tokenizer, const char *filename) {
if (version == 1) {
// version 1 didn't include the EOT token id
// so we assume it is 50256, the EOT in GPT-2
assert(tokenizer->vocab_size == 50257); // let's be defensive here
tokenizer->eot_token = 50256;
} else if (version == 2) {
tokenizer->eot_token = header[3];