From 1dafa60ad972ae43d70080e2e9497c60ea31fe42 Mon Sep 17 00:00:00 2001 From: Aleksa Gordic Date: Thu, 13 Jun 2024 21:27:42 +0200 Subject: [PATCH] Hardcode shuffle info in all train scripts --- train_gpt2.cu | 4 ++-- train_gpt2_fp32.cu | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/train_gpt2.cu b/train_gpt2.cu index f6aa6ea..a8084aa 100644 --- a/train_gpt2.cu +++ b/train_gpt2.cu @@ -1632,8 +1632,8 @@ int main(int argc, char *argv[]) { // build DataLoaders for both train and val DataLoader train_loader, val_loader; - dataloader_init(&train_loader, train_data_pattern, B, T, multi_gpu_config.process_rank, multi_gpu_config.num_processes, 1); - dataloader_init(&val_loader, val_data_pattern, B, T, multi_gpu_config.process_rank, multi_gpu_config.num_processes, 0); + dataloader_init(&train_loader, train_data_pattern, B, T, multi_gpu_config.process_rank, multi_gpu_config.num_processes, true); + dataloader_init(&val_loader, val_data_pattern, B, T, multi_gpu_config.process_rank, multi_gpu_config.num_processes, false); // figure out the number of training steps we will run for int train_num_batches = max_steps; // passed in from command line if (train_num_batches == -1) { diff --git a/train_gpt2_fp32.cu b/train_gpt2_fp32.cu index ebd7c92..df412ea 100644 --- a/train_gpt2_fp32.cu +++ b/train_gpt2_fp32.cu @@ -1634,8 +1634,8 @@ int main(int argc, char *argv[]) { // build DataLoaders for both train and val DataLoader train_loader, val_loader; - dataloader_init(&train_loader, train_data_pattern, B, T, 0, 1); - dataloader_init(&val_loader, val_data_pattern, B, T, 0, 1); + dataloader_init(&train_loader, train_data_pattern, B, T, 0, 1, 1); + dataloader_init(&val_loader, val_data_pattern, B, T, 0, 1, 0); int train_num_batches = train_loader.num_tokens / (B*T); // let's do 1 epoch by default for now int val_num_batches = val_loader.num_tokens / (B*T); if (val_num_batches > val_max_steps) { val_num_batches = val_max_steps; }