aparler/c-llm
1
0
Fork 0
mirror of https://github.com/karpathy/llm.c.git synced 2026-07-28 20:35:09 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' of github.com:karpathy/llm.c

Browse source
This commit is contained in:
Andrej Karpathy 2024-06-18 03:03:37 +00:00
commit efbbdc8f46
2 changed files with 9 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

9
llmc/dataloader.h
View file

@ -103,7 +103,8 @@ void prepare_intra_shard_indices_(DataLoader *loader) {
free(loader->intra_shard_indices);
}
loader->intra_shard_indices = (int*)mallocCheck(loader->shard_num_samples * sizeof(int));
random_permutation_with_init(loader->intra_shard_indices, loader->shard_num_samples, &loader->shuffle_rng, 1);
init_identity_permutation(loader->intra_shard_indices, loader->shard_num_samples);
random_permutation(loader->intra_shard_indices, loader->shard_num_samples, &loader->shuffle_rng);
}
void dataloader_reset(DataLoader *loader) {
@ -111,7 +112,7 @@ void dataloader_reset(DataLoader *loader) {
loader->current_sample_idx = 0;
if (loader->should_shuffle) { // shuffle the shards
random_permutation_with_init(loader->shard_indices, loader->glob_result.gl_pathc, &loader->shuffle_rng, 0);
random_permutation(loader->shard_indices, loader->glob_result.gl_pathc, &loader->shuffle_rng);
}
dataloader_load_shard_(loader, loader->current_shard_idx);
@ -171,9 +172,7 @@ void dataloader_init(DataLoader *loader,
manual_seed(&shuffle_rng, 42 + process_rank);
loader->shuffle_rng = shuffle_rng;
loader->shard_indices = (int*)mallocCheck(loader->glob_result.gl_pathc * sizeof(int));
for (int i = 0; i < loader->glob_result.gl_pathc; i++) {
loader->shard_indices[i] = i; // start with identity permutation
}
init_identity_permutation(loader->shard_indices, loader->glob_result.gl_pathc);
loader->intra_shard_indices = NULL; // dynamically allocated allowing different shard sizes
}

10
llmc/rand.h
View file

@ -220,13 +220,13 @@ void normal_(float* data, unsigned int numel, float mean, float std, mt19937_sta
}
}
void random_permutation_with_init(int* data, int numel, mt19937_state* state, int should_init) {
if (should_init) {
for (int i = 0; i < numel; i++) {
data[i] = i;
}
void init_identity_permutation(int *data, int numel) {
for (int i = 0; i < numel; i++) {
data[i] = i;
}
}
void random_permutation(int* data, int numel, mt19937_state* state) {
for (int i = numel - 1; i > 0; i--) {
// pick an index j in [0, i] with equal probability
int j = randint32(state) % (i + 1);