aparler/c-llm
1
0
Fork 0
mirror of https://github.com/karpathy/llm.c.git synced 2026-07-27 20:25:09 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Use a make pattern rule to simplify Makefile, group kernels into forward/backward/updates/nccls, add some comments and update the README.md

Browse source
This commit is contained in:
austinvhuang 2024-04-27 00:13:01 -04:00
parent d937f93aaa
commit 25d703fbec
2 changed files with 41 additions and 62 deletions
Download patch file Download diff file Expand all files Collapse all files

97
dev/cuda/Makefile
View file

@ -1,88 +1,63 @@
NVCC := $(shell which nvcc 2>/dev/null)
# Makefile for building dev/cuda kernels
# Find nvcc
NVCC := $(shell which nvcc 2>/dev/null)
ifeq ($(NVCC),)
$(error nvcc not found.)
endif
TARGETS = adamw attention_backward attention_forward classifier_fused crossentropy_forward crossentropy_softmax_backward encoder_backward encoder_forward gelu_forward layernorm_backward layernorm_forward matmul_backward matmul_backward_bias matmul_forward nccl_all_reduce residual_forward softmax_forward trimat_forward
# Compiler flags
CFLAGS = -O3 --use_fast_math
MPI_PATHS = -I/usr/lib/x86_64-linux-gnu/openmpi/include -L/usr/lib/x86_64-linux-gnu/openmpi/lib/
%: %.cu
$(NVCC) $(CFLAGS) $< -o $@ -lcublas
TARGETS = adamw attention_backward attention_forward classifier_fused crossentropy_forward crossentropy_softmax_backward encoder_backward encoder_forward gelu_forward layernorm_backward layernorm_forward matmul_backward matmul_backward_bias matmul_forward nccl_all_reduce residual_forward softmax_forward trimat_forward
all: $(TARGETS)
adamw: adamw.cu
$(NVCC) $(CFLAGS) adamw.cu -o adamw
attention_backward: attention_backward.cu
$(NVCC) $(CFLAGS) attention_backward.cu -o attention_backward -lcublas
# Forward kernels
attention_forward: attention_forward.cu
$(NVCC) $(CFLAGS) attention_forward.cu -o attention_forward -lcublas
classifier_fused: classifier_fused.cu
$(NVCC) $(CFLAGS) classifier_fused.cu -o classifier_fused
crossentropy_forward: crossentropy_forward.cu
$(NVCC) $(CFLAGS) crossentropy_forward.cu -o crossentropy_forward
crossentropy_softmax_backward: crossentropy_softmax_backward.cu
$(NVCC) $(CFLAGS) crossentropy_softmax_backward.cu -o crossentropy_softmax_backward
encoder_backward: encoder_backward.cu
$(NVCC) $(CFLAGS) encoder_backward.cu -o encoder_backward
encoder_forward: encoder_forward.cu
$(NVCC) $(CFLAGS) encoder_forward.cu -o encoder_forward
gelu_forward: gelu_forward.cu
$(NVCC) $(CFLAGS) gelu_forward.cu -o gelu_forward
layernorm_backward: layernorm_backward.cu
$(NVCC) $(CFLAGS) layernorm_backward.cu -o layernorm_backward
layernorm_forward: layernorm_forward.cu
$(NVCC) $(CFLAGS) layernorm_forward.cu -o layernorm_forward
matmul_forward: matmul_forward.cu
$(NVCC) $(CFLAGS) -Xcompiler -fopenmp matmul_forward.cu -o matmul_forward -lcublas -lcublasLt
residual_forward: residual_forward.cu
softmax_forward: softmax_forward.cu
trimat_forward: trimat_forward.cu
# Backward kernels
attention_backward: attention_backward.cu
crossentropy_softmax_backward: crossentropy_softmax_backward.cu
encoder_backward: encoder_backward.cu
layernorm_backward: layernorm_backward.cu
matmul_backward_bias: matmul_backward_bias.cu
matmul_backward: matmul_backward.cu
$(NVCC) $(CFLAGS) -Xcompiler -fopenmp matmul_backward.cu -o matmul_backward -lcublas
matmul_backward_bias: matmul_backward_bias.cu
$(NVCC) $(CFLAGS) matmul_backward_bias.cu -o matmul_backward_bias
# Update kernels
matmul_forward: matmul_forward.cu
$(NVCC) $(CFLAGS) -Xcompiler -fopenmp matmul_forward.cu -o matmul_forward -lcublas -lcublasLt
adamw: adamw.cu
# NCCL
nccl_all_reduce: nccl_all_reduce.cu
$(NVCC) -lmpi -lnccl -I/usr/lib/x86_64-linux-gnu/openmpi/include -L/usr/lib/x86_64-linux-gnu/openmpi/lib/ nccl_all_reduce.cu -o nccl_all_reduce
$(NVCC) -lmpi -lnccl $(MPI_PATHS) nccl_all_reduce.cu -o nccl_all_reduce
residual_forward: residual_forward.cu
$(NVCC) $(CFLAGS) residual_forward.cu -o residual_forward
softmax_forward: softmax_forward.cu
$(NVCC) $(CFLAGS) softmax_forward.cu -o softmax_forward
trimat_forward: trimat_forward.cu
$(NVCC) $(CFLAGS) trimat_forward.cu -o trimat_forward -lcublas
run_all: all
@for target in $(TARGETS); do \
echo "\n========================================"; \
echo "Running $$target ..."; \
echo "========================================\n"; \
./$$target; \
done
clean:
rm -f $(TARGETS)
run_all: all
./adamw
./attention_backward
./attention_forward
./classifier_fused
./crossentropy_forward
./crossentropy_softmax_backward
./encoder_backward
./encoder_forward
./gelu_forward
./layernorm_backward
./layernorm_forward
./matmul_backward
./matmul_backward_bias
./matmul_forward
./nccl_all_reduce
./residual_forward
./softmax_forward
./trimat_forward

6
dev/cuda/README.md
View file

@ -1,3 +1,7 @@
# dev/cuda
This directory is scratch space for developing various versions of the needed CUDA kernels. Each file develops a kernel, see the top of each file for instructions on how to compile and run each one.
This directory is scratch space for developing various versions of the needed CUDA kernels. Each file develops a kernel, see the top of each file for instructions on how to compile and run each one using the `nvcc` compiler.
An alternative to invoking `nvcc` manually is to use `make` with the accompanying `Makefile` in this directory. Each kernel has its own `make` build target, invoking `make` for the target builds the associated binary.
For example, `make gelu_forward` builds the forward GELU kernel, creating a binary that can be executed by running `./gelu_forward`. `make` or `make all` builds all the kernels in this directory. To delete all binary build targets, run `make clean`.