diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index e69de29..0000000 diff --git a/README.md b/README.md index d6eb1bb..e7042b9 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ x1 = norch.Tensor([[1, 2], [3, 4]], requires_grad=True).to("cuda") x2 = norch.Tensor([[4, 3], - [2, 1]], requires_grad=True).to("cuda) + [2, 1]], requires_grad=True).to("cuda") x3 = x1 @ x2 result = x3.sum() diff --git a/build/cpu.o b/build/cpu.o index ecb6dd6..6f33bbc 100644 Binary files a/build/cpu.o and b/build/cpu.o differ diff --git a/build/cuda.cu.o b/build/cuda.cu.o index 427e573..437afeb 100644 Binary files a/build/cuda.cu.o and b/build/cuda.cu.o differ diff --git a/build/distributed.o b/build/distributed.o index caf2c5f..5e37508 100644 Binary files a/build/distributed.o and b/build/distributed.o differ diff --git a/build/tensor.o b/build/tensor.o index d48dad3..7b38f19 100644 Binary files a/build/tensor.o and b/build/tensor.o differ diff --git a/examples/train_multigpu.py b/examples/train_multigpu.py index 8cbaaff..8d112cb 100644 --- a/examples/train_multigpu.py +++ b/examples/train_multigpu.py @@ -38,6 +38,7 @@ def main(): ] ) + print("Loading data") train_data, test_data = norch.norchvision.datasets.MNIST.splits(transform=transform, target_transform=target_transform) distributed_sampler = DistributedSampler(dataset=train_data, num_replicas=world_size, rank=rank) train_loader = norch.utils.data.DataLoader(train_data, batch_size=BATCH_SIZE, sampler=distributed_sampler) @@ -58,6 +59,7 @@ def main(): return out + print("Creating model") model = MyModel().to(device) model = DistributedDataParallel(model) criterion = nn.CrossEntropyLoss() @@ -67,8 +69,15 @@ def main(): print(f"Starting training on Rank {rank}/{world_size}\n\n") for epoch in range(epochs): + + avg_loss = 0 + num_steps = 0 + for idx, batch in enumerate(train_loader): + if idx % 100 == 0 and rank == 0: + print(f"Epoch: {epoch}/{epochs} - Step: {idx} / {len(train_loader)}") + inputs, target = batch inputs = inputs.to(device) @@ -83,10 +92,14 @@ def main(): loss.backward() optimizer.step() - + + avg_loss += loss[0] + num_steps += 1 + + avg_loss = avg_loss / num_steps if rank == 0: - print(f'Epoch [{epoch + 1}/{epochs}], Loss: {loss[0]:.4f}') - loss_list.append(loss[0]) + print(f'Epoch [{epoch + 1}/{epochs}], Loss: {avg_loss:.4f}') + loss_list.append(avg_loss) if __name__ == "__main__": main() diff --git a/examples/train_nn_mnist.ipynb b/examples/train_nn_mnist.ipynb index 21e61f7..12c37ce 100644 --- a/examples/train_nn_mnist.ipynb +++ b/examples/train_nn_mnist.ipynb @@ -139,7 +139,11 @@ "optimizer = optim.SGD(model.parameters(), lr=0.01)\n", "loss_list = []\n", "\n", - "for epoch in range(epochs): \n", + "for epoch in range(epochs): \n", + " \n", + " avg_loss = 0 \n", + " num_steps = 0\n", + " \n", " for idx, batch in enumerate(train_loader):\n", "\n", " inputs, target = batch\n", @@ -157,8 +161,12 @@ "\n", " optimizer.step()\n", "\n", - " print(f'Epoch [{epoch + 1}/{epochs}], Loss: {loss[0]:.4f}')\n", - " loss_list.append(loss[0])" + " avg_loss += loss[0]\n", + " num_steps += 1\n", + "\n", + " avg_loss = avg_loss / num_steps\n", + " print(f'Epoch [{epoch + 1}/{epochs}], Loss: {avg_loss:.4f}')\n", + " loss_list.append(avg_loss)" ] }, { diff --git a/examples/train_singlegpu.py b/examples/train_singlegpu.py index 8c58ed6..6b0dfac 100644 --- a/examples/train_singlegpu.py +++ b/examples/train_singlegpu.py @@ -24,6 +24,8 @@ def main(): ] ) + + print("Loading data") train_data, test_data = norch.norchvision.datasets.MNIST.splits(transform=transform, target_transform=target_transform) train_loader = norch.utils.data.DataLoader(train_data, batch_size=BATCH_SIZE) @@ -43,14 +45,23 @@ def main(): return out + print("Creating model") model = MyModel().to(device) criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(model.parameters(), lr=0.01) loss_list = [] - for epoch in range(epochs): + print("Starting training") + for epoch in range(epochs): + + avg_loss = 0 + num_steps = 0 + for idx, batch in enumerate(train_loader): + if idx % 100 == 0: + print(f"Epoch: {epoch}/{epochs} - Step: {idx} / {len(train_loader)}") + inputs, target = batch inputs = inputs.to(device) @@ -64,6 +75,13 @@ def main(): loss.backward() optimizer.step() + + avg_loss += loss[0] + num_steps += 1 + + avg_loss = avg_loss / num_steps + print(f'Epoch [{epoch + 1}/{epochs}], Loss: {avg_loss:.4f}') + loss_list.append(avg_loss) if __name__ == "__main__": diff --git a/norch/__pycache__/__init__.cpython-38.pyc b/norch/__pycache__/__init__.cpython-38.pyc index 9ef6331..873ae18 100644 Binary files a/norch/__pycache__/__init__.cpython-38.pyc and b/norch/__pycache__/__init__.cpython-38.pyc differ diff --git a/norch/__pycache__/tensor.cpython-38.pyc b/norch/__pycache__/tensor.cpython-38.pyc index 4b1058a..ad61d69 100644 Binary files a/norch/__pycache__/tensor.cpython-38.pyc and b/norch/__pycache__/tensor.cpython-38.pyc differ diff --git a/norch/csrc/cuda.cu b/norch/csrc/cuda.cu index 20e600b..268d8a2 100644 --- a/norch/csrc/cuda.cu +++ b/norch/csrc/cuda.cu @@ -42,6 +42,10 @@ __host__ void cuda_to_cpu(Tensor* tensor) { strcpy(tensor->device, device_str); } +__host__ void free_cuda(float* data) { + cudaFree(data); +} + __global__ void add_tensor_cuda_kernel(float* data1, float* data2, float* result_data, int size) { int i = blockIdx.x * blockDim.x + threadIdx.x; @@ -136,7 +140,7 @@ __global__ void sum_tensor_cuda_kernel(float* data, float* result_data, int size __syncthreads(); // Perform block-wise reduction - for (int s = blockDim.x / 2; s > 0; s >>= 1) { + for (int s = blockDim.x / 2; s > 0; s >>= 1) { // s>>=1 --> s = s/2 if (tid < s) { partial_sum[tid] += partial_sum[tid + s]; } diff --git a/norch/csrc/cuda.h b/norch/csrc/cuda.h index 702dea1..f6395f0 100644 --- a/norch/csrc/cuda.h +++ b/norch/csrc/cuda.h @@ -4,6 +4,7 @@ __host__ void cpu_to_cuda(Tensor* tensor, int device_id); __host__ void cuda_to_cpu(Tensor* tensor); + __host__ void free_cuda(float* data); __global__ void add_tensor_cuda_kernel(float* data1, float* data2, float* result_data, int size); __host__ void add_tensor_cuda(Tensor* tensor1, Tensor* tensor2, float* result_data); diff --git a/norch/csrc/tensor.cpp b/norch/csrc/tensor.cpp index 74d8178..54882bd 100644 --- a/norch/csrc/tensor.cpp +++ b/norch/csrc/tensor.cpp @@ -16,37 +16,79 @@ extern "C" { fprintf(stderr, "Memory allocation failed\n"); exit(1); } - tensor->data = data; - tensor->shape = shape; tensor->ndim = ndim; - tensor->device = (char*)malloc(strlen(device) + 1); - if (device != NULL) { - strcpy(tensor->device, device); - } else { - fprintf(stderr, "Memory allocation failed\n"); - exit(-1); - } - tensor->size = 1; for (int i = 0; i < ndim; i++) { tensor->size *= shape[i]; } + tensor->device = strdup(device); tensor->strides = (int*)malloc(ndim * sizeof(int)); - if (tensor->strides == NULL) { + tensor->shape = (int*)malloc(ndim * sizeof(int)); + + if (tensor->device == NULL || tensor->strides == NULL || tensor->shape == NULL) { fprintf(stderr, "Memory allocation failed\n"); exit(1); } + + memcpy(tensor->shape, shape, tensor->ndim * sizeof(int)); + strcpy(tensor->device, device); + + if (strcmp(tensor->device, "cpu") == 0) { + tensor->data = (float*)malloc(tensor->size * sizeof(float)); + memcpy(tensor->data, data, tensor->size * sizeof(float)); + } else { + //already allocated on gpu + tensor->data = data; + } + int stride = 1; for (int i = ndim - 1; i >= 0; i--) { tensor->strides[i] = stride; stride *= shape[i]; } - + return tensor; } + void delete_tensor(Tensor* tensor) { + if (tensor != NULL) { + + if (tensor->strides != NULL) { + free(tensor->strides); + tensor->strides = NULL; + } + + if (tensor->shape != NULL) { + free(tensor->shape); + tensor->shape = NULL; + } + + if (tensor->data != NULL) { + if (strcmp(tensor->device, "cpu") == 0) { + free(tensor->data); + } else { + free_cuda(tensor->data); + } + tensor->data = NULL; + } + + if (tensor->device != NULL) { + free(tensor->device); + tensor->device = NULL; + } + + if (tensor->strides != NULL) { + free(tensor->strides); + tensor->strides = NULL; + } + + free(tensor); + } + } + + float get_item(Tensor* tensor, int* indices) { int index = 0; for (int i = 0; i < tensor->ndim; i++) { diff --git a/norch/csrc/tensor.h b/norch/csrc/tensor.h index 5669d4f..2465125 100644 --- a/norch/csrc/tensor.h +++ b/norch/csrc/tensor.h @@ -5,8 +5,6 @@ typedef struct { float* data; int* strides; int* shape; - int* strides_cuda; - int* shape_cuda; int ndim; int size; char* device; @@ -14,6 +12,7 @@ typedef struct { extern "C" { Tensor* create_tensor(float* data, int* shape, int ndim, char* device); + void delete_tensor(Tensor* tensor); float get_item(Tensor* tensor, int* indices); Tensor* add_tensor(Tensor* tensor1, Tensor* tensor2); Tensor* sum_tensor(Tensor* tensor, int axis, bool keepdims); diff --git a/norch/libtensor.so b/norch/libtensor.so index 68eb2f2..300caf1 100755 Binary files a/norch/libtensor.so and b/norch/libtensor.so differ diff --git a/norch/optim/optimizers/__pycache__/sgd.cpython-38.pyc b/norch/optim/optimizers/__pycache__/sgd.cpython-38.pyc index 0daa1ff..6897844 100644 Binary files a/norch/optim/optimizers/__pycache__/sgd.cpython-38.pyc and b/norch/optim/optimizers/__pycache__/sgd.cpython-38.pyc differ diff --git a/norch/optim/optimizers/sgd.py b/norch/optim/optimizers/sgd.py index 62375f6..616d1b9 100644 --- a/norch/optim/optimizers/sgd.py +++ b/norch/optim/optimizers/sgd.py @@ -26,3 +26,4 @@ class SGD(Optimizer): parameter.detach() velocity.detach() + del parameter diff --git a/norch/tensor.py b/norch/tensor.py index 0a1ecab..0287fb6 100644 --- a/norch/tensor.py +++ b/norch/tensor.py @@ -46,13 +46,16 @@ class Tensor: Tensor._C.create_tensor.argtypes = [ctypes.POINTER(ctypes.c_float), ctypes.POINTER(ctypes.c_int), ctypes.c_int, ctypes.c_char_p] Tensor._C.create_tensor.restype = ctypes.POINTER(CTensor) - self.tensor = Tensor._C.create_tensor( self.data_ctype, self.shape_ctype, self.ndim_ctype, self.device_ctype ) + + del self.data_ctype + del self.shape_ctype + del self.device_ctype else: self.tensor = None, @@ -81,6 +84,12 @@ class Tensor: flat_data, shape = flatten_recursively(nested_list) return flat_data, shape + def __del__(self): + if self.tensor is not None: + Tensor._C.delete_tensor.argtypes = [ctypes.POINTER(CTensor)] + Tensor._C.delete_tensor.restype = None + Tensor._C.delete_tensor(self.tensor) + def __setattr__(self, name, value): if name == 'grad': for hook in self.hooks: