diff --git a/build/cpu.o b/build/cpu.o index 0b75e02..adc39e5 100644 Binary files a/build/cpu.o and b/build/cpu.o differ diff --git a/build/tensor.o b/build/tensor.o index b8560d8..0071d08 100644 Binary files a/build/tensor.o and b/build/tensor.o differ diff --git a/examples/train.ipynb b/examples/train.ipynb index 3a37c54..9c92203 100644 --- a/examples/train.ipynb +++ b/examples/train.ipynb @@ -26,23 +26,28 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Epoch [1/10], Loss: 1.7035\n", - "Epoch [2/10], Loss: 0.7193\n", - "Epoch [3/10], Loss: 0.3068\n", - "Epoch [4/10], Loss: 0.1742\n", - "Epoch [5/10], Loss: 0.1342\n", - "Epoch [6/10], Loss: 0.1232\n", - "Epoch [7/10], Loss: 0.1220\n", - "Epoch [8/10], Loss: 0.1241\n", - "Epoch [9/10], Loss: 0.1270\n", - "Epoch [10/10], Loss: 0.1297\n" + "Invalid axis" + ] + }, + { + "ename": "ValueError", + "evalue": "Matrix multiplication requires 2D tensors", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[7], line 56\u001b[0m\n\u001b[1;32m 53\u001b[0m loss \u001b[38;5;241m=\u001b[39m criterion(outputs, target)\n\u001b[1;32m 55\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mzero_grad()\n\u001b[0;32m---> 56\u001b[0m \u001b[43mloss\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 57\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mstep()\n\u001b[1;32m 59\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mEpoch [\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mepoch\u001b[38;5;250m \u001b[39m\u001b[38;5;241m+\u001b[39m\u001b[38;5;250m \u001b[39m\u001b[38;5;241m1\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m/\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mepochs\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m], Loss: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mloss[\u001b[38;5;241m0\u001b[39m]\u001b[38;5;132;01m:\u001b[39;00m\u001b[38;5;124m.4f\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m)\n", + "File \u001b[0;32m~/Documentos/recreate_pytorch/PyNorch/norch/tensor.py:167\u001b[0m, in \u001b[0;36mTensor.backward\u001b[0;34m(self, gradient)\u001b[0m\n\u001b[1;32m 165\u001b[0m \u001b[38;5;66;03m# Propagate gradients to inputs if not a leaf tensor\u001b[39;00m\n\u001b[1;32m 166\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m tensor\u001b[38;5;241m.\u001b[39mgrad_fn \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 167\u001b[0m grads \u001b[38;5;241m=\u001b[39m \u001b[43mtensor\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgrad_fn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\u001b[43mgrad\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 168\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m tensor, grad \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mzip\u001b[39m(tensor\u001b[38;5;241m.\u001b[39mgrad_fn\u001b[38;5;241m.\u001b[39minput, grads):\n\u001b[1;32m 169\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(tensor, Tensor) \u001b[38;5;129;01mand\u001b[39;00m tensor \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m visited:\n", + "File \u001b[0;32m~/Documentos/recreate_pytorch/PyNorch/norch/autograd/functions.py:90\u001b[0m, in \u001b[0;36mMatmulBackward.backward\u001b[0;34m(self, gradient)\u001b[0m\n\u001b[1;32m 88\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m [aux_sum, x\u001b[38;5;241m.\u001b[39mtranspose(\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m,\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m2\u001b[39m) \u001b[38;5;241m@\u001b[39m gradient]\n\u001b[1;32m 89\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m---> 90\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m [\u001b[43mgradient\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m@\u001b[39;49m\u001b[43m \u001b[49m\u001b[43my\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtranspose\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m-\u001b[39;49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m,\u001b[49m\u001b[38;5;241;43m-\u001b[39;49m\u001b[38;5;241;43m2\u001b[39;49m\u001b[43m)\u001b[49m, x\u001b[38;5;241m.\u001b[39mtranspose(\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m,\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m2\u001b[39m) \u001b[38;5;241m@\u001b[39m gradient]\n", + "File \u001b[0;32m~/Documentos/recreate_pytorch/PyNorch/norch/tensor.py:487\u001b[0m, in \u001b[0;36mTensor.__matmul__\u001b[0;34m(self, other)\u001b[0m\n\u001b[1;32m 484\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 485\u001b[0m \u001b[38;5;66;03m#2D matmul\u001b[39;00m\n\u001b[1;32m 486\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mndim \u001b[38;5;241m!=\u001b[39m \u001b[38;5;241m2\u001b[39m \u001b[38;5;129;01mor\u001b[39;00m other\u001b[38;5;241m.\u001b[39mndim \u001b[38;5;241m!=\u001b[39m \u001b[38;5;241m2\u001b[39m:\n\u001b[0;32m--> 487\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mMatrix multiplication requires 2D tensors\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 489\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mshape[\u001b[38;5;241m1\u001b[39m] \u001b[38;5;241m!=\u001b[39m other\u001b[38;5;241m.\u001b[39mshape[\u001b[38;5;241m0\u001b[39m]:\n\u001b[1;32m 490\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mIncompatible shapes for matrix multiplication\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", + "\u001b[0;31mValueError\u001b[0m: Matrix multiplication requires 2D tensors" ] } ], @@ -87,11 +92,13 @@ "for x in x_values:\n", " y_true.append(math.pow(math.sin(x), 2))\n", "\n", + "batch_size = 5\n", + "\n", "\n", "for epoch in range(epochs):\n", " for x, target in zip(x_values, y_true):\n", - " x = norch.Tensor([[x]]).T\n", - " target = norch.Tensor([[target]]).T\n", + " x = norch.Tensor([[x] for _ in range(batch_size)]).T\n", + " target = norch.Tensor([[target] for _ in range(batch_size)]).T\n", "\n", " x = x.to(device)\n", " target = target.to(device)\n", diff --git a/norch/__pycache__/tensor.cpython-38.pyc b/norch/__pycache__/tensor.cpython-38.pyc index 72d3beb..92de580 100644 Binary files a/norch/__pycache__/tensor.cpython-38.pyc and b/norch/__pycache__/tensor.cpython-38.pyc differ diff --git a/norch/autograd/__pycache__/functions.cpython-38.pyc b/norch/autograd/__pycache__/functions.cpython-38.pyc index a7d3c51..7a8db4b 100644 Binary files a/norch/autograd/__pycache__/functions.cpython-38.pyc and b/norch/autograd/__pycache__/functions.cpython-38.pyc differ diff --git a/norch/autograd/functions.py b/norch/autograd/functions.py index 30e452f..fe31be9 100644 --- a/norch/autograd/functions.py +++ b/norch/autograd/functions.py @@ -26,7 +26,7 @@ class AddBroadcastedBackward: # Sum along axes where the target shape dimension is 1 for i in range(len(shape)): if shape[i] == 1: - gradient = gradient.sum(axis=i) + gradient = gradient.sum(axis=i, keepdim=True) return gradient @@ -126,9 +126,10 @@ class LogBackward: return [grad_input] class SumBackward: - def __init__(self, x, axis=None): + def __init__(self, x, axis=None, keepdim=False): self.input = [x] self.axis = axis + self.keepdim = keepdim def backward(self, gradient): input_shape = self.input[0].shape @@ -136,13 +137,18 @@ class SumBackward: # If axis is None, sum reduces the tensor to a scalar. grad_output = float(gradient.tensor.contents.data[0]) * self.input[0].ones_like() else: + if not self.keepdim: + # Remove dimensions of size 1 from the gradient tensor. + input_shape = [s for i, s in enumerate(input_shape) if i != self.axis] + # Broadcast the gradient to the input shape along the specified axis. grad_output_shape = list(input_shape) - grad_output_shape[self.axis] = 1 + grad_output_shape.insert(self.axis, 1) grad_output = gradient.reshape(grad_output_shape) grad_output = grad_output + self.input[0].zeros_like() return [grad_output] + class ReshapeBackward: def __init__(self, x): self.input = [x] diff --git a/norch/csrc/cpu.cpp b/norch/csrc/cpu.cpp index 3b4e303..ffeadbc 100644 --- a/norch/csrc/cpu.cpp +++ b/norch/csrc/cpu.cpp @@ -205,7 +205,7 @@ void log_tensor_cpu(Tensor* tensor, float* result_data) { } } -void sum_tensor_cpu(Tensor* tensor, float* result_data, int axis) { +void sum_tensor_cpu(Tensor* tensor, float* result_data, int size, int* result_shape, int axis, bool keepdim) { if (axis == -1) { // Sum over all elements float sum = 0.0; @@ -219,35 +219,22 @@ void sum_tensor_cpu(Tensor* tensor, float* result_data, int axis) { return; } - int* result_shape = (int*)malloc((tensor->ndim - 1) * sizeof(int)); - if (result_shape == NULL) { - fprintf(stderr, "Memory allocation failed\n"); - exit(1); - } - int result_size = 1; int axis_stride = tensor->strides[axis]; - int idx = 0; - for (int i = 0; i < tensor->ndim; i++) { - if (i != axis) { - result_shape[idx++] = tensor->shape[i]; - result_size *= tensor->shape[i]; - } - } - - memset(result_data, 0, result_size * sizeof(float)); - for (int i = 0; i < tensor->shape[axis]; i++) { - for (int j = 0; j < result_size; j++) { + for (int j = 0; j < size; j++) { int index = 0; int remainder = j; for (int k = tensor->ndim - 2; k >= 0; k--) { - index += (remainder % result_shape[k]) * tensor->strides[k < axis ? k : k + 1]; + index += (remainder % result_shape[k]) * tensor->strides[k < axis ? k : k + 1]; remainder /= result_shape[k]; } result_data[j] += tensor->data[index + i * axis_stride]; } } + for (int j = 0; j < size; j++) { + printf("%f", result_data[j]); + } } } diff --git a/norch/csrc/cpu.h b/norch/csrc/cpu.h index be875ed..5fe358c 100644 --- a/norch/csrc/cpu.h +++ b/norch/csrc/cpu.h @@ -5,7 +5,7 @@ void add_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data); void add_broadcasted_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data, int* broadcasted_shape, int broadcasted_size); -void sum_tensor_cpu(Tensor* tensor, float* result_data, int axis); +void sum_tensor_cpu(Tensor* tensor, float* result_data, int size, int* shape, int axis, bool keepdim); void sub_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data); void sub_broadcasted_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data, int* broadcasted_shape, int broadcasted_size); void elementwise_mul_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data); diff --git a/norch/csrc/tensor.cpp b/norch/csrc/tensor.cpp index 824737d..c945689 100644 --- a/norch/csrc/tensor.cpp +++ b/norch/csrc/tensor.cpp @@ -172,7 +172,7 @@ extern "C" { } } - Tensor* sum_tensor(Tensor* tensor, int axis) { + Tensor* sum_tensor(Tensor* tensor, int axis, bool keepdim) { char* device = (char*)malloc(strlen(tensor->device) + 1); if (device != NULL) { @@ -192,29 +192,45 @@ extern "C" { shape[0] = 1; ndim = 1; } else { - shape = (int*) malloc((tensor->ndim - 1) * sizeof(int)); - for (int i = 0, j = 0; i < tensor->ndim; ++i) { - if (i != axis) { - shape[j++] = tensor->shape[i]; + + if (keepdim) { + shape = (int*) malloc((tensor->ndim) * sizeof(int)); + for (int i = 0; i < tensor->ndim; i++) { + shape[i] = tensor->shape[i]; } + shape[axis] = 1; + ndim = tensor->ndim; + + } else { + shape = (int*) malloc((tensor->ndim - 1) * sizeof(int)); + for (int i = 0, j = 0; i < tensor->ndim; ++i) { + if (i != axis) { + shape[j++] = tensor->shape[i]; + } + } + ndim = tensor->ndim - 1; } - ndim = tensor->ndim - 1; + } + + int size = 1; + for (int i = 0; i < ndim; i++) { + size *= shape[i]; } if (strcmp(tensor->device, "cuda") == 0) { float* result_data; - cudaMalloc((void**)&result_data, tensor->size * sizeof(float)); + cudaMalloc((void**)&result_data, size * sizeof(float)); sum_tensor_cuda(tensor, result_data); return create_tensor(result_data, shape, ndim, device); } else { - float* result_data = (float*)malloc(1 * sizeof(float)); + float* result_data = (float*)malloc(size * sizeof(float)); if (result_data == NULL) { fprintf(stderr, "Memory allocation failed\n"); exit(1); } - sum_tensor_cpu(tensor, result_data, axis); + sum_tensor_cpu(tensor, result_data, size, shape, axis, keepdim); return create_tensor(result_data, shape, ndim, device); } } diff --git a/norch/csrc/tensor.h b/norch/csrc/tensor.h index 5ff62d0..a32e74c 100644 --- a/norch/csrc/tensor.h +++ b/norch/csrc/tensor.h @@ -16,7 +16,7 @@ extern "C" { Tensor* create_tensor(float* data, int* shape, int ndim, char* device); float get_item(Tensor* tensor, int* indices); Tensor* add_tensor(Tensor* tensor1, Tensor* tensor2); - Tensor* sum_tensor(Tensor* tensor, int axis); + Tensor* sum_tensor(Tensor* tensor, int axis, bool keepdims); Tensor* sub_tensor(Tensor* tensor1, Tensor* tensor2); Tensor* elementwise_mul_tensor(Tensor* tensor1, Tensor* tensor2); Tensor* scalar_mul_tensor(Tensor* tensor, float scalar); diff --git a/norch/libtensor.so b/norch/libtensor.so index a125f3c..09af449 100755 Binary files a/norch/libtensor.so and b/norch/libtensor.so differ diff --git a/norch/nn/modules/__pycache__/linear.cpython-38.pyc b/norch/nn/modules/__pycache__/linear.cpython-38.pyc index d4a4c2c..15ee462 100644 Binary files a/norch/nn/modules/__pycache__/linear.cpython-38.pyc and b/norch/nn/modules/__pycache__/linear.cpython-38.pyc differ diff --git a/norch/tensor.py b/norch/tensor.py index 18192c3..a4a84f6 100644 --- a/norch/tensor.py +++ b/norch/tensor.py @@ -630,20 +630,28 @@ class Tensor: return result_data - def sum(self, axis=-1): - Tensor._C.sum_tensor.argtypes = [ctypes.POINTER(CTensor), ctypes.c_int] + def sum(self, axis=-1, keepdim=False): + Tensor._C.sum_tensor.argtypes = [ctypes.POINTER(CTensor), ctypes.c_int, ctypes.c_bool] Tensor._C.sum_tensor.restype = ctypes.POINTER(CTensor) - result_tensor_ptr = Tensor._C.sum_tensor(self.tensor, axis) + result_tensor_ptr = Tensor._C.sum_tensor(self.tensor, axis, keepdim) result_data = Tensor() result_data.tensor = result_tensor_ptr if axis == -1: - result_data.shape = [1] - result_data.ndim = 1 + if keepdim: + result_data.ndim = self.ndim + result_data.shape = [1] * self.ndim + + else: + result_data.shape = [1] + result_data.ndim = 1 else: - result_data.shape = self.shape[:axis] + self.shape[axis+1:] + if keepdim: + result_data.shape = self.shape[:axis] + [1] + self.shape[axis+1:] + else: + result_data.shape = self.shape[:axis] + self.shape[axis+1:] result_data.ndim = len(result_data.shape) result_data.device = self.device @@ -653,7 +661,7 @@ class Tensor: result_data.requires_grad = self.requires_grad if result_data.requires_grad: - result_data.grad_fn = SumBackward(self, axis) + result_data.grad_fn = SumBackward(self, axis, keepdim=keepdim) return result_data diff --git a/test.py b/test.py new file mode 100644 index 0000000..89801e0 --- /dev/null +++ b/test.py @@ -0,0 +1,24 @@ +import norch + +W1 = norch.Tensor([[1], [2], [3], [4], [5], [6], [7], [8], [9], [10]], requires_grad=True) # A has shape 10x1 +X = norch.Tensor([[1, 2, 3, 4, 5]]) # B has shape 1x5 +B1 = norch.Tensor([[1, 2, 3, 4, 5], [2, 1, 2, 3, 4], [3, 1,2,3, 4], [4, 1, 1, 1, 1,], [5,1,1,1,1], [6,1,1,1,1], [7,1,1,1,1], [8,1,1,1,1], [9,1,1,1,1], [10,1,1,1,1]], requires_grad=True) + + +W2 = norch.Tensor([[1], [2], [3], [4], [5], [6], [7], [8], [9], [10]], requires_grad=True).T +B2 = norch.Tensor([[1, 2, 3, 4, 5]], requires_grad=True) + + +Z1 = W1 @ X + B1 + +# Perform matrix multiplication + +Z2 = W2 @ Z1 + B2 +print(Z2.shape) + +l = Z2.sum() +l.backward() +#print(l) + +print("Resulting matrix shape:", W1.grad) + diff --git a/tests/test_operations.py b/tests/test_operations.py index d6552a6..a09d351 100644 --- a/tests/test_operations.py +++ b/tests/test_operations.py @@ -249,6 +249,24 @@ class TestTensorOperations(unittest.TestCase): self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + def test_sum_axis_keepdim(self): + """ + Test summation of a tensor along a specific axis with keepdim=True + """ + norch_tensor = norch.Tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + norch_result = norch_tensor.sum(axis=1, keepdim=True) + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor = torch.tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + torch_expected = torch.sum(torch_tensor, dim=1, keepdim=True) + + print(torch_result, '\n', torch_expected) + + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + def test_transpose_T(self): """ Test transposition of a tensor: tensor.T