aparler/PyNorch
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

implementing keepdim sum

Browse source
This commit is contained in:
lucasdelimanogueira 2024-05-17 19:08:09 -03:00
parent abcd17e679
commit 5dd3f3d202
15 changed files with 119 additions and 53 deletions
Download patch file Download diff file Expand all files Collapse all files

BIN
build/cpu.o
View file

Binary file not shown.

BIN
build/tensor.o
View file

Binary file not shown.

33
examples/train.ipynb
View file

@ -26,23 +26,28 @@
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": 7,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch [1/10], Loss: 1.7035\n",
"Epoch [2/10], Loss: 0.7193\n",
"Epoch [3/10], Loss: 0.3068\n",
"Epoch [4/10], Loss: 0.1742\n",
"Epoch [5/10], Loss: 0.1342\n",
"Epoch [6/10], Loss: 0.1232\n",
"Epoch [7/10], Loss: 0.1220\n",
"Epoch [8/10], Loss: 0.1241\n",
"Epoch [9/10], Loss: 0.1270\n",
"Epoch [10/10], Loss: 0.1297\n"
"Invalid axis"
]
},
{
"ename": "ValueError",
"evalue": "Matrix multiplication requires 2D tensors",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[7], line 56\u001b[0m\n\u001b[1;32m 53\u001b[0m loss \u001b[38;5;241m=\u001b[39m criterion(outputs, target)\n\u001b[1;32m 55\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mzero_grad()\n\u001b[0;32m---> 56\u001b[0m \u001b[43mloss\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 57\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mstep()\n\u001b[1;32m 59\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mEpoch [\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mepoch\u001b[38;5;250m \u001b[39m\u001b[38;5;241m+\u001b[39m\u001b[38;5;250m \u001b[39m\u001b[38;5;241m1\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m/\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mepochs\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m], Loss: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mloss[\u001b[38;5;241m0\u001b[39m]\u001b[38;5;132;01m:\u001b[39;00m\u001b[38;5;124m.4f\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m)\n",
"File \u001b[0;32m~/Documentos/recreate_pytorch/PyNorch/norch/tensor.py:167\u001b[0m, in \u001b[0;36mTensor.backward\u001b[0;34m(self, gradient)\u001b[0m\n\u001b[1;32m 165\u001b[0m \u001b[38;5;66;03m# Propagate gradients to inputs if not a leaf tensor\u001b[39;00m\n\u001b[1;32m 166\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m tensor\u001b[38;5;241m.\u001b[39mgrad_fn \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 167\u001b[0m grads \u001b[38;5;241m=\u001b[39m \u001b[43mtensor\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgrad_fn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\u001b[43mgrad\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 168\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m tensor, grad \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mzip\u001b[39m(tensor\u001b[38;5;241m.\u001b[39mgrad_fn\u001b[38;5;241m.\u001b[39minput, grads):\n\u001b[1;32m 169\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(tensor, Tensor) \u001b[38;5;129;01mand\u001b[39;00m tensor \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m visited:\n",
"File \u001b[0;32m~/Documentos/recreate_pytorch/PyNorch/norch/autograd/functions.py:90\u001b[0m, in \u001b[0;36mMatmulBackward.backward\u001b[0;34m(self, gradient)\u001b[0m\n\u001b[1;32m 88\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m [aux_sum, x\u001b[38;5;241m.\u001b[39mtranspose(\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m,\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m2\u001b[39m) \u001b[38;5;241m@\u001b[39m gradient]\n\u001b[1;32m 89\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m---> 90\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m [\u001b[43mgradient\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m@\u001b[39;49m\u001b[43m \u001b[49m\u001b[43my\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtranspose\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m-\u001b[39;49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m,\u001b[49m\u001b[38;5;241;43m-\u001b[39;49m\u001b[38;5;241;43m2\u001b[39;49m\u001b[43m)\u001b[49m, x\u001b[38;5;241m.\u001b[39mtranspose(\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m,\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m2\u001b[39m) \u001b[38;5;241m@\u001b[39m gradient]\n",
"File \u001b[0;32m~/Documentos/recreate_pytorch/PyNorch/norch/tensor.py:487\u001b[0m, in \u001b[0;36mTensor.__matmul__\u001b[0;34m(self, other)\u001b[0m\n\u001b[1;32m 484\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 485\u001b[0m \u001b[38;5;66;03m#2D matmul\u001b[39;00m\n\u001b[1;32m 486\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mndim \u001b[38;5;241m!=\u001b[39m \u001b[38;5;241m2\u001b[39m \u001b[38;5;129;01mor\u001b[39;00m other\u001b[38;5;241m.\u001b[39mndim \u001b[38;5;241m!=\u001b[39m \u001b[38;5;241m2\u001b[39m:\n\u001b[0;32m--> 487\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mMatrix multiplication requires 2D tensors\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 489\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mshape[\u001b[38;5;241m1\u001b[39m] \u001b[38;5;241m!=\u001b[39m other\u001b[38;5;241m.\u001b[39mshape[\u001b[38;5;241m0\u001b[39m]:\n\u001b[1;32m 490\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mIncompatible shapes for matrix multiplication\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
"\u001b[0;31mValueError\u001b[0m: Matrix multiplication requires 2D tensors"
]
}
],
@ -87,11 +92,13 @@
"for x in x_values:\n",
" y_true.append(math.pow(math.sin(x), 2))\n",
"\n",
"batch_size = 5\n",
"\n",
"\n",
"for epoch in range(epochs):\n",
" for x, target in zip(x_values, y_true):\n",
" x = norch.Tensor([[x]]).T\n",
" target = norch.Tensor([[target]]).T\n",
" x = norch.Tensor([[x] for _ in range(batch_size)]).T\n",
" target = norch.Tensor([[target] for _ in range(batch_size)]).T\n",
"\n",
" x = x.to(device)\n",
" target = target.to(device)\n",

BIN
norch/__pycache__/tensor.cpython-38.pyc
View file

Binary file not shown.

BIN
norch/autograd/__pycache__/functions.cpython-38.pyc
View file

Binary file not shown.

12
norch/autograd/functions.py
View file

@ -26,7 +26,7 @@ class AddBroadcastedBackward:
# Sum along axes where the target shape dimension is 1
for i in range(len(shape)):
if shape[i] == 1:
gradient = gradient.sum(axis=i)
gradient = gradient.sum(axis=i, keepdim=True)
return gradient
@ -126,9 +126,10 @@ class LogBackward:
return [grad_input]
class SumBackward:
def __init__(self, x, axis=None):
def __init__(self, x, axis=None, keepdim=False):
self.input = [x]
self.axis = axis
self.keepdim = keepdim
def backward(self, gradient):
input_shape = self.input[0].shape
@ -136,13 +137,18 @@ class SumBackward:
# If axis is None, sum reduces the tensor to a scalar.
grad_output = float(gradient.tensor.contents.data[0]) * self.input[0].ones_like()
else:
if not self.keepdim:
# Remove dimensions of size 1 from the gradient tensor.
input_shape = [s for i, s in enumerate(input_shape) if i != self.axis]
# Broadcast the gradient to the input shape along the specified axis.
grad_output_shape = list(input_shape)
grad_output_shape[self.axis] = 1
grad_output_shape.insert(self.axis, 1)
grad_output = gradient.reshape(grad_output_shape)
grad_output = grad_output + self.input[0].zeros_like()
return [grad_output]
class ReshapeBackward:
def __init__(self, x):
self.input = [x]

25
norch/csrc/cpu.cpp
View file

@ -205,7 +205,7 @@ void log_tensor_cpu(Tensor* tensor, float* result_data) {
}
}
void sum_tensor_cpu(Tensor* tensor, float* result_data, int axis) {
void sum_tensor_cpu(Tensor* tensor, float* result_data, int size, int* result_shape, int axis, bool keepdim) {
if (axis == -1) {
// Sum over all elements
float sum = 0.0;
@ -219,35 +219,22 @@ void sum_tensor_cpu(Tensor* tensor, float* result_data, int axis) {
return;
}
int* result_shape = (int*)malloc((tensor->ndim - 1) * sizeof(int));
if (result_shape == NULL) {
fprintf(stderr, "Memory allocation failed\n");
exit(1);
}
int result_size = 1;
int axis_stride = tensor->strides[axis];
int idx = 0;
for (int i = 0; i < tensor->ndim; i++) {
if (i != axis) {
result_shape[idx++] = tensor->shape[i];
result_size *= tensor->shape[i];
}
}
memset(result_data, 0, result_size * sizeof(float));
for (int i = 0; i < tensor->shape[axis]; i++) {
for (int j = 0; j < result_size; j++) {
for (int j = 0; j < size; j++) {
int index = 0;
int remainder = j;
for (int k = tensor->ndim - 2; k >= 0; k--) {
index += (remainder % result_shape[k]) * tensor->strides[k < axis ? k : k + 1];
index += (remainder % result_shape[k]) * tensor->strides[k < axis ? k : k + 1];
remainder /= result_shape[k];
}
result_data[j] += tensor->data[index + i * axis_stride];
}
}
for (int j = 0; j < size; j++) {
printf("%f", result_data[j]);
}
}
}

2
norch/csrc/cpu.h
View file

@ -5,7 +5,7 @@
void add_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data);
void add_broadcasted_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data, int* broadcasted_shape, int broadcasted_size);
void sum_tensor_cpu(Tensor* tensor, float* result_data, int axis);
void sum_tensor_cpu(Tensor* tensor, float* result_data, int size, int* shape, int axis, bool keepdim);
void sub_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data);
void sub_broadcasted_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data, int* broadcasted_shape, int broadcasted_size);
void elementwise_mul_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data);

34
norch/csrc/tensor.cpp
View file

@ -172,7 +172,7 @@ extern "C" {
}
}
Tensor* sum_tensor(Tensor* tensor, int axis) {
Tensor* sum_tensor(Tensor* tensor, int axis, bool keepdim) {
char* device = (char*)malloc(strlen(tensor->device) + 1);
if (device != NULL) {
@ -192,29 +192,45 @@ extern "C" {
shape[0] = 1;
ndim = 1;
} else {
shape = (int*) malloc((tensor->ndim - 1) * sizeof(int));
for (int i = 0, j = 0; i < tensor->ndim; ++i) {
if (i != axis) {
shape[j++] = tensor->shape[i];
if (keepdim) {
shape = (int*) malloc((tensor->ndim) * sizeof(int));
for (int i = 0; i < tensor->ndim; i++) {
shape[i] = tensor->shape[i];
}
shape[axis] = 1;
ndim = tensor->ndim;
} else {
shape = (int*) malloc((tensor->ndim - 1) * sizeof(int));
for (int i = 0, j = 0; i < tensor->ndim; ++i) {
if (i != axis) {
shape[j++] = tensor->shape[i];
}
}
ndim = tensor->ndim - 1;
}
ndim = tensor->ndim - 1;
}
int size = 1;
for (int i = 0; i < ndim; i++) {
size *= shape[i];
}
if (strcmp(tensor->device, "cuda") == 0) {
float* result_data;
cudaMalloc((void**)&result_data, tensor->size * sizeof(float));
cudaMalloc((void**)&result_data, size * sizeof(float));
sum_tensor_cuda(tensor, result_data);
return create_tensor(result_data, shape, ndim, device);
}
else {
float* result_data = (float*)malloc(1 * sizeof(float));
float* result_data = (float*)malloc(size * sizeof(float));
if (result_data == NULL) {
fprintf(stderr, "Memory allocation failed\n");
exit(1);
}
sum_tensor_cpu(tensor, result_data, axis);
sum_tensor_cpu(tensor, result_data, size, shape, axis, keepdim);
return create_tensor(result_data, shape, ndim, device);
}
}

2
norch/csrc/tensor.h
View file

@ -16,7 +16,7 @@ extern "C" {
Tensor* create_tensor(float* data, int* shape, int ndim, char* device);
float get_item(Tensor* tensor, int* indices);
Tensor* add_tensor(Tensor* tensor1, Tensor* tensor2);
Tensor* sum_tensor(Tensor* tensor, int axis);
Tensor* sum_tensor(Tensor* tensor, int axis, bool keepdims);
Tensor* sub_tensor(Tensor* tensor1, Tensor* tensor2);
Tensor* elementwise_mul_tensor(Tensor* tensor1, Tensor* tensor2);
Tensor* scalar_mul_tensor(Tensor* tensor, float scalar);

BIN
norch/libtensor.so
View file

Binary file not shown.

BIN
norch/nn/modules/__pycache__/linear.cpython-38.pyc
View file

Binary file not shown.

22
norch/tensor.py
View file

@ -630,20 +630,28 @@ class Tensor:
return result_data
def sum(self, axis=-1):
Tensor._C.sum_tensor.argtypes = [ctypes.POINTER(CTensor), ctypes.c_int]
def sum(self, axis=-1, keepdim=False):
Tensor._C.sum_tensor.argtypes = [ctypes.POINTER(CTensor), ctypes.c_int, ctypes.c_bool]
Tensor._C.sum_tensor.restype = ctypes.POINTER(CTensor)
result_tensor_ptr = Tensor._C.sum_tensor(self.tensor, axis)
result_tensor_ptr = Tensor._C.sum_tensor(self.tensor, axis, keepdim)
result_data = Tensor()
result_data.tensor = result_tensor_ptr
if axis == -1:
result_data.shape = [1]
result_data.ndim = 1
if keepdim:
result_data.ndim = self.ndim
result_data.shape = [1] * self.ndim
else:
result_data.shape = [1]
result_data.ndim = 1
else:
result_data.shape = self.shape[:axis] + self.shape[axis+1:]
if keepdim:
result_data.shape = self.shape[:axis] + [1] + self.shape[axis+1:]
else:
result_data.shape = self.shape[:axis] + self.shape[axis+1:]
result_data.ndim = len(result_data.shape)
result_data.device = self.device
@ -653,7 +661,7 @@ class Tensor:
result_data.requires_grad = self.requires_grad
if result_data.requires_grad:
result_data.grad_fn = SumBackward(self, axis)
result_data.grad_fn = SumBackward(self, axis, keepdim=keepdim)
return result_data

24
test.py Normal file
View file

@ -0,0 +1,24 @@
import norch
W1 = norch.Tensor([[1], [2], [3], [4], [5], [6], [7], [8], [9], [10]], requires_grad=True) # A has shape 10x1
X = norch.Tensor([[1, 2, 3, 4, 5]]) # B has shape 1x5
B1 = norch.Tensor([[1, 2, 3, 4, 5], [2, 1, 2, 3, 4], [3, 1,2,3, 4], [4, 1, 1, 1, 1,], [5,1,1,1,1], [6,1,1,1,1], [7,1,1,1,1], [8,1,1,1,1], [9,1,1,1,1], [10,1,1,1,1]], requires_grad=True)
W2 = norch.Tensor([[1], [2], [3], [4], [5], [6], [7], [8], [9], [10]], requires_grad=True).T
B2 = norch.Tensor([[1, 2, 3, 4, 5]], requires_grad=True)
Z1 = W1 @ X + B1
# Perform matrix multiplication
Z2 = W2 @ Z1 + B2
print(Z2.shape)
l = Z2.sum()
l.backward()
#print(l)
print("Resulting matrix shape:", W1.grad)

18
tests/test_operations.py
View file

@ -249,6 +249,24 @@ class TestTensorOperations(unittest.TestCase):
self.assertTrue(utils.compare_torch(torch_result, torch_expected))
def test_sum_axis_keepdim(self):
"""
Test summation of a tensor along a specific axis with keepdim=True
"""
norch_tensor = norch.Tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device)
norch_result = norch_tensor.sum(axis=1, keepdim=True)
torch_result = utils.to_torch(norch_result).to(self.device)
torch_tensor = torch.tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device)
torch_expected = torch.sum(torch_tensor, dim=1, keepdim=True)
print(torch_result, '\n', torch_expected)
self.assertTrue(utils.compare_torch(torch_result, torch_expected))
def test_transpose_T(self):
"""
Test transposition of a tensor: tensor.T