From ef272b65eec3ae2ba160c011b180d0697d8ea80b Mon Sep 17 00:00:00 2001 From: lucasdelimanogueira Date: Fri, 24 May 2024 19:48:37 -0300 Subject: [PATCH] fix install nccl setup.py --- build/lib/norch/__init__.py | 9 + build/lib/norch/autograd/__init__.py | 1 + build/lib/norch/autograd/functions.py | 304 ++++ build/lib/norch/csrc/cpu.cpp | 443 +++++ build/lib/norch/csrc/cpu.h | 38 + build/lib/norch/csrc/cuda.cu | 1248 +++++++++++++ build/lib/norch/csrc/cuda.h | 104 ++ build/lib/norch/csrc/distributed.cpp | 65 + build/lib/norch/csrc/distributed.h | 28 + build/lib/norch/csrc/tensor.cpp | 1558 +++++++++++++++++ build/lib/norch/csrc/tensor.h | 45 + build/lib/norch/distributed/__init__.py | 1 + build/lib/norch/distributed/distributed.py | 24 + build/lib/norch/libtensor.so | Bin 0 -> 401208 bytes build/lib/norch/nn/__init__.py | 4 + build/lib/norch/nn/activation.py | 30 + build/lib/norch/nn/functional.py | 32 + build/lib/norch/nn/loss.py | 93 + build/lib/norch/nn/module.py | 107 ++ build/lib/norch/nn/modules/__init__.py | 1 + build/lib/norch/nn/modules/linear.py | 26 + build/lib/norch/nn/parameter.py | 11 + build/lib/norch/norchvision/__init__.py | 1 + .../norch/norchvision/datasets/__init__.py | 1 + build/lib/norch/norchvision/datasets/mnist.py | 90 + build/lib/norch/norchvision/transforms.py | 22 + build/lib/norch/optim/__init__.py | 1 + build/lib/norch/optim/optimizer.py | 22 + build/lib/norch/optim/optimizers/__init__.py | 1 + build/lib/norch/optim/optimizers/sgd.py | 27 + build/lib/norch/tensor.py | 1033 +++++++++++ build/lib/norch/utils/__init__.py | 2 + build/lib/norch/utils/data/__init__.py | 4 + build/lib/norch/utils/data/batch.py | 13 + build/lib/norch/utils/data/dataloader.py | 20 + build/lib/norch/utils/data/dataset.py | 74 + build/lib/norch/utils/data/example.py | 65 + build/lib/norch/utils/utils.py | 120 ++ build/lib/norch/utils/utils_unittests.py | 24 + build/lib/tests/__init__.py | 3 + build/lib/tests/test_autograd.py | 993 +++++++++++ build/lib/tests/test_nn.py | 204 +++ build/lib/tests/test_operations.py | 753 ++++++++ setup.py | 8 +- 44 files changed, 7650 insertions(+), 3 deletions(-) create mode 100644 build/lib/norch/__init__.py create mode 100644 build/lib/norch/autograd/__init__.py create mode 100644 build/lib/norch/autograd/functions.py create mode 100644 build/lib/norch/csrc/cpu.cpp create mode 100644 build/lib/norch/csrc/cpu.h create mode 100644 build/lib/norch/csrc/cuda.cu create mode 100644 build/lib/norch/csrc/cuda.h create mode 100644 build/lib/norch/csrc/distributed.cpp create mode 100644 build/lib/norch/csrc/distributed.h create mode 100644 build/lib/norch/csrc/tensor.cpp create mode 100644 build/lib/norch/csrc/tensor.h create mode 100644 build/lib/norch/distributed/__init__.py create mode 100644 build/lib/norch/distributed/distributed.py create mode 100755 build/lib/norch/libtensor.so create mode 100644 build/lib/norch/nn/__init__.py create mode 100644 build/lib/norch/nn/activation.py create mode 100644 build/lib/norch/nn/functional.py create mode 100644 build/lib/norch/nn/loss.py create mode 100644 build/lib/norch/nn/module.py create mode 100644 build/lib/norch/nn/modules/__init__.py create mode 100644 build/lib/norch/nn/modules/linear.py create mode 100644 build/lib/norch/nn/parameter.py create mode 100644 build/lib/norch/norchvision/__init__.py create mode 100644 build/lib/norch/norchvision/datasets/__init__.py create mode 100644 build/lib/norch/norchvision/datasets/mnist.py create mode 100644 build/lib/norch/norchvision/transforms.py create mode 100644 build/lib/norch/optim/__init__.py create mode 100644 build/lib/norch/optim/optimizer.py create mode 100644 build/lib/norch/optim/optimizers/__init__.py create mode 100644 build/lib/norch/optim/optimizers/sgd.py create mode 100644 build/lib/norch/tensor.py create mode 100644 build/lib/norch/utils/__init__.py create mode 100644 build/lib/norch/utils/data/__init__.py create mode 100644 build/lib/norch/utils/data/batch.py create mode 100644 build/lib/norch/utils/data/dataloader.py create mode 100644 build/lib/norch/utils/data/dataset.py create mode 100644 build/lib/norch/utils/data/example.py create mode 100644 build/lib/norch/utils/utils.py create mode 100644 build/lib/norch/utils/utils_unittests.py create mode 100644 build/lib/tests/__init__.py create mode 100644 build/lib/tests/test_autograd.py create mode 100644 build/lib/tests/test_nn.py create mode 100644 build/lib/tests/test_operations.py diff --git a/build/lib/norch/__init__.py b/build/lib/norch/__init__.py new file mode 100644 index 0000000..c2a3122 --- /dev/null +++ b/build/lib/norch/__init__.py @@ -0,0 +1,9 @@ +from norch.tensor import * +from .nn import * +from .optim import * +from .utils import * +from .norchvision import * + +__version__ = "0.0.4" +__author__ = 'Lucas de Lima Nogueira' +__credits__ = 'Lucas de Lima Nogueira' \ No newline at end of file diff --git a/build/lib/norch/autograd/__init__.py b/build/lib/norch/autograd/__init__.py new file mode 100644 index 0000000..e7d4dc0 --- /dev/null +++ b/build/lib/norch/autograd/__init__.py @@ -0,0 +1 @@ +from .functions import * \ No newline at end of file diff --git a/build/lib/norch/autograd/functions.py b/build/lib/norch/autograd/functions.py new file mode 100644 index 0000000..477837d --- /dev/null +++ b/build/lib/norch/autograd/functions.py @@ -0,0 +1,304 @@ +import math +import norch + +class AddBackward: + def __init__(self, x, y): + self.input = [x, y] + + def backward(self, gradient): + return [gradient, gradient] + +class AddBroadcastedBackward: + def __init__(self, x, y): + self.input = [x, y] + + def backward(self, gradient): + x, y = self.input + grad_x = self._reshape_gradient(gradient, x.shape) + grad_y = self._reshape_gradient(gradient, y.shape) + + return [grad_x, grad_y] + + def _reshape_gradient(self, gradient, shape): + # Reduce gradient dimensions to match the target shape dimensions + while len(gradient.shape) > len(shape): + gradient = gradient.sum(axis=0) + + # Sum along axes where the target shape dimension is 1 + for i in range(len(shape)): + if shape[i] == 1: + gradient = gradient.sum(axis=i, keepdim=True) + + return gradient + +class SubBackward: + def __init__(self, x, y): + self.input = [x, y] + + def backward(self, gradient): + return [gradient, -gradient] + +class SubBroadcastedBackward: + def __init__(self, x, y): + self.input = [x, y] + + def backward(self, gradient): + x, y = self.input + grad_x = self._reshape_gradient(gradient, x.shape) + grad_y = self._reshape_gradient(gradient, y.shape) + return [grad_x, -grad_y] + + def _reshape_gradient(self, gradient, shape): + # Reduce gradient dimensions to match the target shape dimensions + while len(gradient.shape) > len(shape): + gradient = gradient.sum(axis=0) + + # Sum along axes where the target shape dimension is 1 + for i in range(len(shape)): + if shape[i] == 1: + gradient = gradient.sum(axis=i) + return gradient + +class ScalarMulBackward: + def __init__(self, x, scalar): + self.input = [x] + self.scalar = scalar + + def backward(self, gradient): + return [gradient * self.scalar] + +class ElementwiseMulBackward: + def __init__(self, x, y): + self.input = [x, y] + + def backward(self, gradient): + x = self.input[0] + y = self.input[1] + return [y * gradient, x * gradient] + +class MatmulBackward: + def __init__(self, x, y): + self.input = [x, y] + + def backward(self, gradient): + x, y = self.input + + if x.ndim != y.ndim: # broadcasted case + aux = (gradient @ y.transpose(-1,-2)) + aux_sum = aux.sum(axis=0) + return [aux_sum, x.transpose(-1,-2) @ gradient] + else: + return [gradient @ y.transpose(-1,-2), x.transpose(-1,-2) @ gradient] + +"""class PowBackward: + def __init__(self, x, power): + self.input = [x] + self.power = power + + def backward(self, gradient): + return [(gradient * self.power) * (self.input[0]) ** (self.power - 1)]""" + +class PowBackward: + def __init__(self, base, exponent): + self.input = [base, exponent] + + def backward(self, gradient): + base, exponent = self.input[0], self.input[1] + + if isinstance(base, (int, float)): + grad_base = gradient * (base ** (exponent - 1)) + grad_exponent = (gradient * base ** exponent) * math.log(base) + + else: + grad_base = gradient * exponent * (base ** (exponent - 1)) + grad_exponent = (gradient * base ** exponent) * (base.log()) + + return [grad_base, grad_exponent] + + +class LogBackward: + def __init__(self, x): + self.input = [x] + + def backward(self, gradient): + + grad_input = gradient / self.input[0] + + return [grad_input] + +class SumBackward: + def __init__(self, x, axis=None, keepdim=False): + self.input = [x] + self.axis = axis + self.keepdim = keepdim + + def backward(self, gradient): + input_shape = self.input[0].shape.copy() + if self.axis == -1: + # If axis is None, sum reduces the tensor to a scalar. + grad_output = float(gradient[[0] * len(gradient.shape)]) * self.input[0].ones_like() + else: + + if self.keepdim: + input_shape = input_shape[:self.axis] + [1] + input_shape[self.axis+1:] + else: + input_shape = input_shape[:self.axis] + input_shape[self.axis+1:] + + # Broadcast the gradient to the input shape along the specified axis. + grad_output_shape = list(input_shape) + grad_output = gradient.reshape(grad_output_shape) + grad_output = grad_output + self.input[0].zeros_like() + + return [grad_output] + +class ReshapeBackward: + def __init__(self, x): + self.input = [x] + + def backward(self, gradient): + return [gradient.reshape(self.input[0].shape)] + +class TransposeBackward: + def __init__(self, x, axis1, axis2): + self.input = [x] + self.axis1 = axis1 + self.axis2 = axis2 + + def backward(self, gradient): + return [gradient.transpose(self.axis2, self.axis1)] + +class TBackward: + def __init__(self, x): + self.input = [x] + + def backward(self, gradient): + return [gradient.T] + + +class DivisionBackward: + def __init__(self, x, y): + self.input = [x, y] + + def backward(self, gradient): + x, y = self.input + grad_x = gradient / y + grad_y = -1 * gradient * (x / (y * y)) + + return [grad_x, grad_y] + +class SinBackward: + def __init__(self, x): + self.input = [x] + + def backward(self, gradient): + x = self.input[0] + return [gradient * x.cos()] + +class CosBackward: + def __init__(self, x): + self.input = [x] + + def backward(self, gradient): + x = self.input[0] + return [-gradient * x.sin()] + +class MaxBackward: + def __init__(self, x, axis=None, keepdim=False): + self.input = [x] + self.axis = axis + self.keepdim = keepdim + + def backward(self, gradient): + input_shape = self.input[0].shape.copy() + if self.axis == -1: + max_value = self.input[0].max() + mask = self.input[0].equal(max_value) + + grad_output = float(gradient[[0] * len(gradient.shape)]) * self.input[0].ones_like() + + grad_output = (grad_output * mask) / mask.sum()[0] + + else: + + if self.keepdim: + input_shape = input_shape[:self.axis] + [1] + input_shape[self.axis+1:] + else: + input_shape = input_shape[:self.axis] + input_shape[self.axis+1:] + + # Broadcast the gradient to the input shape along the specified axis. + grad_output_shape = list(input_shape) + + grad_output = gradient.reshape(grad_output_shape) + grad_output = grad_output + self.input[0].zeros_like() + max_values = self.input[0].max(axis=self.axis, keepdim=True) + mask = self.input[0].equal(max_values) + + grad_output = (grad_output * mask) + + return [grad_output] + + +class MinBackward: + def __init__(self, x, axis=None, keepdim=False): + self.input = [x] + self.axis = axis + self.keepdim = keepdim + + def backward(self, gradient): + input_shape = self.input[0].shape.copy() + if self.axis == -1: + min_value = self.input[0].min() + mask = self.input[0].equal(min_value) + + grad_output = float(gradient[[0] * len(gradient.shape)]) * self.input[0].ones_like() + + grad_output = (grad_output * mask) / mask.sum()[0] + + else: + if self.keepdim: + input_shape = input_shape[:self.axis] + [1] + input_shape[self.axis+1:] + else: + input_shape = input_shape[:self.axis] + input_shape[self.axis+1:] + + # Broadcast the gradient to the input shape along the specified axis. + grad_output_shape = list(input_shape) + grad_output = gradient.reshape(grad_output_shape) + grad_output = grad_output + self.input[0].zeros_like() + max_values = self.input[0].min(axis=self.axis, keepdim=True) + mask = self.input[0].equal(max_values) + + grad_output = (grad_output * mask) + + return [grad_output] + + +class CrossEntropyLossBackward: + def __init__(self, logits, targets): + self.input = [logits, targets] + + def backward(self, gradient): + logits, targets = self.input + + if logits.ndim == 1: + softmax = norch.softmax(logits, dim=0) + grad_logits = (softmax - targets) + + elif logits.ndim == 2: + # batched + batch_size = logits.shape[0] + softmax = norch.softmax(logits, dim=1) + + grad_logits = (softmax - targets) / batch_size + + return [grad_logits, None] # targets do not have a gradient + +class SigmoidBackward: + def __init__(self, input): + self.input = [input] + + def backward(self, gradient): + sigmoid_x = self.input[0].sigmoid() + grad_input = gradient * sigmoid_x * (1 - sigmoid_x) + + return [grad_input] + diff --git a/build/lib/norch/csrc/cpu.cpp b/build/lib/norch/csrc/cpu.cpp new file mode 100644 index 0000000..63e1615 --- /dev/null +++ b/build/lib/norch/csrc/cpu.cpp @@ -0,0 +1,443 @@ +#include "tensor.h" +#include +#include +#include +#include + +void add_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data) { + + for (int i = 0; i < tensor1->size; i++) { + result_data[i] = tensor1->data[i] + tensor2->data[i]; + } +} + +void add_broadcasted_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data, int* broadcasted_shape, int broadcasted_size) { + int max_ndim = tensor1->ndim > tensor2->ndim ? tensor1->ndim : tensor2->ndim; + + // Calculate strides for broadcasting + int* strides1 = (int*)malloc(max_ndim * sizeof(int)); + int* strides2 = (int*)malloc(max_ndim * sizeof(int)); + if (strides1 == NULL || strides2 == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + + int stride1 = 1, stride2 = 1; + for (int i = max_ndim - 1; i >= 0; i--) { + int dim1 = i < tensor1->ndim ? tensor1->shape[tensor1->ndim - max_ndim + i] : 1; + int dim2 = i < tensor2->ndim ? tensor2->shape[tensor2->ndim - max_ndim + i] : 1; + strides1[i] = dim1 == broadcasted_shape[i] ? stride1 : 0; + strides2[i] = dim2 == broadcasted_shape[i] ? stride2 : 0; + stride1 *= (dim1 == broadcasted_shape[i]) ? dim1 : 1; + stride2 *= (dim2 == broadcasted_shape[i]) ? dim2 : 1; + } + + // Perform element-wise addition with broadcasting + for (int i = 0; i < broadcasted_size; i++) { + int index1 = 0, index2 = 0; + int linear_index = i; + for (int j = max_ndim - 1; j >= 0; j--) { + int pos = linear_index % broadcasted_shape[j]; + linear_index /= broadcasted_shape[j]; + if (strides1[j] != 0) index1 += pos * strides1[j]; + if (strides2[j] != 0) index2 += pos * strides2[j]; + } + result_data[i] = tensor1->data[index1] + tensor2->data[index2]; + } + + // Free strides + free(strides1); + free(strides2); +} + + +void sub_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data) { + + for (int i = 0; i < tensor1->size; i++) { + result_data[i] = tensor1->data[i] - tensor2->data[i]; + } +} + +void sub_broadcasted_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data, int* broadcasted_shape, int broadcasted_size) { + int max_ndim = tensor1->ndim > tensor2->ndim ? tensor1->ndim : tensor2->ndim; + + // Calculate strides for broadcasting + int* strides1 = (int*)malloc(max_ndim * sizeof(int)); + int* strides2 = (int*)malloc(max_ndim * sizeof(int)); + if (strides1 == NULL || strides2 == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + + int stride1 = 1, stride2 = 1; + for (int i = max_ndim - 1; i >= 0; i--) { + int dim1 = i < tensor1->ndim ? tensor1->shape[tensor1->ndim - max_ndim + i] : 1; + int dim2 = i < tensor2->ndim ? tensor2->shape[tensor2->ndim - max_ndim + i] : 1; + strides1[i] = dim1 == broadcasted_shape[i] ? stride1 : 0; + strides2[i] = dim2 == broadcasted_shape[i] ? stride2 : 0; + stride1 *= (dim1 == broadcasted_shape[i]) ? dim1 : 1; + stride2 *= (dim2 == broadcasted_shape[i]) ? dim2 : 1; + } + + // Perform element-wise addition with broadcasting + for (int i = 0; i < broadcasted_size; i++) { + int index1 = 0, index2 = 0; + int linear_index = i; + for (int j = max_ndim - 1; j >= 0; j--) { + int pos = linear_index % broadcasted_shape[j]; + linear_index /= broadcasted_shape[j]; + if (strides1[j] != 0) index1 += pos * strides1[j]; + if (strides2[j] != 0) index2 += pos * strides2[j]; + } + result_data[i] = tensor1->data[index1] - tensor2->data[index2]; + } + + // Free strides + free(strides1); + free(strides2); +} + +void elementwise_mul_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data) { + + for (int i = 0; i < tensor1->size; i++) { + result_data[i] = tensor1->data[i] * tensor2->data[i]; + } +} + +void scalar_mul_tensor_cpu(Tensor* tensor, float scalar, float* result_data) { + + for (int i = 0; i < tensor->size; i++) { + result_data[i] = scalar * tensor->data[i]; + } +} + +void scalar_div_tensor_cpu(float scalar, Tensor* tensor, float* result_data) { + + for (int i = 0; i < tensor->size; i++) { + result_data[i] = scalar / tensor->data[i]; + } +} + +void tensor_div_scalar_cpu(Tensor* tensor, float scalar, float* result_data) { + + for (int i = 0; i < tensor->size; i++) { + result_data[i] = tensor->data[i] / scalar; + } +} + +void tensor_div_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data) { + + for (int i = 0; i < tensor1->size; i++) { + result_data[i] = tensor1->data[i] / tensor2->data[i]; + } +} + +void matmul_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data) { + + for (int i = 0; i < tensor1->shape[0]; i++) { + for (int j = 0; j < tensor2->shape[1]; j++) { + float sum = 0.0; + for (int k = 0; k < tensor1->shape[1]; k++) { + sum += tensor1->data[i * tensor1->shape[1] + k] * tensor2->data[k * tensor2->shape[1] + j]; + } + result_data[i * tensor2->shape[1] + j] = sum; + } + } +} + + +void broadcasted_batched_matmul_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data) { + + int result_data_stride = tensor1->shape[0] * tensor2->shape[2]; + + for (int batch = 0; batch < tensor2->shape[0]; batch++) { + + for (int i = 0; i < tensor1->shape[0]; i++) { + for (int j = 0; j < tensor2->shape[2]; j++) { + float sum = 0.0; + for (int k = 0; k < tensor1->shape[1]; k++) { + sum += tensor1->data[i * tensor1->shape[1] + k] * tensor2->data[batch*tensor2->strides[0] + (k * tensor2->shape[2] + j)]; + } + result_data[(batch * result_data_stride) + (i * tensor2->shape[2] + j)] = sum; + } + } + } +} + +void batched_matmul_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data) { + int result_data_stride = tensor1->shape[1] * tensor2->shape[2]; + + for (int batch = 0; batch < tensor2->shape[0]; batch++) { + + for (int i = 0; i < tensor1->shape[1]; i++) { + for (int j = 0; j < tensor2->shape[2]; j++) { + float sum = 0.0; + for (int k = 0; k < tensor1->shape[2]; k++) { + sum += tensor1->data[(batch * tensor1->strides[0]) + i * tensor1->shape[2] + k] * tensor2->data[batch*tensor2->strides[0] + (k * tensor2->shape[2] + j)]; + } + result_data[(batch * result_data_stride) + (i * tensor2->shape[2] + j)] = sum; + } + } + } +} + +void scalar_pow_tensor_cpu(float base, Tensor* tensor, float* result_data) { + + for (int i = 0; i < tensor->size; i++) { + result_data[i] = powf(base, tensor->data[i]); + } +} + +void tensor_pow_scalar_cpu(Tensor* tensor, float exponent, float* result_data) { + + for (int i = 0; i < tensor->size; i++) { + result_data[i] = powf(tensor->data[i], exponent); + } +} + +void log_tensor_cpu(Tensor* tensor, float* result_data) { + for (int i = 0; i < tensor->size; i++) { + result_data[i] = logf(tensor->data[i]); + } +} + +void sum_tensor_cpu(Tensor* tensor, float* result_data, int size, int* result_shape, int axis) { + if (axis == -1) { + // Sum over all elements + float sum = 0.0; + for (int i = 0; i < tensor->size; i++) { + sum += tensor->data[i]; + } + *result_data = sum; + } else { + if (axis < 0 || axis >= tensor->ndim) { + printf("Invalid axis"); + return; + } + + int axis_stride = tensor->strides[axis]; + + for (int i = 0; i < tensor->shape[axis]; i++) { + for (int j = 0; j < size; j++) { + int index = 0; + int remainder = j; + for (int k = tensor->ndim - 2; k >= 0; k--) { + index += (remainder % result_shape[k]) * tensor->strides[k < axis ? k : k + 1]; + remainder /= result_shape[k]; + } + result_data[j] += tensor->data[index + i * axis_stride]; + } + } + } +} + +void max_tensor_cpu(Tensor* tensor, float* result_data, int size, int* result_shape, int axis) { + if (axis == -1) { + float max_value = -INFINITY; + for (int i = 0; i < tensor->size; i++) { + max_value = fmax(max_value, tensor->data[i]); + } + *result_data = max_value; + } else { + for (int i = 0; i < size; i++) { + result_data[i] = -INFINITY; + } + if (axis < 0 || axis >= tensor->ndim) { + printf("Invalid axis"); + return; + } + + int axis_stride = tensor->strides[axis]; + + for (int i = 0; i < tensor->shape[axis]; i++) { + for (int j = 0; j < size; j++) { + int index = 0; + int remainder = j; + for (int k = tensor->ndim - 2; k >= 0; k--) { + index += (remainder % result_shape[k]) * tensor->strides[k < axis ? k : k + 1]; + remainder /= result_shape[k]; + } + result_data[j] = fmax(result_data[j], tensor->data[index + i * axis_stride]); + } + } + } +} + +void min_tensor_cpu(Tensor* tensor, float* result_data, int size, int* result_shape, int axis) { + if (axis == -1) { + float min_value = INFINITY; + for (int i = 0; i < tensor->size; i++) { + min_value = fmin(min_value, tensor->data[i]); + } + *result_data = min_value; + } else { + for (int i = 0; i < size; i++) { + result_data[i] = INFINITY; + } + if (axis < 0 || axis >= tensor->ndim) { + printf("Invalid axis"); + return; + } + + int axis_stride = tensor->strides[axis]; + + for (int i = 0; i < tensor->shape[axis]; i++) { + for (int j = 0; j < size; j++) { + int index = 0; + int remainder = j; + for (int k = tensor->ndim - 2; k >= 0; k--) { + index += (remainder % result_shape[k]) * tensor->strides[k < axis ? k : k + 1]; + remainder /= result_shape[k]; + } + result_data[j] = fmin(result_data[j], tensor->data[index + i * axis_stride]); + } + } + } +} + +void equal_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data) { + + for (int i = 0; i < tensor1->size; i++) { + result_data[i] = (tensor1->data[i] == tensor2->data[i]) ? 1.0f : 0.0f; + } +} + +void equal_broadcasted_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data, int* broadcasted_shape, int broadcasted_size) { + int max_ndim = tensor1->ndim > tensor2->ndim ? tensor1->ndim : tensor2->ndim; + + // Calculate strides for broadcasting + int* strides1 = (int*)malloc(max_ndim * sizeof(int)); + int* strides2 = (int*)malloc(max_ndim * sizeof(int)); + if (strides1 == NULL || strides2 == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + + int stride1 = 1, stride2 = 1; + for (int i = max_ndim - 1; i >= 0; i--) { + int dim1 = i < tensor1->ndim ? tensor1->shape[tensor1->ndim - max_ndim + i] : 1; + int dim2 = i < tensor2->ndim ? tensor2->shape[tensor2->ndim - max_ndim + i] : 1; + strides1[i] = dim1 == broadcasted_shape[i] ? stride1 : 0; + strides2[i] = dim2 == broadcasted_shape[i] ? stride2 : 0; + stride1 *= (dim1 == broadcasted_shape[i]) ? dim1 : 1; + stride2 *= (dim2 == broadcasted_shape[i]) ? dim2 : 1; + } + + // Perform element-wise equal with broadcasting + for (int i = 0; i < broadcasted_size; i++) { + int index1 = 0, index2 = 0; + int linear_index = i; + for (int j = max_ndim - 1; j >= 0; j--) { + int pos = linear_index % broadcasted_shape[j]; + linear_index /= broadcasted_shape[j]; + if (strides1[j] != 0) index1 += pos * strides1[j]; + if (strides2[j] != 0) index2 += pos * strides2[j]; + } + result_data[i] = (tensor1->data[index1] == tensor2->data[index2]) ? 1.0f : 0.0f; + } + + // Free strides + free(strides1); + free(strides2); +} + + +void ones_like_tensor_cpu(Tensor* tensor, float* result_data) { + + for (int i = 0; i < tensor->size; i++) { + result_data[i] = 1.0; + } +} + +void zeros_like_tensor_cpu(Tensor* tensor, float* result_data) { + + for (int i = 0; i < tensor->size; i++) { + result_data[i] = 0.0; + } +} + +void transpose_1D_tensor_cpu(Tensor* tensor, float* result_data) { + + for (int i = 0; i < tensor->shape[0]; i++) { + result_data[i] = tensor->data[i]; + } +} + +void transpose_2D_tensor_cpu(Tensor* tensor, float* result_data) { + int rows = tensor->shape[0]; + int cols = tensor->shape[1]; + + for (int i = 0; i < rows; i++) { + for (int j = 0; j < cols; j++) { + result_data[j * rows + i] = tensor->data[i * cols + j]; + } + } +} + +void transpose_3D_tensor_cpu(Tensor* tensor, float* result_data) { + int batch = tensor->shape[0]; + int rows = tensor->shape[1]; + int cols = tensor->shape[2]; + + for (int i = 0; i < batch; i++) { + for (int j = 0; j < rows; j++) { + for (int k = 0; k < cols; k++) { + result_data[k * rows * batch + j * batch + i] = tensor->data[i * rows * cols + j * cols + k]; + } + } + } +} + +void assign_tensor_cpu(Tensor* tensor, float* result_data) { + + for (int i = 0; i < tensor->size; i++) { + result_data[i] = tensor->data[i]; + } +} + +void make_contiguous_tensor_cpu(Tensor* tensor, float* result_data, int* new_strides) { + + for (int i = 0; i < tensor->size; i++) { + int index = 0; + int offset = i; + for (int j = 0; j < tensor->ndim; j++) { + index += (offset / new_strides[j]) * tensor->strides[j]; + offset %= new_strides[j]; + } + result_data[i] = tensor->data[index]; + } + + // Free old data and update tensor properties + free(tensor->data); + free(tensor->strides); + tensor->data = result_data; + tensor->strides = new_strides; +} + +void sin_tensor_cpu(Tensor* tensor, float* result_data) { + for (int i = 0; i < tensor->size; i++) { + result_data[i] = sinf(tensor->data[i]); + } +} + +void sigmoid_tensor_cpu(Tensor* tensor, float* result_data) { + for (int i = 0; i < tensor->size; i++) { + // avoid overflow + if (tensor->data[i] >= 0) { + + float z = expf(-tensor->data[i]); + result_data[i] = 1 / (1 + z); + + } else { + + float z = expf(tensor->data[i]); + result_data[i] = z / (1 + z); + } + } +} + +void cos_tensor_cpu(Tensor* tensor, float* result_data) { + for (int i = 0; i < tensor->size; i++) { + result_data[i] = cosf(tensor->data[i]); + } +} diff --git a/build/lib/norch/csrc/cpu.h b/build/lib/norch/csrc/cpu.h new file mode 100644 index 0000000..9a27a26 --- /dev/null +++ b/build/lib/norch/csrc/cpu.h @@ -0,0 +1,38 @@ +#ifndef CPU_H +#define CPU_H + +#include "tensor.h" + +void add_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data); +void add_broadcasted_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data, int* broadcasted_shape, int broadcasted_size); +void sum_tensor_cpu(Tensor* tensor, float* result_data, int size, int* shape, int axis); +void max_tensor_cpu(Tensor* tensor, float* result_data, int size, int* result_shape, int axis); +void min_tensor_cpu(Tensor* tensor, float* result_data, int size, int* result_shape, int axis); +void sub_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data); +void sub_broadcasted_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data, int* broadcasted_shape, int broadcasted_size); +void elementwise_mul_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data); +void scalar_div_tensor_cpu(float scalar, Tensor* tensor, float* result_data); +void tensor_div_scalar_cpu(Tensor* tensor, float scalar, float* result_data); +void tensor_div_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data); +void matmul_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data); +void broadcasted_batched_matmul_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data); +void batched_matmul_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data); +void scalar_pow_tensor_cpu(float base, Tensor* tensor, float* result_data); +void tensor_pow_scalar_cpu(Tensor* tensor, float exponent, float* result_data); +void log_tensor_cpu(Tensor* tensor, float* result_data); +void scalar_mul_tensor_cpu(Tensor* tensor, float scalar, float* result_data); +void equal_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data); +void equal_broadcasted_tensor_cpu(Tensor* tensor1, Tensor* tensor2, float* result_data, int* broadcasted_shape, int broadcasted_size); +void ones_like_tensor_cpu(Tensor* tensor, float* result_data); +void zeros_like_tensor_cpu(Tensor* tensor, float* result_data); +void transpose_1D_tensor_cpu(Tensor* tensor, float* result_data); +void transpose_2D_tensor_cpu(Tensor* tensor, float* result_data); +void transpose_3D_tensor_cpu(Tensor* tensor, float* result_data); +void transpose_axes_cpu(Tensor* tensor, float* result_data, int axis1, int axis2, int* new_shape); +void assign_tensor_cpu(Tensor* tensor, float* result_data); +void make_contiguous_tensor_cpu(Tensor* tensor, float* result_data, int* new_strides); +void sin_tensor_cpu(Tensor* tensor, float* result_data); +void cos_tensor_cpu(Tensor* tensor, float* result_data); +void sigmoid_tensor_cpu(Tensor* tensor, float* result_data); + +#endif /* CPU_H */ diff --git a/build/lib/norch/csrc/cuda.cu b/build/lib/norch/csrc/cuda.cu new file mode 100644 index 0000000..e39c20c --- /dev/null +++ b/build/lib/norch/csrc/cuda.cu @@ -0,0 +1,1248 @@ +#include "tensor.h" +#include +#include +#include +#include + +#define THREADS_PER_BLOCK 128 +#define TILE_SIZE 32 + +__host__ void cpu_to_cuda(Tensor* tensor, int device_id) { + + int deviceCount; + cudaGetDeviceCount(&deviceCount); + if (device_id >= deviceCount) { + fprintf(stderr, "Could not send tensor to device %d, only %d devices available\n", device_id, deviceCount); + exit(1); + } + + cudaSetDevice(device_id); + + float* data_tmp; + cudaMalloc((void **)&data_tmp, tensor->size * sizeof(float)); + cudaMemcpy(data_tmp, tensor->data, tensor->size * sizeof(float), cudaMemcpyHostToDevice); + + tensor->data = data_tmp; + + const char* device_str = "cuda"; + tensor->device = (char*)malloc(strlen(device_str) + 1); + strcpy(tensor->device, device_str); +} + +__host__ void cuda_to_cpu(Tensor* tensor) { + float* data_tmp = (float*)malloc(tensor->size * sizeof(float)); + + cudaMemcpy(data_tmp, tensor->data, tensor->size * sizeof(float), cudaMemcpyDeviceToHost); + cudaFree(tensor->data); + + tensor->data = data_tmp; + + const char* device_str = "cpu"; + tensor->device = (char*)malloc(strlen(device_str) + 1); + strcpy(tensor->device, device_str); + + printf("Successfully sent tensor to: %s\n", tensor->device); +} + +__global__ void add_tensor_cuda_kernel(float* data1, float* data2, float* result_data, int size) { + + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i < size) { + result_data[i] = data1[i] + data2[i]; + } +} + +__host__ void add_tensor_cuda(Tensor* tensor1, Tensor* tensor2, float* result_data) { + + int number_of_blocks = (tensor1->size + THREADS_PER_BLOCK - 1) / THREADS_PER_BLOCK; + add_tensor_cuda_kernel<<>>(tensor1->data, tensor2->data, result_data, tensor1->size); + + cudaError_t error = cudaGetLastError(); + if (error != cudaSuccess) { + printf("CUDA error: %s\n", cudaGetErrorString(error)); + exit(-1); + } + + cudaDeviceSynchronize(); +} + +__global__ void add_broadcasted_tensor_cuda_kernel(float* data1, float* data2, float* result_data, int* broadcasted_shape, int* strides1, int*strides2, int max_ndim, int size) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= size) return; + + int index1 = 0, index2 = 0; + int linear_index = i; + for (int j = max_ndim - 1; j >= 0; j--) { + int pos = linear_index % broadcasted_shape[j]; + linear_index /= broadcasted_shape[j]; + if (strides1[j] != 0) index1 += pos * strides1[j]; + if (strides2[j] != 0) index2 += pos * strides2[j]; + } + result_data[i] = data1[index1] + data2[index2]; +} + +__host__ void add_broadcasted_tensor_cuda(Tensor* tensor1, Tensor* tensor2, float* result_data, int* broadcasted_shape, int broadcasted_size) { + int max_ndim = tensor1->ndim > tensor2->ndim ? tensor1->ndim : tensor2->ndim; + + int* strides1 = (int*)malloc(max_ndim * sizeof(int)); + int* strides2 = (int*)malloc(max_ndim * sizeof(int)); + if (strides1 == NULL || strides2 == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + + int stride1 = 1, stride2 = 1; + for (int i = max_ndim - 1; i >= 0; i--) { + int dim1 = i < tensor1->ndim ? tensor1->shape[tensor1->ndim - max_ndim + i] : 1; + int dim2 = i < tensor2->ndim ? tensor2->shape[tensor2->ndim - max_ndim + i] : 1; + strides1[i] = dim1 == broadcasted_shape[i] ? stride1 : 0; + strides2[i] = dim2 == broadcasted_shape[i] ? stride2 : 0; + stride1 *= (dim1 == broadcasted_shape[i]) ? dim1 : 1; + stride2 *= (dim2 == broadcasted_shape[i]) ? dim2 : 1; + } + + int* d_broadcasted_shape; + int* d_strides1; + int* d_strides2; + + cudaMalloc((void**)&d_broadcasted_shape, max_ndim * sizeof(int)); + cudaMemcpy(d_broadcasted_shape, broadcasted_shape, max_ndim * sizeof(int), cudaMemcpyHostToDevice); + + cudaMalloc((void**)&d_strides1, max_ndim * sizeof(int)); + cudaMemcpy(d_strides1, strides1, max_ndim * sizeof(int), cudaMemcpyHostToDevice); + + cudaMalloc((void**)&d_strides2, max_ndim * sizeof(int)); + cudaMemcpy(d_strides2, strides2, max_ndim * sizeof(int), cudaMemcpyHostToDevice); + + int number_of_blocks = (broadcasted_size + THREADS_PER_BLOCK - 1) / THREADS_PER_BLOCK; + add_broadcasted_tensor_cuda_kernel<<>>(tensor1->data, tensor2->data, result_data, d_broadcasted_shape, d_strides1, d_strides2, max_ndim, broadcasted_size); + + cudaError_t error = cudaGetLastError(); + if (error != cudaSuccess) { + printf("CUDA error: %s\n", cudaGetErrorString(error)); + exit(-1); + } + + cudaDeviceSynchronize(); + cudaFree(d_broadcasted_shape); +} + +__global__ void sum_tensor_cuda_kernel(float* data, float* result_data, int size) { + __shared__ float partial_sum[THREADS_PER_BLOCK * sizeof(float)]; + + int tid = threadIdx.x; + int i = blockIdx.x * blockDim.x + threadIdx.x; + + partial_sum[tid] = (i < size) ? data[i] : 0; + + __syncthreads(); + + // Perform block-wise reduction + for (int s = blockDim.x / 2; s > 0; s >>= 1) { + if (tid < s) { + partial_sum[tid] += partial_sum[tid + s]; + } + __syncthreads(); + } + + // Write block sum to global memory + if (tid == 0) { + result_data[blockIdx.x] = partial_sum[0]; + } +} + +__global__ void sum_tensor_cuda_kernel_axis(float* data, float* result_data, int* strides, int* shape, int axis, int ndim, int axis_stride, int size, int result_size) { + int tid = blockIdx.x * blockDim.x + threadIdx.x; + + if (tid < result_size) { + for (int i = 0; i < shape[axis]; i++) { + int index = 0; + int remainder = tid; + for (int k = ndim - 2; k >= 0; k--) { + index += (remainder % shape[k < axis ? k : k + 1]) * strides[k < axis ? k : k + 1]; + remainder /= shape[k < axis ? k : k + 1]; + } + index += i * axis_stride; + + atomicAdd(&result_data[tid], data[index]); + } + } +} + + +__host__ void sum_tensor_cuda(Tensor* tensor, float* result_data, int axis) { + + if (axis == -1) { + cudaMemcpy(result_data, tensor->data, tensor->size * sizeof(float), cudaMemcpyHostToDevice); + + int num_blocks = (tensor->size + THREADS_PER_BLOCK - 1) / THREADS_PER_BLOCK; + + // First-level reduction + sum_tensor_cuda_kernel<<>>(tensor->data, result_data, tensor->size); + + // If necessary, perform multiple levels of reduction + while (num_blocks > 1) { + int num_blocks_next = (num_blocks + THREADS_PER_BLOCK - 1) / THREADS_PER_BLOCK; + sum_tensor_cuda_kernel<<>>(result_data, result_data, num_blocks); + num_blocks = num_blocks_next; + } + + cudaError_t error = cudaGetLastError(); + if (error != cudaSuccess) { + printf("CUDA error: %s\n", cudaGetErrorString(error)); + exit(-1); + } + + cudaDeviceSynchronize(); + + } else { + int axis_stride = tensor->strides[axis]; + + // Calculate the size of the resulting tensor + int result_size = 1; + for (int i = 0; i < tensor->ndim; i++) { + if (i != axis) { + result_size *= tensor->shape[i]; + } + } + + // Allocate memory for strides and shape on the device + int* d_strides; + int* d_shape; + cudaMalloc(&d_strides, tensor->ndim * sizeof(int)); + cudaMalloc(&d_shape, tensor->ndim * sizeof(int)); + cudaMemcpy(d_strides, tensor->strides, tensor->ndim * sizeof(int), cudaMemcpyHostToDevice); + cudaMemcpy(d_shape, tensor->shape, tensor->ndim * sizeof(int), cudaMemcpyHostToDevice); + + // Initialize result_data to 0 + cudaMemset(result_data, 0, result_size * sizeof(float)); + + int num_threads = result_size; + int num_blocks = (num_threads + THREADS_PER_BLOCK - 1) / THREADS_PER_BLOCK; + sum_tensor_cuda_kernel_axis<<>>(tensor->data, result_data, d_strides, d_shape, axis, tensor->ndim, axis_stride, tensor->size, result_size); + + cudaError_t error = cudaGetLastError(); + if (error != cudaSuccess) { + printf("CUDA error: %s\n", cudaGetErrorString(error)); + exit(-1); + } + + cudaDeviceSynchronize(); + + // Free allocated memory + cudaFree(d_strides); + cudaFree(d_shape); + } +} + +__global__ void max_tensor_cuda_kernel(float* data, float* result_data, int size) { + __shared__ float partial_max[THREADS_PER_BLOCK * sizeof(float)]; + + int tid = threadIdx.x; + int i = blockIdx.x * blockDim.x + threadIdx.x; + + partial_max[tid] = (i < size) ? data[i] : -FLT_MAX; + + __syncthreads(); + + // Perform block-wise reduction + for (int s = blockDim.x / 2; s > 0; s >>= 1) { + if (tid < s) { + partial_max[tid] = fmax(partial_max[tid], partial_max[tid + s]); + } + __syncthreads(); + } + + // Write block sum to global memory + if (tid == 0) { + result_data[blockIdx.x] = partial_max[0]; + } +} + +__device__ float atomicMaxFloat(float* address, float val) { + int* address_as_int = (int*)address; + int old = *address_as_int, assumed; + + do { + assumed = old; + old = atomicCAS(address_as_int, assumed, + __float_as_int(fmaxf(val, __int_as_float(assumed)))); + } while (assumed != old); + + return __int_as_float(old); +} + +__global__ void max_tensor_cuda_kernel_axis(float* data, float* result_data, int* strides, int* shape, int axis, int ndim, int axis_stride, int size, int result_size) { + int tid = blockIdx.x * blockDim.x + threadIdx.x; + + if (tid < result_size) { + for (int i = 0; i < shape[axis]; i++) { + int index = 0; + int remainder = tid; + for (int k = ndim - 2; k >= 0; k--) { + index += (remainder % shape[k < axis ? k : k + 1]) * strides[k < axis ? k : k + 1]; + remainder /= shape[k < axis ? k : k + 1]; + } + index += i * axis_stride; + + atomicMaxFloat(&result_data[tid], data[index]); + } + } +} + +__host__ void max_tensor_cuda(Tensor* tensor, float* result_data, int axis) { + + if (axis == -1) { + cudaMemcpy(result_data, tensor->data, tensor->size * sizeof(float), cudaMemcpyHostToDevice); + + int num_blocks = (tensor->size + THREADS_PER_BLOCK - 1) / THREADS_PER_BLOCK; + + // First-level reduction + max_tensor_cuda_kernel<<>>(tensor->data, result_data, tensor->size); + + // If necessary, perform multiple levels of reduction + while (num_blocks > 1) { + int num_blocks_next = (num_blocks + THREADS_PER_BLOCK - 1) / THREADS_PER_BLOCK; + max_tensor_cuda_kernel<<>>(result_data, result_data, num_blocks); + num_blocks = num_blocks_next; + } + + cudaError_t error = cudaGetLastError(); + if (error != cudaSuccess) { + printf("CUDA error: %s\n", cudaGetErrorString(error)); + exit(-1); + } + + cudaDeviceSynchronize(); + + } else { + int axis_stride = tensor->strides[axis]; + + // Calculate the size of the resulting tensor + int result_size = 1; + for (int i = 0; i < tensor->ndim; i++) { + if (i != axis) { + result_size *= tensor->shape[i]; + } + } + + // Allocate memory for strides and shape on the device + int* d_strides; + int* d_shape; + cudaMalloc(&d_strides, tensor->ndim * sizeof(int)); + cudaMalloc(&d_shape, tensor->ndim * sizeof(int)); + cudaMemcpy(d_strides, tensor->strides, tensor->ndim * sizeof(int), cudaMemcpyHostToDevice); + cudaMemcpy(d_shape, tensor->shape, tensor->ndim * sizeof(int), cudaMemcpyHostToDevice); + + // Initialize result_data to 0 + float neg_inf = -FLT_MAX; + cudaMemset(result_data, *reinterpret_cast(&neg_inf), result_size * sizeof(float)); + + int num_threads = result_size; + int num_blocks = (num_threads + THREADS_PER_BLOCK - 1) / THREADS_PER_BLOCK; + max_tensor_cuda_kernel_axis<<>>(tensor->data, result_data, d_strides, d_shape, axis, tensor->ndim, axis_stride, tensor->size, result_size); + + cudaError_t error = cudaGetLastError(); + if (error != cudaSuccess) { + printf("CUDA error: %s\n", cudaGetErrorString(error)); + exit(-1); + } + + cudaDeviceSynchronize(); + + // Free allocated memory + cudaFree(d_strides); + cudaFree(d_shape); + } +} + +__global__ void min_tensor_cuda_kernel(float* data, float* result_data, int size) { + __shared__ float partial_min[THREADS_PER_BLOCK * sizeof(float)]; + + int tid = threadIdx.x; + int i = blockIdx.x * blockDim.x + threadIdx.x; + + partial_min[tid] = (i < size) ? data[i] : FLT_MAX; + + __syncthreads(); + + // Perform block-wise reduction + for (int s = blockDim.x / 2; s > 0; s >>= 1) { + if (tid < s) { + partial_min[tid] = fmin(partial_min[tid], partial_min[tid + s]); + } + __syncthreads(); + } + + // Write block sum to global memory + if (tid == 0) { + result_data[blockIdx.x] = partial_min[0]; + } +} + +__device__ float atomicMinFloat(float* address, float val) { + int* address_as_int = (int*)address; + int old = *address_as_int, assumed; + + do { + assumed = old; + old = atomicCAS(address_as_int, assumed, + __float_as_int(fminf(val, __int_as_float(assumed)))); + } while (assumed != old); + + return __int_as_float(old); +} + +__global__ void min_tensor_cuda_kernel_axis(float* data, float* result_data, int* strides, int* shape, int axis, int ndim, int axis_stride, int size, int result_size) { + int tid = blockIdx.x * blockDim.x + threadIdx.x; + + if (tid < result_size) { + for (int i = 0; i < shape[axis]; i++) { + int index = 0; + int remainder = tid; + for (int k = ndim - 2; k >= 0; k--) { + index += (remainder % shape[k < axis ? k : k + 1]) * strides[k < axis ? k : k + 1]; + remainder /= shape[k < axis ? k : k + 1]; + } + index += i * axis_stride; + + atomicMinFloat(&result_data[tid], data[index]); + } + } +} + +__host__ void min_tensor_cuda(Tensor* tensor, float* result_data, int axis) { + + if (axis == -1) { + cudaMemcpy(result_data, tensor->data, tensor->size * sizeof(float), cudaMemcpyHostToDevice); + + int num_blocks = (tensor->size + THREADS_PER_BLOCK - 1) / THREADS_PER_BLOCK; + + // First-level reduction + min_tensor_cuda_kernel<<>>(tensor->data, result_data, tensor->size); + + // If necessary, perform multiple levels of reduction + while (num_blocks > 1) { + int num_blocks_next = (num_blocks + THREADS_PER_BLOCK - 1) / THREADS_PER_BLOCK; + min_tensor_cuda_kernel<<>>(result_data, result_data, num_blocks); + num_blocks = num_blocks_next; + } + + cudaError_t error = cudaGetLastError(); + if (error != cudaSuccess) { + printf("CUDA error: %s\n", cudaGetErrorString(error)); + exit(-1); + } + + cudaDeviceSynchronize(); + + } else { + int axis_stride = tensor->strides[axis]; + + // Calculate the size of the resulting tensor + int result_size = 1; + for (int i = 0; i < tensor->ndim; i++) { + if (i != axis) { + result_size *= tensor->shape[i]; + } + } + + // Allocate memory for strides and shape on the device + int* d_strides; + int* d_shape; + cudaMalloc(&d_strides, tensor->ndim * sizeof(int)); + cudaMalloc(&d_shape, tensor->ndim * sizeof(int)); + cudaMemcpy(d_strides, tensor->strides, tensor->ndim * sizeof(int), cudaMemcpyHostToDevice); + cudaMemcpy(d_shape, tensor->shape, tensor->ndim * sizeof(int), cudaMemcpyHostToDevice); + + // Initialize result_data to 0 + float inf = FLT_MAX; + cudaMemset(result_data, *reinterpret_cast(&inf), result_size * sizeof(float)); + + int num_threads = result_size; + int num_blocks = (num_threads + THREADS_PER_BLOCK - 1) / THREADS_PER_BLOCK; + min_tensor_cuda_kernel_axis<<>>(tensor->data, result_data, d_strides, d_shape, axis, tensor->ndim, axis_stride, tensor->size, result_size); + + cudaError_t error = cudaGetLastError(); + if (error != cudaSuccess) { + printf("CUDA error: %s\n", cudaGetErrorString(error)); + exit(-1); + } + + cudaDeviceSynchronize(); + + // Free allocated memory + cudaFree(d_strides); + cudaFree(d_shape); + } +} + + + +__global__ void sub_tensor_cuda_kernel(float* data1, float* data2, float* result_data, int size) { + + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i < size) { + result_data[i] = data1[i] - data2[i]; + } +} + +__host__ void sub_tensor_cuda(Tensor* tensor1, Tensor* tensor2, float* result_data) { + + int number_of_blocks = (tensor1->size + THREADS_PER_BLOCK - 1) / THREADS_PER_BLOCK; + sub_tensor_cuda_kernel<<>>(tensor1->data, tensor2->data, result_data, tensor1->size); + + cudaError_t error = cudaGetLastError(); + if (error != cudaSuccess) { + printf("CUDA error: %s\n", cudaGetErrorString(error)); + exit(-1); + } + + cudaDeviceSynchronize(); +} + +__global__ void sub_broadcasted_tensor_cuda_kernel(float* data1, float* data2, float* result_data, int* broadcasted_shape, int* strides1, int*strides2, int max_ndim, int size) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= size) return; + + int index1 = 0, index2 = 0; + int linear_index = i; + for (int j = max_ndim - 1; j >= 0; j--) { + int pos = linear_index % broadcasted_shape[j]; + linear_index /= broadcasted_shape[j]; + if (strides1[j] != 0) index1 += pos * strides1[j]; + if (strides2[j] != 0) index2 += pos * strides2[j]; + } + result_data[i] = data1[index1] - data2[index2]; +} + +__host__ void sub_broadcasted_tensor_cuda(Tensor* tensor1, Tensor* tensor2, float* result_data, int* broadcasted_shape, int broadcasted_size) { + int max_ndim = tensor1->ndim > tensor2->ndim ? tensor1->ndim : tensor2->ndim; + + int* strides1 = (int*)malloc(max_ndim * sizeof(int)); + int* strides2 = (int*)malloc(max_ndim * sizeof(int)); + if (strides1 == NULL || strides2 == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + + int stride1 = 1, stride2 = 1; + for (int i = max_ndim - 1; i >= 0; i--) { + int dim1 = i < tensor1->ndim ? tensor1->shape[tensor1->ndim - max_ndim + i] : 1; + int dim2 = i < tensor2->ndim ? tensor2->shape[tensor2->ndim - max_ndim + i] : 1; + strides1[i] = dim1 == broadcasted_shape[i] ? stride1 : 0; + strides2[i] = dim2 == broadcasted_shape[i] ? stride2 : 0; + stride1 *= (dim1 == broadcasted_shape[i]) ? dim1 : 1; + stride2 *= (dim2 == broadcasted_shape[i]) ? dim2 : 1; + } + + int* d_broadcasted_shape; + int* d_strides1; + int* d_strides2; + + cudaMalloc((void**)&d_broadcasted_shape, max_ndim * sizeof(int)); + cudaMemcpy(d_broadcasted_shape, broadcasted_shape, max_ndim * sizeof(int), cudaMemcpyHostToDevice); + + cudaMalloc((void**)&d_strides1, max_ndim * sizeof(int)); + cudaMemcpy(d_strides1, strides1, max_ndim * sizeof(int), cudaMemcpyHostToDevice); + + cudaMalloc((void**)&d_strides2, max_ndim * sizeof(int)); + cudaMemcpy(d_strides2, strides2, max_ndim * sizeof(int), cudaMemcpyHostToDevice); + + int number_of_blocks = (broadcasted_size + THREADS_PER_BLOCK - 1) / THREADS_PER_BLOCK; + sub_broadcasted_tensor_cuda_kernel<<>>(tensor1->data, tensor2->data, result_data, d_broadcasted_shape, d_strides1, d_strides2, max_ndim, broadcasted_size); + + cudaError_t error = cudaGetLastError(); + if (error != cudaSuccess) { + printf("CUDA error: %s\n", cudaGetErrorString(error)); + exit(-1); + } + + cudaDeviceSynchronize(); + cudaFree(d_broadcasted_shape); +} + +__global__ void elementwise_mul_tensor_cuda_kernel(float* data1, float* data2, float* result_data, int size) { + + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i < size) { + result_data[i] = data1[i] * data2[i]; + } +} + +__host__ void elementwise_mul_tensor_cuda(Tensor* tensor1, Tensor* tensor2, float* result_data) { + + int number_of_blocks = (tensor1->size + THREADS_PER_BLOCK - 1) / THREADS_PER_BLOCK; + elementwise_mul_tensor_cuda_kernel<<>>(tensor1->data, tensor2->data, result_data, tensor1->size); + + cudaError_t error = cudaGetLastError(); + if (error != cudaSuccess) { + printf("CUDA error: %s\n", cudaGetErrorString(error)); + exit(-1); + } + + cudaDeviceSynchronize(); +} + +__global__ void scalar_mul_tensor_cuda_kernel(float* data, float scalar, float* result_data, int size) { + + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i < size) { + result_data[i] = scalar * data[i]; + } +} + +__host__ void scalar_mul_tensor_cuda(Tensor* tensor, float scalar, float* result_data) { + + int number_of_blocks = (tensor->size + THREADS_PER_BLOCK - 1) / THREADS_PER_BLOCK; + scalar_mul_tensor_cuda_kernel<<>>(tensor->data, scalar, result_data, tensor->size); + + cudaError_t error = cudaGetLastError(); + if (error != cudaSuccess) { + printf("CUDA error: %s\n", cudaGetErrorString(error)); + exit(-1); + } + + cudaDeviceSynchronize(); +} + +__global__ void scalar_div_tensor_cuda_kernel(float scalar, float* data, float* result_data, int size) { + + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i < size) { + result_data[i] = scalar / data[i]; + } +} + +__host__ void scalar_div_tensor_cuda(float scalar, Tensor* tensor, float* result_data) { + + int number_of_blocks = (tensor->size + THREADS_PER_BLOCK - 1) / THREADS_PER_BLOCK; + scalar_div_tensor_cuda_kernel<<>>(scalar, tensor->data, result_data, tensor->size); + + cudaError_t error = cudaGetLastError(); + if (error != cudaSuccess) { + printf("CUDA error: %s\n", cudaGetErrorString(error)); + exit(-1); + } + + cudaDeviceSynchronize(); +} + +__global__ void tensor_div_scalar_cuda_kernel(float* data, float scalar, float* result_data, int size) { + + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i < size) { + result_data[i] = data[i] / scalar; + } +} + +__host__ void tensor_div_scalar_cuda(Tensor* tensor, float scalar, float* result_data) { + + int number_of_blocks = (tensor->size + THREADS_PER_BLOCK - 1) / THREADS_PER_BLOCK; + tensor_div_scalar_cuda_kernel<<>>(tensor->data, scalar, result_data, tensor->size); + + cudaError_t error = cudaGetLastError(); + if (error != cudaSuccess) { + printf("CUDA error: %s\n", cudaGetErrorString(error)); + exit(-1); + } + + cudaDeviceSynchronize(); +} + +__global__ void tensor_div_tensor_cuda_kernel(float* data1, float* data2, float* result_data, int size) { + + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i < size) { + result_data[i] = data1[i] / data2[i]; + } +} + +__host__ void tensor_div_tensor_cuda(Tensor* tensor1, Tensor* tensor2, float* result_data) { + + int number_of_blocks = (tensor1->size + THREADS_PER_BLOCK - 1) / THREADS_PER_BLOCK; + tensor_div_tensor_cuda_kernel<<>>(tensor1->data, tensor2->data, result_data, tensor1->size); + + cudaError_t error = cudaGetLastError(); + if (error != cudaSuccess) { + printf("CUDA error: %s\n", cudaGetErrorString(error)); + exit(-1); + } + + cudaDeviceSynchronize(); +} + +/*__global__ void matmul_tensor_cuda_kernel(float* data1, float* data2, float* result_data, int rows1, int cols1, int cols2) { + + int row = blockIdx.y * blockDim.y + threadIdx.y; + int col = blockIdx.x * blockDim.x + threadIdx.x; + + if (row < rows1 && col < cols2) { + float sum = 0.0; + for (int k = 0; k < cols1; k++) { + sum += data1[row * cols1 + k] * data2[k * cols2 + col]; + } + result_data[row * cols2 + col] = sum; + } + +}*/ + +__global__ void matmul_tensor_cuda_kernel(float* data1, float* data2, float* result_data, int rows1, int cols1, int cols2) { + + // Shared memory for tiles + __shared__ float tile1[TILE_SIZE][TILE_SIZE]; + __shared__ float tile2[TILE_SIZE][TILE_SIZE]; + + // Thread indices + int tx = threadIdx.x; + int ty = threadIdx.y; + + // Output position + int row = blockIdx.y * blockDim.y + threadIdx.y; + int col = blockIdx.x * blockDim.x + threadIdx.x; + + float sum = 0.0; + + // Iterate over tiles + for (int i = 0; i < (cols1 + TILE_SIZE - 1) / TILE_SIZE; ++i) { + + // Load tiles into shared memory + if (row < rows1 && i * TILE_SIZE + tx < cols1) + tile1[ty][tx] = data1[row * cols1 + i * TILE_SIZE + tx]; + else + tile1[ty][tx] = 0.0; + + if (col < cols2 && i * TILE_SIZE + ty < cols1) + tile2[ty][tx] = data2[(i * TILE_SIZE + ty) * cols2 + col]; + else + tile2[ty][tx] = 0.0; + + // Synchronize threads + __syncthreads(); + + // Accumulate sum + for (int k = 0; k < TILE_SIZE; ++k) + sum += tile1[ty][k] * tile2[k][tx]; + + // Synchronize threads + __syncthreads(); + } + + // Write result to global memory + if (row < rows1 && col < cols2) + result_data[row * cols2 + col] = sum; +} + + +__host__ void matmul_tensor_cuda(Tensor* tensor1, Tensor* tensor2, float* result_data) { + + int rows1 = tensor1->shape[0]; + int cols1 = tensor1->shape[1]; + int cols2 = tensor2->shape[1]; + + dim3 threadsPerBlock(16, 16); + dim3 number_of_blocks((cols2 + threadsPerBlock.x - 1) / threadsPerBlock.x, (rows1 + threadsPerBlock.y - 1) / threadsPerBlock.y); + matmul_tensor_cuda_kernel<<>>(tensor1->data, tensor2->data, result_data, rows1, cols1, cols2); + + + cudaError_t error = cudaGetLastError(); + if (error != cudaSuccess) { + printf("CUDA error: %s\n", cudaGetErrorString(error)); + exit(-1); + } + + cudaDeviceSynchronize(); +} + +__global__ void batched_matmul_tensor_cuda_kernel(float* data1, float* data2, float* result_data, int batch_size, int rows1, int cols1, int cols2) { + int batch = blockIdx.z; + int row = blockIdx.y * blockDim.y + threadIdx.y; + int col = blockIdx.x * blockDim.x + threadIdx.x; + + if (row < rows1 && col < cols2) { + float sum = 0.0f; + for (int k = 0; k < cols1; ++k) { + sum += data1[batch * rows1 * cols1 + row * cols1 + k] * + data2[batch * cols1 * cols2 + k * cols2 + col]; + } + result_data[batch * rows1 * cols2 + row * cols2 + col] = sum; + } +} + +__host__ void batched_matmul_tensor_cuda(Tensor* tensor1, Tensor* tensor2, float* result_data) { + + int batch_size = tensor2->shape[0]; + int rows1 = tensor1->shape[1]; + int cols1 = tensor1->shape[2]; + int cols2 = tensor2->shape[2]; + + dim3 threadsPerBlock(16, 16); + dim3 number_of_blocks((cols2 + threadsPerBlock.x - 1) / threadsPerBlock.x, (rows1 + threadsPerBlock.y - 1) / threadsPerBlock.y, batch_size); + batched_matmul_tensor_cuda_kernel<<>>(tensor1->data, tensor2->data, result_data, batch_size, rows1, cols1, cols2); + + + cudaError_t error = cudaGetLastError(); + if (error != cudaSuccess) { + printf("CUDA error: %s\n", cudaGetErrorString(error)); + exit(-1); + } + + cudaDeviceSynchronize(); +} + +__global__ void broadcasted_batched_matmul_tensor_cuda_kernel(float* data1, float* data2, float* result_data, int batch_size, int rows1, int cols1, int cols2) { + int batch = blockIdx.z; + int row = blockIdx.y * blockDim.y + threadIdx.y; + int col = blockIdx.x * blockDim.x + threadIdx.x; + + if (row < rows1 && col < cols2) { + float sum = 0.0f; + for (int k = 0; k < cols1; ++k) { + sum += data1[row * cols1 + k] * + data2[batch * cols1 * cols2 + k * cols2 + col]; + } + result_data[batch * rows1 * cols2 + row * cols2 + col] = sum; + } +} + +__host__ void broadcasted_batched_matmul_tensor_cuda(Tensor* tensor1, Tensor* tensor2, float* result_data) { + + int batch_size = tensor2->shape[0]; + int rows1 = tensor1->shape[0]; + int cols1 = tensor1->shape[1]; + int cols2 = tensor2->shape[2]; + + dim3 threadsPerBlock(16, 16); + dim3 number_of_blocks((cols2 + threadsPerBlock.x - 1) / threadsPerBlock.x, (rows1 + threadsPerBlock.y - 1) / threadsPerBlock.y, batch_size); + broadcasted_batched_matmul_tensor_cuda_kernel<<>>(tensor1->data, tensor2->data, result_data, batch_size, rows1, cols1, cols2); + + + cudaError_t error = cudaGetLastError(); + if (error != cudaSuccess) { + printf("CUDA error: %s\n", cudaGetErrorString(error)); + exit(-1); + } + + cudaDeviceSynchronize(); +} + +__global__ void tensor_pow_scalar_cuda_kernel(float* data, float exponent, float* result_data, int size) { + + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i < size) { + result_data[i] = powf(data[i], exponent); + } +} + +__host__ void tensor_pow_scalar_cuda(Tensor* tensor, float exponent, float* result_data) { + + int number_of_blocks = (tensor->size + THREADS_PER_BLOCK - 1) / THREADS_PER_BLOCK; + tensor_pow_scalar_cuda_kernel<<>>(tensor->data, exponent, result_data, tensor->size); + + cudaError_t error = cudaGetLastError(); + if (error != cudaSuccess) { + printf("CUDA error: %s\n", cudaGetErrorString(error)); + exit(-1); + } + + cudaDeviceSynchronize(); +} + +__global__ void scalar_pow_tensor_cuda_kernel(float base, float* data, float* result_data, int size) { + + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i < size) { + result_data[i] = powf(base, data[i]); + } +} + +__host__ void scalar_pow_tensor_cuda(float base, Tensor* tensor, float* result_data) { + + int number_of_blocks = (tensor->size + THREADS_PER_BLOCK - 1) / THREADS_PER_BLOCK; + scalar_pow_tensor_cuda_kernel<<>>(base, tensor->data, result_data, tensor->size); + + cudaError_t error = cudaGetLastError(); + if (error != cudaSuccess) { + printf("CUDA error: %s\n", cudaGetErrorString(error)); + exit(-1); + } + + cudaDeviceSynchronize(); +} + +__global__ void log_tensor_cuda_kernel(float* data, float* result_data, int size) { + + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i < size) { + result_data[i] = logf(data[i]); + } +} + +__host__ void log_tensor_cuda(Tensor* tensor, float* result_data) { + + int number_of_blocks = (tensor->size + THREADS_PER_BLOCK - 1) / THREADS_PER_BLOCK; + log_tensor_cuda_kernel<<>>(tensor->data, result_data, tensor->size); + + cudaError_t error = cudaGetLastError(); + if (error != cudaSuccess) { + printf("CUDA error: %s\n", cudaGetErrorString(error)); + exit(-1); + } + + cudaDeviceSynchronize(); +} + +__global__ void equal_tensor_cuda_kernel(float* data1, float* data2, float* result_data, int size) { + + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i < size) { + result_data[i] = (data1[i] == data2[i]) ? 1.0f : 0.0f; + } +} + +__host__ void equal_tensor_cuda(Tensor* tensor1, Tensor* tensor2, float* result_data) { + + int number_of_blocks = (tensor1->size + THREADS_PER_BLOCK - 1) / THREADS_PER_BLOCK; + equal_tensor_cuda_kernel<<>>(tensor1->data, tensor2->data, result_data, tensor1->size); + + cudaError_t error = cudaGetLastError(); + if (error != cudaSuccess) { + printf("CUDA error: %s\n", cudaGetErrorString(error)); + exit(-1); + } + + cudaDeviceSynchronize(); +} + +__global__ void equal_broadcasted_tensor_cuda_kernel(float* data1, float* data2, float* result_data, int* broadcasted_shape, int* strides1, int*strides2, int max_ndim, int size) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i >= size) return; + + int index1 = 0, index2 = 0; + int linear_index = i; + for (int j = max_ndim - 1; j >= 0; j--) { + int pos = linear_index % broadcasted_shape[j]; + linear_index /= broadcasted_shape[j]; + if (strides1[j] != 0) index1 += pos * strides1[j]; + if (strides2[j] != 0) index2 += pos * strides2[j]; + } + result_data[i] = (data1[index1] == data2[index2]) ? 1.0f : 0.0f; +} + +__host__ void equal_broadcasted_tensor_cuda(Tensor* tensor1, Tensor* tensor2, float* result_data, int* broadcasted_shape, int broadcasted_size) { + int max_ndim = tensor1->ndim > tensor2->ndim ? tensor1->ndim : tensor2->ndim; + + int* strides1 = (int*)malloc(max_ndim * sizeof(int)); + int* strides2 = (int*)malloc(max_ndim * sizeof(int)); + if (strides1 == NULL || strides2 == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + + int stride1 = 1, stride2 = 1; + for (int i = max_ndim - 1; i >= 0; i--) { + int dim1 = i < tensor1->ndim ? tensor1->shape[tensor1->ndim - max_ndim + i] : 1; + int dim2 = i < tensor2->ndim ? tensor2->shape[tensor2->ndim - max_ndim + i] : 1; + strides1[i] = dim1 == broadcasted_shape[i] ? stride1 : 0; + strides2[i] = dim2 == broadcasted_shape[i] ? stride2 : 0; + stride1 *= (dim1 == broadcasted_shape[i]) ? dim1 : 1; + stride2 *= (dim2 == broadcasted_shape[i]) ? dim2 : 1; + } + + int* d_broadcasted_shape; + int* d_strides1; + int* d_strides2; + + cudaMalloc((void**)&d_broadcasted_shape, max_ndim * sizeof(int)); + cudaMemcpy(d_broadcasted_shape, broadcasted_shape, max_ndim * sizeof(int), cudaMemcpyHostToDevice); + + cudaMalloc((void**)&d_strides1, max_ndim * sizeof(int)); + cudaMemcpy(d_strides1, strides1, max_ndim * sizeof(int), cudaMemcpyHostToDevice); + + cudaMalloc((void**)&d_strides2, max_ndim * sizeof(int)); + cudaMemcpy(d_strides2, strides2, max_ndim * sizeof(int), cudaMemcpyHostToDevice); + + int number_of_blocks = (broadcasted_size + THREADS_PER_BLOCK - 1) / THREADS_PER_BLOCK; + equal_broadcasted_tensor_cuda_kernel<<>>(tensor1->data, tensor2->data, result_data, d_broadcasted_shape, d_strides1, d_strides2, max_ndim, broadcasted_size); + + cudaError_t error = cudaGetLastError(); + if (error != cudaSuccess) { + printf("CUDA error: %s\n", cudaGetErrorString(error)); + exit(-1); + } + + cudaDeviceSynchronize(); + cudaFree(d_broadcasted_shape); +} + + +__global__ void ones_like_tensor_cuda_kernel(float* data, float* result_data, int size) { + + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i < size) { + result_data[i] = 1.0; + } +} + +__host__ void ones_like_tensor_cuda(Tensor* tensor, float* result_data) { + + int number_of_blocks = (tensor->size + THREADS_PER_BLOCK - 1) / THREADS_PER_BLOCK; + ones_like_tensor_cuda_kernel<<>>(tensor->data, result_data, tensor->size); + + cudaError_t error = cudaGetLastError(); + if (error != cudaSuccess) { + printf("CUDA error: %s\n", cudaGetErrorString(error)); + exit(-1); + } + + cudaDeviceSynchronize(); +} + +__global__ void zeros_like_tensor_cuda_kernel(float* data, float* result_data, int size) { + + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i < size) { + result_data[i] = 0.0; + } +} + +__host__ void zeros_like_tensor_cuda(Tensor* tensor, float* result_data) { + + int number_of_blocks = (tensor->size + THREADS_PER_BLOCK - 1) / THREADS_PER_BLOCK; + zeros_like_tensor_cuda_kernel<<>>(tensor->data, result_data, tensor->size); + + cudaError_t error = cudaGetLastError(); + if (error != cudaSuccess) { + printf("CUDA error: %s\n", cudaGetErrorString(error)); + exit(-1); + } + + cudaDeviceSynchronize(); +} + + +__global__ void transpose_1D_tensor_cuda_kernel(float* data, float* result_data, int size) { + + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i < size) { + result_data[i] = data[i]; + } +} + +__host__ void transpose_1D_tensor_cuda(Tensor* tensor, float* result_data) { + + int number_of_blocks = (tensor->size + THREADS_PER_BLOCK - 1) / THREADS_PER_BLOCK; + transpose_1D_tensor_cuda_kernel<<>>(tensor->data, result_data, tensor->size); + + cudaError_t error = cudaGetLastError(); + if (error != cudaSuccess) { + printf("CUDA error: %s\n", cudaGetErrorString(error)); + exit(-1); + } + + cudaDeviceSynchronize(); +} + +__global__ void transpose_2D_tensor_cuda_kernel(float* data, float* result_data, int rows, int cols) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + int j = blockIdx.y * blockDim.y + threadIdx.y; + + if (i < rows && j < cols) { + result_data[j * rows + i] = data[i * cols + j]; + } +} + +__host__ void transpose_2D_tensor_cuda(Tensor* tensor, float* result_data) { + + int rows = tensor->shape[0]; + int cols = tensor->shape[1]; + + dim3 threadsPerBlock(16, 16); + dim3 number_of_blocks((rows + threadsPerBlock.x - 1) / threadsPerBlock.x, (cols + threadsPerBlock.y - 1) / threadsPerBlock.y); + transpose_2D_tensor_cuda_kernel<<>>(tensor->data, result_data, rows, cols); + + + cudaError_t error = cudaGetLastError(); + if (error != cudaSuccess) { + printf("CUDA error: %s\n", cudaGetErrorString(error)); + exit(-1); + } + + cudaDeviceSynchronize(); +} + +__global__ void transpose_3D_tensor_cuda_kernel(float* data, float* result_data, int batch, int rows, int cols) { + int i = blockIdx.x * blockDim.x + threadIdx.x; + int j = blockIdx.y * blockDim.y + threadIdx.y; + int k = blockIdx.z * blockDim.z + threadIdx.z; + + if (i < batch && j < rows && k < cols) { + result_data[k * rows * batch + j * batch + i] = data[i * rows * cols + j * cols + k]; + } +} + +__host__ void transpose_3D_tensor_cuda(Tensor* tensor, float* result_data) { + + int batch = tensor->shape[0]; + int rows = tensor->shape[1]; + int cols = tensor->shape[2]; + + dim3 threadsPerBlock(8, 8, 8); + dim3 number_of_blocks((batch + threadsPerBlock.x - 1) / threadsPerBlock.x, (rows + threadsPerBlock.y - 1) / threadsPerBlock.y, (cols + threadsPerBlock.z - 1) / threadsPerBlock.z); + transpose_3D_tensor_cuda_kernel<<>>(tensor->data, result_data, batch, rows, cols); + + + cudaError_t error = cudaGetLastError(); + if (error != cudaSuccess) { + printf("CUDA error: %s\n", cudaGetErrorString(error)); + exit(-1); + } + + cudaDeviceSynchronize(); +} + + +__global__ void assign_tensor_cuda_kernel(float* data, float* result_data, int size) { + + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i < size) { + result_data[i] = data[i]; + } +} + +__host__ void assign_tensor_cuda(Tensor* tensor, float* result_data) { + + int number_of_blocks = (tensor->size + THREADS_PER_BLOCK - 1) / THREADS_PER_BLOCK; + assign_tensor_cuda_kernel<<>>(tensor->data, result_data, tensor->size); + + cudaError_t error = cudaGetLastError(); + if (error != cudaSuccess) { + printf("CUDA error: %s\n", cudaGetErrorString(error)); + exit(-1); + } + + cudaDeviceSynchronize(); +} + +__global__ void make_contiguous_tensor_cuda_kernel(float* data, float* result_data, int ndim, int size, int* strides, int* new_strides) { + + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i < size) { + int index = 0; + int offset = i; + for (int j = 0; j < ndim; j++) { + index += (offset / new_strides[j]) * strides[j]; + offset %= new_strides[j]; + } + result_data[i] = data[index]; + } +} + +__host__ void make_contiguous_tensor_cuda(Tensor* tensor, float* result_data, int* new_strides) { + + int* d_strides; + cudaMalloc((void **)&d_strides, tensor->ndim * sizeof(int)); + cudaMemcpy(d_strides, tensor->strides, tensor->ndim * sizeof(int), cudaMemcpyHostToDevice); + + int* d_new_strides; + cudaMalloc((void **)&d_new_strides, tensor->ndim * sizeof(int)); + cudaMemcpy(d_new_strides, new_strides, tensor->ndim * sizeof(int), cudaMemcpyHostToDevice); + + int number_of_blocks = (tensor->size + THREADS_PER_BLOCK - 1) / THREADS_PER_BLOCK; + make_contiguous_tensor_cuda_kernel<<>>(tensor->data, result_data, tensor->ndim, tensor->size, d_strides, d_new_strides); + + cudaError_t error = cudaGetLastError(); + if (error != cudaSuccess) { + printf("CUDA error: %s\n", cudaGetErrorString(error)); + exit(-1); + } + + cudaDeviceSynchronize(); + + // Free old data and update tensor properties + cudaFree(tensor->data); + free(tensor->strides); + tensor->data = result_data; + tensor->strides = new_strides; +} + +__global__ void sin_tensor_cuda_kernel(float* data, float* result_data, int size) { + + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i < size) { + result_data[i] = sinf(data[i]); + } +} + +__host__ void sin_tensor_cuda(Tensor* tensor, float* result_data) { + + int number_of_blocks = (tensor->size + THREADS_PER_BLOCK - 1) / THREADS_PER_BLOCK; + sin_tensor_cuda_kernel<<>>(tensor->data, result_data, tensor->size); + + cudaError_t error = cudaGetLastError(); + if (error != cudaSuccess) { + printf("CUDA error: %s\n", cudaGetErrorString(error)); + exit(-1); + } + + cudaDeviceSynchronize(); +} + +__global__ void cos_tensor_cuda_kernel(float* data, float* result_data, int size) { + + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i < size) { + result_data[i] = cosf(data[i]); + } +} + +__host__ void cos_tensor_cuda(Tensor* tensor, float* result_data) { + + int number_of_blocks = (tensor->size + THREADS_PER_BLOCK - 1) / THREADS_PER_BLOCK; + cos_tensor_cuda_kernel<<>>(tensor->data, result_data, tensor->size); + + cudaError_t error = cudaGetLastError(); + if (error != cudaSuccess) { + printf("CUDA error: %s\n", cudaGetErrorString(error)); + exit(-1); + } + + cudaDeviceSynchronize(); +} + +__global__ void sigmoid_tensor_cuda_kernel(float* data, float* result_data, int size) { + + int i = blockIdx.x * blockDim.x + threadIdx.x; + + if (i < size) { + // avoid overflow + if (data[i] >= 0) { + + float z = expf(-data[i]); + result_data[i] = 1 / (1 + z); + + } else { + + float z = expf(data[i]); + result_data[i] = z / (1 + z); + } + } +} + +__host__ void sigmoid_tensor_cuda(Tensor* tensor, float* result_data) { + + int number_of_blocks = (tensor->size + THREADS_PER_BLOCK - 1) / THREADS_PER_BLOCK; + sigmoid_tensor_cuda_kernel<<>>(tensor->data, result_data, tensor->size); + + cudaError_t error = cudaGetLastError(); + if (error != cudaSuccess) { + printf("CUDA error: %s\n", cudaGetErrorString(error)); + exit(-1); + } + + cudaDeviceSynchronize(); +} + + + + diff --git a/build/lib/norch/csrc/cuda.h b/build/lib/norch/csrc/cuda.h new file mode 100644 index 0000000..702dea1 --- /dev/null +++ b/build/lib/norch/csrc/cuda.h @@ -0,0 +1,104 @@ +#ifndef CUDA_KERNEL_H_ +#define CUDA_KERNEL_H_ + + + __host__ void cpu_to_cuda(Tensor* tensor, int device_id); + __host__ void cuda_to_cpu(Tensor* tensor); + + __global__ void add_tensor_cuda_kernel(float* data1, float* data2, float* result_data, int size); + __host__ void add_tensor_cuda(Tensor* tensor1, Tensor* tensor2, float* result_data); + + __global__ void add_broadcasted_tensor_cuda_kernel(float* data1, float* data2, float* result_data, int* broadcasted_shape, int* strides1, int*strides2, int max_ndim, int size); + __host__ void add_broadcasted_tensor_cuda(Tensor* tensor1, Tensor* tensor2, float* result_data, int* broadcasted_shape, int broadcasted_size); + + __global__ void sub_broadcasted_tensor_cuda_kernel(float* data1, float* data2, float* result_data, int* broadcasted_shape, int* strides1, int*strides2, int max_ndim, int size); + __host__ void sub_broadcasted_tensor_cuda(Tensor* tensor1, Tensor* tensor2, float* result_data, int* broadcasted_shape, int broadcasted_size); + + __global__ void sum_tensor_cuda_kernel(float* data, float* result_data); + __global__ void sum_tensor_cuda_kernel_axis(float* data, float* result_data, int* strides, int* shape, int axis, int ndim, int axis_stride, int size, int result_size); + __host__ void sum_tensor_cuda(Tensor* tensor, float* result_data, int axis); + + __global__ void max_tensor_cuda_kernel(float* data, float* result_data); + __global__ void max_tensor_cuda_kernel_axis(float* data, float* result_data, int* strides, int* shape, int axis, int ndim, int axis_stride, int size, int result_size); + __host__ void max_tensor_cuda(Tensor* tensor, float* result_data, int axis); + + __global__ void min_tensor_cuda_kernel(float* data, float* result_data); + __global__ void min_tensor_cuda_kernel_axis(float* data, float* result_data, int* strides, int* shape, int axis, int ndim, int axis_stride, int size, int result_size); + __host__ void min_tensor_cuda(Tensor* tensor, float* result_data, int axis); + + __global__ void sub_tensor_cuda_kernel(float* data1, float* data2, float* result_data, int size); + __host__ void sub_tensor_cuda(Tensor* tensor1, Tensor* tensor2, float* result_data); + + __global__ void elementwise_mul_tensor_cuda_kernel(float* data1, float* data2, float* result_data, int size); + __host__ void elementwise_mul_tensor_cuda(Tensor* tensor1, Tensor* tensor2, float* result_data); + + __global__ void scalar_mul_tensor_cuda_kernel(float* data, float scalar, float* result_data, int size); + __host__ void scalar_mul_tensor_cuda(Tensor* tensor, float scalar, float* result_data); + + __global__ void scalar_div_tensor_cuda_kernel(float scalar, float* data, float* result_data, int size); + __host__ void scalar_div_tensor_cuda(float scalar, Tensor* tensor, float* result_data); + + __global__ void tensor_div_scalar_cuda_kernel(float* data, float scalar, float* result_data, int size); + __host__ void tensor_div_scalar_cuda(Tensor* tensor, float scalar, float* result_data); + + __global__ void tensor_div_tensor_cuda_kernel(float* data1, float* data2, float* result_data, int size); + __host__ void tensor_div_tensor_cuda(Tensor* tensor1, Tensor* tensor2, float* result_data); + + __global__ void matmul_tensor_cuda_kernel(float* data1, float* data2, float* result_data, int rows1, int cols1, int cols2); + __host__ void matmul_tensor_cuda(Tensor* tensor1, Tensor* tensor2, float* result_data); + + __global__ void batched_matmul_tensor_cuda_kernel(float* data1, float* data2, float* result_data, int batch_size, int rows1, int cols1, int cols2); + __host__ void batched_matmul_tensor_cuda(Tensor* tensor1, Tensor* tensor2, float* result_data); + + __global__ void broadcasted_batched_matmul_tensor_cuda_kernel(float* data1, float* data2, float* result_data, int batch_size, int rows1, int cols1, int cols2); + __host__ void broadcasted_batched_matmul_tensor_cuda(Tensor* tensor1, Tensor* tensor2, float* result_data); + + __global__ void tensor_pow_scalar_cuda_kernel(float* data, float exponent, float* result_data, int size); + __host__ void tensor_pow_scalar_cuda(Tensor* tensor, float exponent, float* result_data); + + __global__ void scalar_pow_tensor_cuda_kernel(float base, float* data, float* result_data, int size); + __host__ void scalar_pow_tensor_cuda(float base, Tensor* tensor, float* result_data); + + __global__ void log_tensor_cuda_kernel(float* data, float* result_data, int size); + __host__ void log_tensor_cuda(Tensor* tensor, float* result_data); + + __global__ void equal_tensor_cuda_kernel(float* data1, float* data2, float* result_data, int size); + __host__ void equal_tensor_cuda(Tensor* tensor1, Tensor* tensor2, float* result_data); + + __global__ void equal_broadcasted_tensor_cuda_kernel(float* data1, float* data2, float* result_data, int* broadcasted_shape, int* strides1, int*strides2, int max_ndim, int size); + __host__ void equal_broadcasted_tensor_cuda(Tensor* tensor1, Tensor* tensor2, float* result_data, int* broadcasted_shape, int broadcasted_size); + + __global__ void ones_like_tensor_cuda_kernel(float* data, float* result_data, int size); + __host__ void ones_like_tensor_cuda(Tensor* tensor, float* result_data); + + __global__ void zeros_like_tensor_cuda_kernel(float* data, float* result_data, int size); + __host__ void zeros_like_tensor_cuda(Tensor* tensor, float* result_data); + + __global__ void transpose_1D_tensor_cuda_kernel(float* data, float* result_data, int rows, int cols); + __host__ void transpose_1D_tensor_cuda(Tensor* tensor, float* result_data); + + __global__ void transpose_2D_tensor_cuda_kernel(float* data, float* result_data, int rows, int cols); + __host__ void transpose_2D_tensor_cuda(Tensor* tensor, float* result_data); + + __global__ void transpose_3D_tensor_cuda_kernel(float* data, float* result_data, int batch, int rows, int cols); + __host__ void transpose_3D_tensor_cuda(Tensor* tensor, float* result_data); + + __global__ void assign_tensor_cuda_kernel(float* data, float* result_data, int size); + __host__ void assign_tensor_cuda(Tensor* tensor, float* result_data); + + __global__ void make_contiguous_tensor_cuda_kernel(float* data, float* result_data, int ndim, int size, int* strides, int* new_strides); + __host__ void make_contiguous_tensor_cuda(Tensor* tensor, float* result_data, int* new_strides); + + __global__ void sin_tensor_cuda_kernel(float* data, float* result_data, int size); + __host__ void sin_tensor_cuda(Tensor* tensor, float* result_data); + + __global__ void cos_tensor_cuda_kernel(float* data, float* result_data, int size); + __host__ void cos_tensor_cuda(Tensor* tensor, float* result_data); + + __global__ void sigmoid_tensor_cuda_kernel(float* data, float* result_data, int size); + __host__ void sigmoid_tensor_cuda(Tensor* tensor, float* result_data); + + + + +#endif /* CUDA_KERNEL_H_ */ diff --git a/build/lib/norch/csrc/distributed.cpp b/build/lib/norch/csrc/distributed.cpp new file mode 100644 index 0000000..8124ec0 --- /dev/null +++ b/build/lib/norch/csrc/distributed.cpp @@ -0,0 +1,65 @@ +#include +#include +#include +#include "distributed.h" +#include "tensor.h" +#include "cuda.h" +#include +#include + +int rank; +int world_size; +ncclComm_t nccl_comm; + +extern "C" { + +void init_process_group(int env_rank, int env_world_size) { + + rank = env_rank; + world_size = env_world_size; + + // init MPI + MPI_CHECK(MPI_Init(NULL, NULL)); + + + ncclUniqueId nccl_id; + + if (rank == 0) { + ncclGetUniqueId(&nccl_id); + } + + // send nccl unique id to all processes + MPI_CHECK(MPI_Bcast((void *)&nccl_id, sizeof(nccl_id), MPI_BYTE, 0, MPI_COMM_WORLD)); + + // init NCCL communication group + NCCL_CHECK(ncclCommInitRank(&nccl_comm, world_size, nccl_id, rank)); + +} + +void broadcast_tensor(Tensor* tensor) { + + cudaStream_t stream; + cudaStreamCreate(&stream); + + NCCL_CHECK(ncclBroadcast(tensor->data, tensor->data, tensor->size * sizeof(float), ncclFloat, 0, nccl_comm, stream)); + cudaStreamSynchronize(stream); + cudaStreamDestroy(stream); +} + +void allreduce_sum_tensor(Tensor* tensor) { + cudaStream_t stream; + cudaStreamCreate(&stream); + + // Perform NCCL AllReduce operation to calculate the sum of all tensors across all processes + NCCL_CHECK(ncclAllReduce(tensor->data, tensor->data, tensor->size, ncclFloat, ncclSum, nccl_comm, stream)); + + cudaStreamSynchronize(stream); + cudaStreamDestroy(stream); +} + +void end_process_group() { + MPI_CHECK(MPI_Finalize()); + NCCL_CHECK(ncclCommDestroy(nccl_comm)); +} + +} diff --git a/build/lib/norch/csrc/distributed.h b/build/lib/norch/csrc/distributed.h new file mode 100644 index 0000000..1040fc0 --- /dev/null +++ b/build/lib/norch/csrc/distributed.h @@ -0,0 +1,28 @@ +#ifndef DISTRIBUTED_H +#define DISTRIBUTED_H + +#define MPI_CHECK(cmd) do { \ + int e = cmd; \ + if( e != MPI_SUCCESS ) { \ + printf("Failed: MPI error %s:%d '%d'\n", \ + __FILE__,__LINE__, e); \ + exit(EXIT_FAILURE); \ + } \ +} while(0) + +#define NCCL_CHECK(cmd) do { \ + ncclResult_t r = cmd; \ + if (r!= ncclSuccess) { \ + printf("Failed, NCCL error %s:%d '%s'\n", \ + __FILE__,__LINE__,ncclGetErrorString(r)); \ + exit(EXIT_FAILURE); \ + } \ +} while(0) + +extern "C" { + void init_process_group(int rank, int world_size); + void broadcast_tensor(Tensor* tensor); + void allreduce_sum_tensor(Tensor* tensor); +} + +#endif /* DISTRIBUTED_H */ diff --git a/build/lib/norch/csrc/tensor.cpp b/build/lib/norch/csrc/tensor.cpp new file mode 100644 index 0000000..725b9f5 --- /dev/null +++ b/build/lib/norch/csrc/tensor.cpp @@ -0,0 +1,1558 @@ +#include +#include +#include +#include +#include +#include "tensor.h" +#include "cuda.h" +#include "cpu.h" + +extern "C" { + + Tensor* create_tensor(float* data, int* shape, int ndim, char* device) { + + Tensor* tensor = (Tensor*)malloc(sizeof(Tensor)); + if (tensor == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + tensor->data = data; + tensor->shape = shape; + tensor->ndim = ndim; + + tensor->device = (char*)malloc(strlen(device) + 1); + if (device != NULL) { + strcpy(tensor->device, device); + } else { + fprintf(stderr, "Memory allocation failed\n"); + exit(-1); + } + + tensor->size = 1; + for (int i = 0; i < ndim; i++) { + tensor->size *= shape[i]; + } + + tensor->strides = (int*)malloc(ndim * sizeof(int)); + if (tensor->strides == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + int stride = 1; + for (int i = ndim - 1; i >= 0; i--) { + tensor->strides[i] = stride; + stride *= shape[i]; + } + + return tensor; + } + + float get_item(Tensor* tensor, int* indices) { + int index = 0; + for (int i = 0; i < tensor->ndim; i++) { + index += indices[i] * tensor->strides[i]; + } + + float result; + if (strcmp(tensor->device, "cuda") == 0) { + cudaMemcpy(&result, tensor->data + index, sizeof(float), cudaMemcpyDeviceToHost); + } else { + result = tensor->data[index]; + } + + return result; + } + + void to_device(Tensor* tensor, char* target_device) { + int device_id = 0; + char* endptr; + long num = strtol(target_device, &endptr, 10); + if (*endptr == '\0') { + device_id = (int)num; + } + + if ((strcmp(target_device, "cuda") == 0) && (strcmp(tensor->device, "cpu") == 0)) { + cpu_to_cuda(tensor, device_id); + } + + else if ((strcmp(target_device, "cpu") == 0) && (strcmp(tensor->device, "cuda") == 0)) { + cuda_to_cpu(tensor); + } + } + + Tensor* add_tensor(Tensor* tensor1, Tensor* tensor2) { + if (tensor1->ndim != tensor2->ndim) { + fprintf(stderr, "Tensors must have the same number of dimensions %d and %d for addition\n", tensor1->ndim, tensor2->ndim); + exit(1); + } + + if (strcmp(tensor1->device, tensor2->device) != 0) { + fprintf(stderr, "Tensors must be on the same device: %s and %s\n", tensor1->device, tensor2->device); + exit(1); + } + + char* device = (char*)malloc(strlen(tensor1->device) + 1); + if (device != NULL) { + strcpy(device, tensor1->device); + } else { + fprintf(stderr, "Memory allocation failed\n"); + exit(-1); + } + int ndim = tensor1->ndim; + int* shape = (int*)malloc(ndim * sizeof(int)); + if (shape == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + + for (int i = 0; i < ndim; i++) { + if (tensor1->shape[i] != tensor2->shape[i]) { + fprintf(stderr, "Tensors must have the same shape %d and %d at index %d for addition\n", tensor1->shape[i], tensor2->shape[i], i); + exit(1); + } + shape[i] = tensor1->shape[i]; + } + + if (strcmp(tensor1->device, "cuda") == 0) { + + float* result_data; + cudaMalloc((void **)&result_data, tensor1->size * sizeof(float)); + add_tensor_cuda(tensor1, tensor2, result_data); + return create_tensor(result_data, shape, ndim, device); + } + else { + float* result_data = (float*)malloc(tensor1->size * sizeof(float)); + if (result_data == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + add_tensor_cpu(tensor1, tensor2, result_data); + return create_tensor(result_data, shape, ndim, device); + } + } + + Tensor* add_broadcasted_tensor(Tensor* tensor1, Tensor* tensor2) { + + if (strcmp(tensor1->device, tensor2->device) != 0) { + fprintf(stderr, "Tensors must be on the same device: %s and %s\n", tensor1->device, tensor2->device); + exit(1); + } + + int max_ndim = tensor1->ndim > tensor2->ndim ? tensor1->ndim : tensor2->ndim; + + // Determine the broadcasted shape + int* broadcasted_shape = (int*)malloc(max_ndim * sizeof(int)); + if (broadcasted_shape == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + for (int i = 0; i < max_ndim; i++) { + int dim1 = i < tensor1->ndim ? tensor1->shape[tensor1->ndim - 1 - i] : 1; + int dim2 = i < tensor2->ndim ? tensor2->shape[tensor2->ndim - 1 - i] : 1; + if (dim1 != dim2 && dim1 != 1 && dim2 != 1) { + fprintf(stderr, "Shapes are not compatible for broadcasting\n"); + exit(1); + } + broadcasted_shape[max_ndim - 1 - i] = dim1 > dim2 ? dim1 : dim2; + } + + int broadcasted_size = 1; + for (int i = 0; i < max_ndim; i++) { + broadcasted_size *= broadcasted_shape[i]; + } + + if (strcmp(tensor1->device, "cuda") == 0) { + float* result_data; + cudaMalloc((void **)&result_data, broadcasted_size * sizeof(float)); + add_broadcasted_tensor_cuda(tensor1, tensor2, result_data, broadcasted_shape, broadcasted_size); + return create_tensor(result_data, broadcasted_shape, max_ndim, tensor1->device); + } + else { + float* result_data = (float*)malloc(broadcasted_size * sizeof(float)); + if (result_data == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + + add_broadcasted_tensor_cpu(tensor1, tensor2, result_data, broadcasted_shape, broadcasted_size); + return create_tensor(result_data, broadcasted_shape, max_ndim, tensor1->device); + } + } + + Tensor* sum_tensor(Tensor* tensor, int axis, bool keepdim) { + char* device = (char*)malloc(strlen(tensor->device) + 1); + if (device != NULL) { + strcpy(device, tensor->device); + } else { + fprintf(stderr, "Memory allocation failed\n"); + exit(-1); + } + int ndim; + int* shape; + + if (axis > tensor->ndim - 1) { + fprintf(stderr, "Error: axis argument %d must be smaller than tensor dimension %d", axis, tensor->ndim); + } + + if (axis == -1) { + + shape = (int*) malloc(sizeof(int)); + shape[0] = 1; + ndim = 1; + } else { + shape = (int*) malloc((tensor->ndim - 1) * sizeof(int)); + for (int i = 0, j = 0; i < tensor->ndim; ++i) { + if (i != axis) { + shape[j++] = tensor->shape[i]; + } + } + ndim = tensor->ndim - 1; + } + + int axis_size = 1; + for (int i = 0; i < ndim; i++) { + axis_size *= shape[i]; + } + + if (strcmp(tensor->device, "cuda") == 0) { + + float* result_data; + if (axis == -1) { + cudaMalloc((void**)&result_data, tensor->size * sizeof(float)); + } else { + cudaMalloc((void**)&result_data, axis_size * sizeof(float)); + } + sum_tensor_cuda(tensor, result_data, axis); + + if (keepdim) { + if (axis == -1){ + ndim = tensor->ndim; + shape = (int*) malloc((tensor->ndim) * sizeof(int)); + for (int i = 0; i < tensor->ndim; i++) { + shape[i] = 1; + } + } else { + shape = (int*) malloc((tensor->ndim) * sizeof(int)); + for (int i = 0; i < tensor->ndim; i++) { + shape[i] = tensor->shape[i]; + } + shape[axis] = 1; + ndim = tensor->ndim; + } + + } + return create_tensor(result_data, shape, ndim, device); + } + else { + float* result_data = (float*)calloc(axis_size, sizeof(float)); + if (result_data == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + + sum_tensor_cpu(tensor, result_data, axis_size, shape, axis); + + if (keepdim) { + if (axis == -1){ + ndim = tensor->ndim; + shape = (int*) malloc((tensor->ndim) * sizeof(int)); + for (int i = 0; i < tensor->ndim; i++) { + shape[i] = 1; + } + } else { + shape = (int*) malloc((tensor->ndim) * sizeof(int)); + for (int i = 0; i < tensor->ndim; i++) { + shape[i] = tensor->shape[i]; + } + shape[axis] = 1; + ndim = tensor->ndim; + } + + } + return create_tensor(result_data, shape, ndim, device); + } + } + + Tensor* max_tensor(Tensor* tensor, int axis, bool keepdim) { + char* device = (char*)malloc(strlen(tensor->device) + 1); + if (device != NULL) { + strcpy(device, tensor->device); + } else { + fprintf(stderr, "Memory allocation failed\n"); + exit(-1); + } + int ndim; + int* shape; + if (axis == -1) { + shape = (int*) malloc(sizeof(int)); + shape[0] = 1; + ndim = 1; + } else { + shape = (int*) malloc((tensor->ndim - 1) * sizeof(int)); + for (int i = 0, j = 0; i < tensor->ndim; ++i) { + if (i != axis) { + shape[j++] = tensor->shape[i]; + } + } + ndim = tensor->ndim - 1; + } + + int axis_size = 1; + for (int i = 0; i < ndim; i++) { + axis_size *= shape[i]; + } + + if (strcmp(tensor->device, "cuda") == 0) { + + float* result_data; + if (axis == -1) { + cudaMalloc((void**)&result_data, tensor->size * sizeof(float)); + } else { + cudaMalloc((void**)&result_data, axis_size * sizeof(float)); + } + max_tensor_cuda(tensor, result_data, axis); + + if (keepdim) { + if (axis == -1){ + ndim = tensor->ndim; + shape = (int*) malloc((tensor->ndim) * sizeof(int)); + for (int i = 0; i < tensor->ndim; i++) { + shape[i] = 1; + } + } else { + shape = (int*) malloc((tensor->ndim) * sizeof(int)); + for (int i = 0; i < tensor->ndim; i++) { + shape[i] = tensor->shape[i]; + } + shape[axis] = 1; + ndim = tensor->ndim; + } + + } + return create_tensor(result_data, shape, ndim, device); + + } + else { + float* result_data = (float*)malloc(axis_size * sizeof(float)); + if (result_data == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + max_tensor_cpu(tensor, result_data, axis_size, shape, axis); + + if (keepdim) { + if (axis == -1){ + ndim = tensor->ndim; + shape = (int*) malloc((tensor->ndim) * sizeof(int)); + for (int i = 0; i < tensor->ndim; i++) { + shape[i] = 1; + } + } else { + shape = (int*) malloc((tensor->ndim) * sizeof(int)); + for (int i = 0; i < tensor->ndim; i++) { + shape[i] = tensor->shape[i]; + } + shape[axis] = 1; + ndim = tensor->ndim; + } + + } + return create_tensor(result_data, shape, ndim, device); + } + } + + Tensor* min_tensor(Tensor* tensor, int axis, bool keepdim) { + char* device = (char*)malloc(strlen(tensor->device) + 1); + if (device != NULL) { + strcpy(device, tensor->device); + } else { + fprintf(stderr, "Memory allocation failed\n"); + exit(-1); + } + int ndim; + int* shape; + if (axis == -1) { + shape = (int*) malloc(sizeof(int)); + shape[0] = 1; + ndim = 1; + } else { + shape = (int*) malloc((tensor->ndim - 1) * sizeof(int)); + for (int i = 0, j = 0; i < tensor->ndim; ++i) { + if (i != axis) { + shape[j++] = tensor->shape[i]; + } + } + ndim = tensor->ndim - 1; + } + + int axis_size = 1; + for (int i = 0; i < ndim; i++) { + axis_size *= shape[i]; + } + + if (strcmp(tensor->device, "cuda") == 0) { + + float* result_data; + if (axis == -1) { + cudaMalloc((void**)&result_data, tensor->size * sizeof(float)); + } else { + cudaMalloc((void**)&result_data, axis_size * sizeof(float)); + } + min_tensor_cuda(tensor, result_data, axis); + + if (keepdim) { + if (axis == -1){ + ndim = tensor->ndim; + shape = (int*) malloc((tensor->ndim) * sizeof(int)); + for (int i = 0; i < tensor->ndim; i++) { + shape[i] = 1; + } + } else { + shape = (int*) malloc((tensor->ndim) * sizeof(int)); + for (int i = 0; i < tensor->ndim; i++) { + shape[i] = tensor->shape[i]; + } + shape[axis] = 1; + ndim = tensor->ndim; + } + + } + return create_tensor(result_data, shape, ndim, device); + + } + else { + float* result_data = (float*)malloc(axis_size * sizeof(float)); + if (result_data == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + min_tensor_cpu(tensor, result_data, axis_size, shape, axis); + + if (keepdim) { + if (axis == -1){ + ndim = tensor->ndim; + shape = (int*) malloc((tensor->ndim) * sizeof(int)); + for (int i = 0; i < tensor->ndim; i++) { + shape[i] = 1; + } + } else { + shape = (int*) malloc((tensor->ndim) * sizeof(int)); + for (int i = 0; i < tensor->ndim; i++) { + shape[i] = tensor->shape[i]; + } + shape[axis] = 1; + ndim = tensor->ndim; + } + + } + return create_tensor(result_data, shape, ndim, device); + } + } + + Tensor* sub_tensor(Tensor* tensor1, Tensor* tensor2) { + if (tensor1->ndim != tensor2->ndim) { + fprintf(stderr, "Tensors must have the same number of dimensions %d and %d for subtraction\n", tensor1->ndim, tensor2->ndim); + exit(1); + } + + if (strcmp(tensor1->device, tensor2->device) != 0) { + fprintf(stderr, "Tensors must be on the same device: %s and %s\n", tensor1->device, tensor2->device); + exit(1); + } + + char* device = (char*)malloc(strlen(tensor1->device) + 1); + if (device != NULL) { + strcpy(device, tensor1->device); + } else { + fprintf(stderr, "Memory allocation failed\n"); + exit(-1); + } + int ndim = tensor1->ndim; + int* shape = (int*)malloc(ndim * sizeof(int)); + if (shape == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + + for (int i = 0; i < ndim; i++) { + if (tensor1->shape[i] != tensor2->shape[i]) { + fprintf(stderr, "Tensors must have the same shape %d and %d at index %d for subtraction\n", tensor1->shape[i], tensor2->shape[i], i); + exit(1); + } + shape[i] = tensor1->shape[i]; + } + + if (strcmp(tensor1->device, "cuda") == 0) { + + float* result_data; + cudaMalloc((void **)&result_data, tensor1->size * sizeof(float)); + sub_tensor_cuda(tensor1, tensor2, result_data); + return create_tensor(result_data, shape, ndim, device); + } + else { + float* result_data = (float*)malloc(tensor1->size * sizeof(float)); + if (result_data == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + sub_tensor_cpu(tensor1, tensor2, result_data); + return create_tensor(result_data, shape, ndim, device); + } + } + + Tensor* sub_broadcasted_tensor(Tensor* tensor1, Tensor* tensor2) { + + if (strcmp(tensor1->device, tensor2->device) != 0) { + fprintf(stderr, "Tensors must be on the same device: %s and %s\n", tensor1->device, tensor2->device); + exit(1); + } + + int max_ndim = tensor1->ndim > tensor2->ndim ? tensor1->ndim : tensor2->ndim; + + // Determine the broadcasted shape + int* broadcasted_shape = (int*)malloc(max_ndim * sizeof(int)); + if (broadcasted_shape == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + for (int i = 0; i < max_ndim; i++) { + int dim1 = i < tensor1->ndim ? tensor1->shape[tensor1->ndim - 1 - i] : 1; + int dim2 = i < tensor2->ndim ? tensor2->shape[tensor2->ndim - 1 - i] : 1; + if (dim1 != dim2 && dim1 != 1 && dim2 != 1) { + fprintf(stderr, "Shapes are not compatible for broadcasting\n"); + exit(1); + } + broadcasted_shape[max_ndim - 1 - i] = dim1 > dim2 ? dim1 : dim2; + } + + int broadcasted_size = 1; + for (int i = 0; i < max_ndim; i++) { + broadcasted_size *= broadcasted_shape[i]; + } + + if (strcmp(tensor1->device, "cuda") == 0) { + float* result_data; + cudaMalloc((void **)&result_data, broadcasted_size * sizeof(float)); + sub_broadcasted_tensor_cuda(tensor1, tensor2, result_data, broadcasted_shape, broadcasted_size); + return create_tensor(result_data, broadcasted_shape, max_ndim, tensor1->device); + } + else { + float* result_data = (float*)malloc(broadcasted_size * sizeof(float)); + if (result_data == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + + sub_broadcasted_tensor_cpu(tensor1, tensor2, result_data, broadcasted_shape, broadcasted_size); + return create_tensor(result_data, broadcasted_shape, max_ndim, tensor1->device); + } + } + + Tensor* elementwise_mul_tensor(Tensor* tensor1, Tensor* tensor2) { + if (tensor1->ndim != tensor2->ndim) { + fprintf(stderr, "Tensors must have the same number of dimensions %d and %d for element-wise multiplication\n", tensor1->ndim, tensor2->ndim); + exit(1); + } + + if (strcmp(tensor1->device, tensor2->device) != 0) { + fprintf(stderr, "Tensors must be on the same device: %s and %s\n", tensor1->device, tensor2->device); + exit(1); + } + + char* device = (char*)malloc(strlen(tensor1->device) + 1); + if (device != NULL) { + strcpy(device, tensor1->device); + } else { + fprintf(stderr, "Memory allocation failed\n"); + exit(-1); + } + int ndim = tensor1->ndim; + int* shape = (int*)malloc(ndim * sizeof(int)); + if (shape == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + + for (int i = 0; i < ndim; i++) { + if (tensor1->shape[i] != tensor2->shape[i]) { + fprintf(stderr, "Tensors must have the same shape %d and %d at index %d for element-wise multiplication\n", tensor1->shape[i], tensor2->shape[i], i); + exit(1); + } + shape[i] = tensor1->shape[i]; + } + + if (strcmp(tensor1->device, "cuda") == 0) { + + float* result_data; + cudaMalloc((void **)&result_data, tensor1->size * sizeof(float)); + elementwise_mul_tensor_cuda(tensor1, tensor2, result_data); + return create_tensor(result_data, shape, ndim, device); + } + else { + float* result_data = (float*)malloc(tensor1->size * sizeof(float)); + if (result_data == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + elementwise_mul_tensor_cpu(tensor1, tensor2, result_data); + return create_tensor(result_data, shape, ndim, device); + } + } + + Tensor* scalar_mul_tensor(Tensor* tensor, float scalar) { + + char* device = (char*)malloc(strlen(tensor->device) + 1); + if (device != NULL) { + strcpy(device, tensor->device); + } else { + fprintf(stderr, "Memory allocation failed\n"); + exit(-1); + } + int ndim = tensor->ndim; + int* shape = (int*)malloc(ndim * sizeof(int)); + if (shape == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + + for (int i = 0; i < ndim; i++) { + shape[i] = tensor->shape[i]; + } + + if (strcmp(tensor->device, "cuda") == 0) { + + float* result_data; + cudaMalloc((void **)&result_data, tensor->size * sizeof(float)); + scalar_mul_tensor_cuda(tensor, scalar, result_data); + return create_tensor(result_data, shape, ndim, device); + } + else { + float* result_data = (float*)malloc(tensor->size * sizeof(float)); + if (result_data == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + scalar_mul_tensor_cpu(tensor, scalar, result_data); + return create_tensor(result_data, shape, ndim, device); + } + } + + Tensor* scalar_div_tensor(float scalar, Tensor* tensor) { + + char* device = (char*)malloc(strlen(tensor->device) + 1); + if (device != NULL) { + strcpy(device, tensor->device); + } else { + fprintf(stderr, "Memory allocation failed\n"); + exit(-1); + } + int ndim = tensor->ndim; + int* shape = (int*)malloc(ndim * sizeof(int)); + if (shape == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + + for (int i = 0; i < ndim; i++) { + shape[i] = tensor->shape[i]; + } + + if (strcmp(tensor->device, "cuda") == 0) { + + float* result_data; + cudaMalloc((void **)&result_data, tensor->size * sizeof(float)); + scalar_div_tensor_cuda(scalar, tensor, result_data); + return create_tensor(result_data, shape, ndim, device); + } + else { + float* result_data = (float*)malloc(tensor->size * sizeof(float)); + if (result_data == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + scalar_div_tensor_cpu(scalar, tensor, result_data); + return create_tensor(result_data, shape, ndim, device); + } + } + + Tensor* tensor_div_scalar(Tensor* tensor, float scalar) { + + char* device = (char*)malloc(strlen(tensor->device) + 1); + if (device != NULL) { + strcpy(device, tensor->device); + } else { + fprintf(stderr, "Memory allocation failed\n"); + exit(-1); + } + int ndim = tensor->ndim; + int* shape = (int*)malloc(ndim * sizeof(int)); + if (shape == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + + for (int i = 0; i < ndim; i++) { + shape[i] = tensor->shape[i]; + } + + if (strcmp(tensor->device, "cuda") == 0) { + + float* result_data; + cudaMalloc((void **)&result_data, tensor->size * sizeof(float)); + tensor_div_scalar_cuda(tensor, scalar, result_data); + return create_tensor(result_data, shape, ndim, device); + } + else { + float* result_data = (float*)malloc(tensor->size * sizeof(float)); + if (result_data == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + tensor_div_scalar_cpu(tensor, scalar, result_data); + return create_tensor(result_data, shape, ndim, device); + } + } + + Tensor* tensor_div_tensor(Tensor* tensor1, Tensor* tensor2) { + if (tensor1->ndim != tensor2->ndim) { + fprintf(stderr, "Tensors must have the same number of dimensions %d and %d for element-wise division\n", tensor1->ndim, tensor2->ndim); + exit(1); + } + + if (strcmp(tensor1->device, tensor2->device) != 0) { + fprintf(stderr, "Tensors must be on the same device: %s and %s\n", tensor1->device, tensor2->device); + exit(1); + } + + char* device = (char*)malloc(strlen(tensor1->device) + 1); + if (device != NULL) { + strcpy(device, tensor1->device); + } else { + fprintf(stderr, "Memory allocation failed\n"); + exit(-1); + } + int ndim = tensor1->ndim; + int* shape = (int*)malloc(ndim * sizeof(int)); + if (shape == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + + for (int i = 0; i < ndim; i++) { + if (tensor1->shape[i] != tensor2->shape[i]) { + fprintf(stderr, "Tensors must have the same shape %d and %d at index %d for division\n", tensor1->shape[i], tensor2->shape[i], i); + exit(1); + } + shape[i] = tensor1->shape[i]; + } + + if (strcmp(tensor1->device, "cuda") == 0) { + + float* result_data; + cudaMalloc((void **)&result_data, tensor1->size * sizeof(float)); + tensor_div_tensor_cuda(tensor1, tensor2, result_data); + return create_tensor(result_data, shape, ndim, device); + } + else { + float* result_data = (float*)malloc(tensor1->size * sizeof(float)); + if (result_data == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + tensor_div_tensor_cpu(tensor1, tensor2, result_data); + return create_tensor(result_data, shape, ndim, device); + } + } + + Tensor* matmul_tensor(Tensor* tensor1, Tensor* tensor2) { + //MxN @ NxP = MxP + // Check if tensors have compatible shapes for matrix multiplication + if (tensor1->shape[1] != tensor2->shape[0]) { + fprintf(stderr, "Incompatible shapes for matrix multiplication %dx%d and %dx%d\n", tensor1->shape[0], tensor1->shape[1], tensor2->shape[0], tensor2->shape[1]); + exit(1); + } + + if (strcmp(tensor1->device, tensor2->device) != 0) { + fprintf(stderr, "Tensors must be on the same device: %s and %s\n", tensor1->device, tensor2->device); + exit(1); + } + + char* device = (char*)malloc(strlen(tensor1->device) + 1); + if (device != NULL) { + strcpy(device, tensor1->device); + } else { + fprintf(stderr, "Memory allocation failed\n"); + exit(-1); + } + int ndim = tensor1->ndim + tensor2->ndim - 2; + int* shape = (int*)malloc(ndim * sizeof(int)); + if (shape == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + for (int i = 0; i < tensor1->ndim - 1; i++) { + shape[i] = tensor1->shape[i]; + } + for (int i = tensor1->ndim - 1; i < ndim; i++) { + shape[i] = tensor2->shape[i - tensor1->ndim + 2]; + } + + int size = 1; + for (int i = 0; i < ndim; i++) { + size *= shape[i]; + } + + float* result_data = (float*)malloc(size * sizeof(float)); + if (result_data == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + + if (strcmp(tensor1->device, "cuda") == 0) { + + float* result_data; + cudaMalloc((void **)&result_data, size * sizeof(float)); + matmul_tensor_cuda(tensor1, tensor2, result_data); + return create_tensor(result_data, shape, ndim, device); + } + else { + float* result_data = (float*)malloc(size * sizeof(float)); + if (result_data == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + matmul_tensor_cpu(tensor1, tensor2, result_data); + return create_tensor(result_data, shape, ndim, device); + } + } + + Tensor* broadcasted_batched_matmul_tensor(Tensor* tensor1, Tensor* tensor2) { + //BATCHxMxP = MxN @ BATCHxNxP + // Check if tensors have compatible shapes for matrix multiplication + if (tensor1->shape[1] != tensor2->shape[1]) { + fprintf(stderr, "Incompatible shapes for broadcasted batched matrix multiplication %dx%d and %dx%dx%d\n", tensor1->shape[0], tensor1->shape[1], tensor2->shape[0], tensor2->shape[1], tensor2->shape[2]); + exit(1); + } + + if (strcmp(tensor1->device, tensor2->device) != 0) { + fprintf(stderr, "Tensors must be on the same device: %s and %s\n", tensor1->device, tensor2->device); + exit(1); + } + + char* device = (char*)malloc(strlen(tensor1->device) + 1); + if (device != NULL) { + strcpy(device, tensor1->device); + } else { + fprintf(stderr, "Memory allocation failed\n"); + exit(-1); + } + + int ndim = 3; + int* shape = (int*)malloc(ndim * sizeof(int)); + if (shape == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + + shape[0] = tensor2->shape[0];; + shape[1] = tensor1->shape[0]; + shape[2] = tensor2->shape[2]; + + int size = 1; + for (int i = 0; i < ndim; i++) { + size *= shape[i]; + } + + float* result_data = (float*)malloc(size * sizeof(float)); + if (result_data == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + + if (strcmp(tensor1->device, "cuda") == 0) { + + float* result_data; + cudaMalloc((void **)&result_data, size * sizeof(float)); + broadcasted_batched_matmul_tensor_cuda(tensor1, tensor2, result_data); + return create_tensor(result_data, shape, ndim, device); + } + else { + float* result_data = (float*)malloc(size * sizeof(float)); + if (result_data == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + broadcasted_batched_matmul_tensor_cpu(tensor1, tensor2, result_data); + return create_tensor(result_data, shape, ndim, device); + } + + } + + Tensor* batched_matmul_tensor(Tensor* tensor1, Tensor* tensor2) { + //BATCHxMxP = BATCHxMxN @ BATCHxNxP + // Check if tensors have compatible shapes for matrix multiplication + + if (tensor1->shape[0] != tensor2->shape[0]) { + fprintf(stderr, "Tensors must have same batch dimension for batch matmul %d and %d\n", tensor1->shape[0], tensor2->shape[0]); + exit(1); + } + + if (tensor1->shape[2] != tensor2->shape[1]) { + fprintf(stderr, "Incompatible shapes for matrix multiplication %dx%d and %dx%d\n", tensor1->shape[0], tensor1->shape[1], tensor2->shape[0], tensor2->shape[1]); + exit(1); + } + + if (strcmp(tensor1->device, tensor2->device) != 0) { + fprintf(stderr, "Tensors must be on the same device: %s and %s\n", tensor1->device, tensor2->device); + exit(1); + } + + char* device = (char*)malloc(strlen(tensor1->device) + 1); + if (device != NULL) { + strcpy(device, tensor1->device); + } else { + fprintf(stderr, "Memory allocation failed\n"); + exit(-1); + } + + int ndim = 3; + int* shape = (int*)malloc(ndim * sizeof(int)); + if (shape == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + + shape[0] = tensor2->shape[0];; + shape[1] = tensor1->shape[1]; + shape[2] = tensor2->shape[2]; + + int size = 1; + for (int i = 0; i < ndim; i++) { + size *= shape[i]; + } + + float* result_data = (float*)malloc(size * sizeof(float)); + if (result_data == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + + if (strcmp(tensor1->device, "cuda") == 0) { + + float* result_data; + cudaMalloc((void **)&result_data, size * sizeof(float)); + batched_matmul_tensor_cuda(tensor1, tensor2, result_data); + return create_tensor(result_data, shape, ndim, device); + } + else { + float* result_data = (float*)malloc(size * sizeof(float)); + if (result_data == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + batched_matmul_tensor_cpu(tensor1, tensor2, result_data); + return create_tensor(result_data, shape, ndim, device); + } + + } + + + Tensor* tensor_pow_scalar(Tensor* tensor, float exponent) { + char* device = (char*)malloc(strlen(tensor->device) + 1); + if (device != NULL) { + strcpy(device, tensor->device); + } else { + fprintf(stderr, "Memory allocation failed\n"); + exit(-1); + } + int ndim = tensor->ndim; + int* shape = (int*)malloc(ndim * sizeof(int)); + if (shape == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + + for (int i = 0; i < ndim; i++) { + shape[i] = tensor->shape[i]; + } + + if (strcmp(tensor->device, "cuda") == 0) { + + float* result_data; + cudaMalloc((void **)&result_data, tensor->size * sizeof(float)); + tensor_pow_scalar_cuda(tensor, exponent, result_data); + return create_tensor(result_data, shape, ndim, device); + } + else { + float* result_data = (float*)malloc(tensor->size * sizeof(float)); + if (result_data == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + tensor_pow_scalar_cpu(tensor, exponent, result_data); + return create_tensor(result_data, shape, ndim, device); + } + } + + Tensor* scalar_pow_tensor(float base, Tensor* tensor) { + char* device = (char*)malloc(strlen(tensor->device) + 1); + if (device != NULL) { + strcpy(device, tensor->device); + } else { + fprintf(stderr, "Memory allocation failed\n"); + exit(-1); + } + int ndim = tensor->ndim; + int* shape = (int*)malloc(ndim * sizeof(int)); + if (shape == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + + for (int i = 0; i < ndim; i++) { + shape[i] = tensor->shape[i]; + } + + if (strcmp(tensor->device, "cuda") == 0) { + + float* result_data; + cudaMalloc((void **)&result_data, tensor->size * sizeof(float)); + scalar_pow_tensor_cuda(base, tensor, result_data); + return create_tensor(result_data, shape, ndim, device); + } + else { + float* result_data = (float*)malloc(tensor->size * sizeof(float)); + if (result_data == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + scalar_pow_tensor_cpu(base, tensor, result_data); + return create_tensor(result_data, shape, ndim, device); + } + } + + Tensor* log_tensor(Tensor* tensor) { + char* device = (char*)malloc(strlen(tensor->device) + 1); + if (device != NULL) { + strcpy(device, tensor->device); + } else { + fprintf(stderr, "Memory allocation failed\n"); + exit(-1); + } + int ndim = tensor->ndim; + int* shape = (int*)malloc(ndim * sizeof(int)); + if (shape == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + + for (int i = 0; i < ndim; i++) { + shape[i] = tensor->shape[i]; + } + + if (strcmp(tensor->device, "cuda") == 0) { + + float* result_data; + cudaMalloc((void **)&result_data, tensor->size * sizeof(float)); + log_tensor_cuda(tensor, result_data); + return create_tensor(result_data, shape, ndim, device); + } + else { + float* result_data = (float*)malloc(tensor->size * sizeof(float)); + if (result_data == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + log_tensor_cpu(tensor, result_data); + return create_tensor(result_data, shape, ndim, device); + } + } + + Tensor* reshape_tensor(Tensor* tensor, int* new_shape, int new_ndim) { + char* device = (char*)malloc(strlen(tensor->device) + 1); + if (device != NULL) { + strcpy(device, tensor->device); + } else { + fprintf(stderr, "Memory allocation failed\n"); + exit(-1); + } + + int ndim = new_ndim; + int* shape = (int*)malloc(ndim * sizeof(int)); + if (shape == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + + for (int i = 0; i < ndim; i++) { + shape[i] = new_shape[i]; + } + + // Calculate the total number of elements in the new shape + int size = 1; + for (int i = 0; i < new_ndim; i++) { + size *= shape[i]; + } + + // Check if the total number of elements matches the current tensor's size + if (size != tensor->size) { + fprintf(stderr, "Cannot reshape tensor. Total number of elements in new shape does not match the current size of the tensor.\n"); + exit(1); + } + + if (strcmp(tensor->device, "cuda") == 0) { + + float* result_data; + cudaMalloc((void **)&result_data, tensor->size * sizeof(float)); + assign_tensor_cuda(tensor, result_data); + return create_tensor(result_data, shape, ndim, device); + } + else { + float* result_data = (float*)malloc(tensor->size * sizeof(float)); + if (result_data == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + assign_tensor_cpu(tensor, result_data); + return create_tensor(result_data, shape, ndim, device); + } + } + + Tensor* equal_tensor(Tensor* tensor1, Tensor* tensor2) { + if (tensor1->ndim != tensor2->ndim) { + fprintf(stderr, "Tensors must have the same number of dimensions %d and %d for equal\n", tensor1->ndim, tensor2->ndim); + exit(1); + } + + if (strcmp(tensor1->device, tensor2->device) != 0) { + fprintf(stderr, "Tensors must be on the same device: %s and %s\n", tensor1->device, tensor2->device); + exit(1); + } + + char* device = (char*)malloc(strlen(tensor1->device) + 1); + if (device != NULL) { + strcpy(device, tensor1->device); + } else { + fprintf(stderr, "Memory allocation failed\n"); + exit(-1); + } + int ndim = tensor1->ndim; + int* shape = (int*)malloc(ndim * sizeof(int)); + if (shape == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + + for (int i = 0; i < ndim; i++) { + if (tensor1->shape[i] != tensor2->shape[i]) { + fprintf(stderr, "Tensors must have the same shape %d and %d at index %d for equal\n", tensor1->shape[i], tensor2->shape[i], i); + exit(1); + } + shape[i] = tensor1->shape[i]; + } + + if (strcmp(tensor1->device, "cuda") == 0) { + + float* result_data; + cudaMalloc((void **)&result_data, tensor1->size * sizeof(float)); + equal_tensor_cuda(tensor1, tensor2, result_data); + return create_tensor(result_data, shape, ndim, device); + } + else { + float* result_data = (float*)malloc(tensor1->size * sizeof(float)); + if (result_data == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + equal_tensor_cpu(tensor1, tensor2, result_data); + return create_tensor(result_data, shape, ndim, device); + } + } + + Tensor* equal_broadcasted_tensor(Tensor* tensor1, Tensor* tensor2) { + + if (strcmp(tensor1->device, tensor2->device) != 0) { + fprintf(stderr, "Tensors must be on the same device: %s and %s\n", tensor1->device, tensor2->device); + exit(1); + } + + int max_ndim = tensor1->ndim > tensor2->ndim ? tensor1->ndim : tensor2->ndim; + + // Determine the broadcasted shape + int* broadcasted_shape = (int*)malloc(max_ndim * sizeof(int)); + if (broadcasted_shape == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + for (int i = 0; i < max_ndim; i++) { + int dim1 = i < tensor1->ndim ? tensor1->shape[tensor1->ndim - 1 - i] : 1; + int dim2 = i < tensor2->ndim ? tensor2->shape[tensor2->ndim - 1 - i] : 1; + if (dim1 != dim2 && dim1 != 1 && dim2 != 1) { + fprintf(stderr, "Shapes are not compatible for broadcasting\n"); + exit(1); + } + broadcasted_shape[max_ndim - 1 - i] = dim1 > dim2 ? dim1 : dim2; + } + + int broadcasted_size = 1; + for (int i = 0; i < max_ndim; i++) { + broadcasted_size *= broadcasted_shape[i]; + } + + if (strcmp(tensor1->device, "cuda") == 0) { + float* result_data; + cudaMalloc((void **)&result_data, broadcasted_size * sizeof(float)); + equal_broadcasted_tensor_cuda(tensor1, tensor2, result_data, broadcasted_shape, broadcasted_size); + return create_tensor(result_data, broadcasted_shape, max_ndim, tensor1->device); + } + else { + float* result_data = (float*)malloc(broadcasted_size * sizeof(float)); + if (result_data == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + + equal_broadcasted_tensor_cpu(tensor1, tensor2, result_data, broadcasted_shape, broadcasted_size); + return create_tensor(result_data, broadcasted_shape, max_ndim, tensor1->device); + } + } + + + Tensor* ones_like_tensor(Tensor* tensor) { + char* device = (char*)malloc(strlen(tensor->device) + 1); + if (device != NULL) { + strcpy(device, tensor->device); + } else { + fprintf(stderr, "Memory allocation failed\n"); + exit(-1); + } + int ndim = tensor->ndim; + int* shape = (int*)malloc(ndim * sizeof(int)); + if (shape == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + + for (int i = 0; i < ndim; i++) { + shape[i] = tensor->shape[i]; + } + + if (strcmp(tensor->device, "cuda") == 0) { + + float* result_data; + cudaMalloc((void **)&result_data, tensor->size * sizeof(float)); + ones_like_tensor_cuda(tensor, result_data); + return create_tensor(result_data, shape, ndim, device); + } + else { + float* result_data = (float*)malloc(tensor->size * sizeof(float)); + if (result_data == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + ones_like_tensor_cpu(tensor, result_data); + return create_tensor(result_data, shape, ndim, device); + } + } + + Tensor* zeros_like_tensor(Tensor* tensor) { + char* device = (char*)malloc(strlen(tensor->device) + 1); + if (device != NULL) { + strcpy(device, tensor->device); + } else { + fprintf(stderr, "Memory allocation failed\n"); + exit(-1); + } + int ndim = tensor->ndim; + int* shape = (int*)malloc(ndim * sizeof(int)); + if (shape == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + + for (int i = 0; i < ndim; i++) { + shape[i] = tensor->shape[i]; + } + + if (strcmp(tensor->device, "cuda") == 0) { + + float* result_data; + cudaMalloc((void **)&result_data, tensor->size * sizeof(float)); + zeros_like_tensor_cuda(tensor, result_data); + return create_tensor(result_data, shape, ndim, device); + } + else { + float* result_data = (float*)malloc(tensor->size * sizeof(float)); + if (result_data == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + zeros_like_tensor_cpu(tensor, result_data); + return create_tensor(result_data, shape, ndim, device); + } + } + + Tensor* sin_tensor(Tensor* tensor) { + char* device = (char*)malloc(strlen(tensor->device) + 1); + if (device != NULL) { + strcpy(device, tensor->device); + } else { + fprintf(stderr, "Memory allocation failed\n"); + exit(-1); + } + int ndim = tensor->ndim; + int* shape = (int*)malloc(ndim * sizeof(int)); + if (shape == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + + for (int i = 0; i < ndim; i++) { + shape[i] = tensor->shape[i]; + } + + if (strcmp(tensor->device, "cuda") == 0) { + + float* result_data; + cudaMalloc((void **)&result_data, tensor->size * sizeof(float)); + sin_tensor_cuda(tensor, result_data); + return create_tensor(result_data, shape, ndim, device); + } + else { + float* result_data = (float*)malloc(tensor->size * sizeof(float)); + if (result_data == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + sin_tensor_cpu(tensor, result_data); + return create_tensor(result_data, shape, ndim, device); + } + } + + Tensor* cos_tensor(Tensor* tensor) { + char* device = (char*)malloc(strlen(tensor->device) + 1); + if (device != NULL) { + strcpy(device, tensor->device); + } else { + fprintf(stderr, "Memory allocation failed\n"); + exit(-1); + } + int ndim = tensor->ndim; + int* shape = (int*)malloc(ndim * sizeof(int)); + if (shape == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + + for (int i = 0; i < ndim; i++) { + shape[i] = tensor->shape[i]; + } + + if (strcmp(tensor->device, "cuda") == 0) { + + float* result_data; + cudaMalloc((void **)&result_data, tensor->size * sizeof(float)); + cos_tensor_cuda(tensor, result_data); + return create_tensor(result_data, shape, ndim, device); + } + else { + float* result_data = (float*)malloc(tensor->size * sizeof(float)); + if (result_data == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + cos_tensor_cpu(tensor, result_data); + return create_tensor(result_data, shape, ndim, device); + } + } + + Tensor* sigmoid_tensor(Tensor* tensor) { + char* device = (char*)malloc(strlen(tensor->device) + 1); + if (device != NULL) { + strcpy(device, tensor->device); + } else { + fprintf(stderr, "Memory allocation failed\n"); + exit(-1); + } + int ndim = tensor->ndim; + int* shape = (int*)malloc(ndim * sizeof(int)); + if (shape == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + + for (int i = 0; i < ndim; i++) { + shape[i] = tensor->shape[i]; + } + + if (strcmp(tensor->device, "cuda") == 0) { + + float* result_data; + cudaMalloc((void **)&result_data, tensor->size * sizeof(float)); + sigmoid_tensor_cuda(tensor, result_data); + return create_tensor(result_data, shape, ndim, device); + } + else { + float* result_data = (float*)malloc(tensor->size * sizeof(float)); + if (result_data == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + sigmoid_tensor_cpu(tensor, result_data); + return create_tensor(result_data, shape, ndim, device); + } + } + + Tensor* transpose_tensor(Tensor* tensor) { + char* device = (char*)malloc(strlen(tensor->device) + 1); + if (device != NULL) { + strcpy(device, tensor->device); + } else { + fprintf(stderr, "Memory allocation failed\n"); + exit(-1); + } + + int ndim = tensor->ndim; + int* shape = (int*)malloc(ndim * sizeof(int)); + if (shape == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(-1); + } + + for (int i = 0; i < ndim; i++) { + shape[i] = tensor->shape[ndim - 1 - i]; + } + + int size = tensor->size; + + if (strcmp(tensor->device, "cuda") == 0) { + + float* result_data; + cudaMalloc((void **)&result_data, size * sizeof(float)); + switch (ndim) { + case 1: + transpose_1D_tensor_cuda(tensor, result_data); + break; + case 2: + transpose_2D_tensor_cuda(tensor, result_data); + break; + case 3: + transpose_3D_tensor_cuda(tensor, result_data); + break; + default: + fprintf(stderr, "Transpose only supports tensors up to 3 dimensions.\n"); + exit(-1); + } + return create_tensor(result_data, shape, ndim, device); + } + else { + float* result_data = (float*)malloc(size * sizeof(float)); + if (result_data == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + switch (ndim) { + case 1: + transpose_1D_tensor_cpu(tensor, result_data); + break; + case 2: + transpose_2D_tensor_cpu(tensor, result_data); + break; + case 3: + transpose_3D_tensor_cpu(tensor, result_data); + break; + default: + fprintf(stderr, "Transpose only supports tensors up to 3 dimensions.\n"); + exit(-1); + } + return create_tensor(result_data, shape, ndim, device); + } + } + + Tensor* transpose_axes_tensor(Tensor* tensor, int axis1, int axis2) { + char* device = (char*)malloc(strlen(tensor->device) + 1); + if (device != NULL) { + strcpy(device, tensor->device); + } else { + fprintf(stderr, "Memory allocation failed\n"); + exit(-1); + } + + int ndim = tensor->ndim; + int* shape = (int*)malloc(ndim * sizeof(int)); + if (shape == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(-1); + } + + for (int i = 0; i < ndim; i++) { + shape[i] = tensor->shape[i]; + } + + shape[axis1] = tensor->shape[axis2]; + shape[axis2] = tensor->shape[axis1]; + + int size = tensor->size; + + if (strcmp(tensor->device, "cuda") == 0) { + + float* result_data; + cudaMalloc((void **)&result_data, size * sizeof(float)); + assign_tensor_cuda(tensor, result_data); + + Tensor* new_tensor = create_tensor(result_data, shape, ndim, device); + for (int i = 0; i < ndim; i++) { + new_tensor->strides[i] = tensor->strides[i]; + } + new_tensor->strides[axis1] = tensor->strides[axis2]; + new_tensor->strides[axis2] = tensor->strides[axis1]; + make_contiguous(new_tensor); + return new_tensor; + } + else { + float* result_data = (float*)malloc(size * sizeof(float)); + if (result_data == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + exit(1); + } + assign_tensor_cpu(tensor, result_data); + + Tensor* new_tensor = create_tensor(result_data, shape, ndim, device); + for (int i = 0; i < ndim; i++) { + new_tensor->strides[i] = tensor->strides[i]; + } + new_tensor->strides[axis1] = tensor->strides[axis2]; + new_tensor->strides[axis2] = tensor->strides[axis1]; + make_contiguous(new_tensor); + return new_tensor; + } + } + + void make_contiguous(Tensor* tensor) { + + int* new_strides = (int*)malloc(tensor->ndim * sizeof(int)); + if (new_strides == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + } + + // Calculate new strides assuming C-contiguous order + int stride = 1; + for (int i = tensor->ndim - 1; i >= 0; i--) { + new_strides[i] = stride; + stride *= tensor->shape[i]; + } + + if (strcmp(tensor->device, "cuda") == 0) { + float* result_data; + cudaMalloc((void **)&result_data, tensor->size * sizeof(float)); + make_contiguous_tensor_cuda(tensor, result_data, new_strides); + + } else { + float* result_data = (float*)malloc(tensor->size * sizeof(float)); + if (result_data == NULL) { + fprintf(stderr, "Memory allocation failed\n"); + } + make_contiguous_tensor_cpu(tensor, result_data, new_strides); + } + } +} diff --git a/build/lib/norch/csrc/tensor.h b/build/lib/norch/csrc/tensor.h new file mode 100644 index 0000000..5669d4f --- /dev/null +++ b/build/lib/norch/csrc/tensor.h @@ -0,0 +1,45 @@ +#ifndef TENSOR_H +#define TENSOR_H + +typedef struct { + float* data; + int* strides; + int* shape; + int* strides_cuda; + int* shape_cuda; + int ndim; + int size; + char* device; +} Tensor; + +extern "C" { + Tensor* create_tensor(float* data, int* shape, int ndim, char* device); + float get_item(Tensor* tensor, int* indices); + Tensor* add_tensor(Tensor* tensor1, Tensor* tensor2); + Tensor* sum_tensor(Tensor* tensor, int axis, bool keepdims); + Tensor* max_tensor(Tensor* tensor, int axis, bool keepdim); + Tensor* min_tensor(Tensor* tensor, int axis, bool keepdim); + Tensor* sub_tensor(Tensor* tensor1, Tensor* tensor2); + Tensor* elementwise_mul_tensor(Tensor* tensor1, Tensor* tensor2); + Tensor* scalar_mul_tensor(Tensor* tensor, float scalar); + Tensor* scalar_div_tensor(float scalar, Tensor* tensor); + Tensor* tensor_div_scalar(Tensor* tensor, float scalar); + Tensor* tensor_div_tensor(Tensor* tensor1, Tensor* tensor2); + Tensor* reshape_tensor(Tensor* tensor, int* new_shape, int new_ndim); + Tensor* matmul_tensor(Tensor* tensor1, Tensor* tensor2); + Tensor* tensor_pow_scalar(Tensor* tensor, float exponent); + Tensor* scalar_pow_tensor(float base, Tensor* tensor); + Tensor* log_tensor(Tensor* tensor); + Tensor* equal_tensor(Tensor* tensor1, Tensor* tensor2); + Tensor* equal_broadcasted_tensor(Tensor* tensor1, Tensor* tensor2); + void to_device(Tensor* tensor, char* device); + Tensor* ones_like_tensor(Tensor* tensor); + Tensor* zeros_like_tensor(Tensor* tensor); + Tensor* sin_tensor(Tensor* tensor); + Tensor* cos_tensor(Tensor* tensor); + Tensor* transpose_tensor(Tensor* tensor); + Tensor* transpose_axes_tensor(Tensor* tensor, int axis1, int axis2); + void make_contiguous(Tensor* tensor); +} + +#endif /* TENSOR_H */ diff --git a/build/lib/norch/distributed/__init__.py b/build/lib/norch/distributed/__init__.py new file mode 100644 index 0000000..a979451 --- /dev/null +++ b/build/lib/norch/distributed/__init__.py @@ -0,0 +1 @@ +from .distributed import * \ No newline at end of file diff --git a/build/lib/norch/distributed/distributed.py b/build/lib/norch/distributed/distributed.py new file mode 100644 index 0000000..d033ede --- /dev/null +++ b/build/lib/norch/distributed/distributed.py @@ -0,0 +1,24 @@ +import os +import ctypes +from norch import Tensor, CTensor + +def init_process_group(rank, world_size, backend='nccl'): + + Tensor._C.init_process_group.argtypes = [ctypes.c_int, ctypes.c_int] + Tensor._C.init_process_group.restype = None + + Tensor._C.init_process_group(rank, world_size) + +def broadcast_tensor(tensor): + + Tensor._C.broadcast_tensor.argtypes = [ctypes.POINTER(CTensor)] + Tensor._C.broadcast_tensor.restype = None + + Tensor._C.broadcast_tensor(tensor) + +def allreduce_sum_tensor(tensor): + + Tensor._C.allreduce_mean_tensor.argtypes = [ctypes.POINTER(CTensor)] + Tensor._C.allreduce_mean_tensor.restype = None + + Tensor._C.allreduce_mean_tensor(tensor) diff --git a/build/lib/norch/libtensor.so b/build/lib/norch/libtensor.so new file mode 100755 index 0000000000000000000000000000000000000000..b70187e20ebe1e6f6697ada3e36411d6a0bb0549 GIT binary patch literal 401208 zcmeFad0Z9c`~N@S5}FIHQK^9|YAT1#V`hSi9`}QYk*08D5tL09Max3X!puTTv&F(v zBYRs|SyY->TC`YLS+rPKSy)+Umb@$Tciqd}=XGw*Xz$PWzu%*&+r#TQ*Id`V&3)f9 zX9jMKnKahd$Hy?YX2x{}Q@JAri;_Evy0$b;j3~oyTuT4;GJ0~`=4wyVAIha~^V9Gb zHq3hq_M^Xyt$i+EDY*HG$MJY&zQU3A8NEty^AnF_B?uhIqZ)z3JwJ;Bg}b^S@o65% zys_gNqs{8ES=eFj>K6jS+bRAO?x_*B^nUOr77d)_P%;#_(BTj2RGYIE=0VD~tet!?w;b=6R!fQtwNX{6m`eHf#~C z{j1vfl^M-TONv{!+2(6>-V)x^x5Jj==7FOOKdP-2e!Yvo=xLM%8&QEqajnl|qx_ne zv?wti^NAVde{BmN-{w6HU%wRN_Dg)G8Ah>Tl$NC!!-AZ}fl(xk&W3MDo2^livCUdV zMHwMc{Q~=M^7r-kJJP&!jp5_)4XpBu3hc9g%L{>y<$?aiD-E(TQhbfpNfj$wZ;wvt z$@;s0l87M9(Z_ghfAfG&{xgi_f%h+OKG#^W z+Glz3L}v#oK-3!(R`d((6yiH3ZiP`+>=PW=((nuH*u1%a(3PF5?MW#uqx_Akb%E7U z>)YJX+p#OQ`KB3ufvp2>AJfUulqx3zcdP+Z0YM^wA((jZS zDg8-_ZGW?WWQSzKnlGj1lv+|^{fcdu(q{)sohfyZ>@xbioYIw)x={+E)QeJYO8qGH zr!CmWJSQ;MPFpv1QE^f`gjL`st>O`$ZEQYUdVx|UrI#t~p|qFMtCXrJ?WgoQrGu2r?M?dmZA$M*_8xs6rt|@&k0^ai z=~GHyP&!KK7^SZ%eM{+kO7)a}q;!%J+kTtDIz#DqN@pqkLFpW&^OV@;>r3Mo zrAsLJQDR#g`fN+7J*AG6I#arg(&dz{pwx}hRg~EL5=5ytr9PB`DGj7_H6^wUq0bQc zb0~d=QHr26jM6ofhEp0rX(Xj+N~0;UZ5(}$r!;}mWJ*&h#Zro=bUmdTDBVbjZL{e! ziPBt3bT0EPjXu*U&8M_LvP}BSl0RqD=R)~&0eu$9pKqnlVoED0t)z52rPY+~q_mb& zDW!Fk)>AUKjr8;Vlpdt?u;iQNXBmAyM(J@%TPe|yXKbhR1f_CHJ19LxX(y!$O1miS zrc_Dk6-rfL*JRN{~mDo=zE+08zsFo>Lz@5HmjgxhZd@MNSLdRUWzNKhFg? zmwmdU=f>acXimTS&^!VqE_dhkE#e$!^?A_bOf6UP--AkTW*duy%%ONep)0#bXGU68S;cxu#^1E%1Js#j_`~C|H(jJVP z((92IzkbobI_8t~0o_gvdMIz%D|1J*b`DLO6Oujc@07y84QekmsfskD6DOQ@Ag;$I=~Ne*XF@ztx2sFFX9=sVT1)w|XGx zM2}~WbbrovYU2GHj zw#b;<{=hTmiyw>`aCXI)hdMsw=({(#;Kq}K9(rZp+xaa|rv-E!GV}G+x^p$g(+9?N zY1@BPuR}*3`7CnopU?UhWJH%|1V!1(o_lxEZ+FiB^sNCco^5y6#sQUA^}hYhVUBB? zPnuVs@<9A|JNovUdg$QBhjt!%XSdJ9MPtrHkM3uC?dL_u{%oCFKfGP%c1OnCdT{=n zF|Ya_%|0`A{kq_}(Sz1*d97>I(A#@;ioSB_tAp&P;tS3$4LyxM7kDmC~!cPY6$p7oPUnXDoTGbB`=brIDJbGOBsz3I; zeaqR_tG}5Uzj)L03$Bm%@6|7P=(qJV7v0gnQ_;AXhO3X;m#2RC?G4#4ubA@a&XZ>+ zzS8@Z`#zig)^+9K9S?-t7xu|p@Wr>j`Hy!0uy*_BoswU9s^yGp@;fc~?ylmwcMY(| z{jzGv6Y-bdmG$;3SCw8nq}`jnFK_s*yyo^cZJU)|`sa?9%Rji&XT%!^lYEaIdVc70 zmF=EB+$x}A#(#SEuWC7F^mEsr$=)z;CfA5ZP{UUVXwot{dMm6w2f27 zjo<$2tGjws*7gnC)$;wmPt=X+p4qQFJi31Nfj{;P>2_0i`9odS z&M*4##+CgZT`?naQdpnEa~9k^;^_1zeD<{&J>|vJ&ZkyweLk%w_<^EHgOBZ)`qA8k z+wN!;-`zK3^SL8W$1S<)v3*aq3~S8$d;P)}Z)kV^^<@uSm*2_%%a@-$Ug=3c%dSKfBZsnO>$9_lrI?70;qADGkjig!K>JCS++laJXq)<3xD-Xl97 z{iQT&=uO*lcij46<>T8zM%4H}v~lUo-2e3I_jdB#y@QtC`u60n0xo;8?Vg+d_~h*R z`oE^<1pcz^!|k7~n|d^+;g6o@Zn}MFSk3v9zb$>{zwM`XT-vNzMCVzzZk_mcx4IQK zUlTaZSbop;?{bo^ezM2umskJ1E$qOc0h3Lo_C8~5)R`swq>pL*Xm`K_v(iwpOTUG{LniGh93%&!SvkvgL;{`V@He`++wl)i=JN`pEver-!}?#UY|6)xa-Q;W)JQA`m5WYd3*n``=V~@UGw~< zPfeaM?z$QGuDJZ~j*EUA`DoCKqn1s6rC0I9>z|BRbaTzS5%!mjV|^xl`PT=_|Joku zxPJAi5u0zG)auQ#Ef4JerRLcB4i7AUt^Nj{{7(tZ!aorvm@jBb(@9` znsjQ$Pvh4Al-BC9`*##oOq8`+mb<( z9$)bH4X<3k;?W)79WUL#tnZ{(Vjf=CX-G-uHgO;A2^cbT-)~2nx9RzA>9We--nr+g zU0Db3s(Jg7z={w1P3SxR-DA;3gX%x>|2yKtd)AHk##|40`#f?vt)sl`@Co(Kub{P= zH~Vjg(K_CXpCAkS#dhz08biGEv%S3YHG$rFvjN`utiIm){*m7Kr4iow!#%un`!(MA z-k#q1z1_X@0XFaaN{e>+X|i|w8!hs4t3^Ezw5Xr{)V|)@{iVyi^T=-AdGjIO`LlGl z!JGdj0p9tTtGx4g>Vw|=+-6Y^Z(8KDeWZ6kW63`?Ws_|^5pQuH>*}2!9_pR{Y0)26 zS=96E*LwH!w?+S~u{cjpTeNR+KkxoyuJF!p>Fu3=d8K#mXTd8i&i6Ena&_tC-GBUG z?>x@pyk0cOyZu)d?KsP#9S_jWHE;dlQ49ZV>H5i=eW67?+-wndmqkDK5A^Q;yGy_}@zP=}+gj zzAd!qw`VQtp~PaGx#V*1<$chi-rlqrADUb6-!0mAjm3F!Fvxp;X7u;Yzq#5w-+Y~S ze!oR|U$U6L_Jw-)Gt^=pd4L+qTmQMmVtl(H%)5R6w%+;M7U%Ep!QSnsSoHI;7UyYy zi+Vn3Q4gP4%-07k{5)gPUR4(LGuEQK{VnqIphbOtZZVJCYjHjEXKEvS9mX1-x1#Z=UU9mw}gB5)7+vT=2+CjQHybAnMJ$wv*;(c zk-fM2xyE81>1J{MCR+5j8jE#N2a9suWKqwJ7X9-{i}HSK!9TYc=XYAv|6>;8|0xSU zoh{C<7ik>xHeY{l(H~A&oG&k0jH?4J#y9qTmA854DT{GzMWlE8Ef)PP(xM)+Eb>3e zqMk!6`pKIX=lgVvap@y`nAFX)7%H{~n9$!M`oyZnZeSMp(=j$1K|G zPmA(Sw;0FHS*-uBi1FU;z834l=@#|#rbXPL7U%DKW4y<`!=j(}w3sjcvgo&WS-mU$VD?V=WxHwrQ+xV8+XyhCM$mz@CGdagyYkk;Pvi5#!Dh+h==@@S z+OaR{X`3qfyHq)xzbiPMwtU;d$C-A<71Kq2sQdA)fcgoGTP&aBqGsaTT53n0PxeJT zZ8RSPrrpN~q0%Y7yW2OHY!hee7U-9w3 zo$=Zf!Iw(@3N;Y#SMqtiDjC;@8kCi{;!f_HaA4N|L7IP={TcQU2W|KA ze+}an*$?N)erVRSA=|f`*1iu!i~L`fA>vXs^3BIj@DA*u4B7%^{>M{=u(u9?VB zPxcTJZ3Dz^Gog4?QbqY?W zIp6m65%wwU;T_r*@_*@B5ZMpkyi0KQEDqahQyx{9k$wOO9^`nh8$b zl5aifd}ndLkmI4*-=2{DZ4rB*iZ-*~#`z0-hi3mH<)7tq(|D0jLuAXiger>pZ{AAy z$>ZMWxgi?xS^dY#^V&R*K9cpeggul-+l|sbK(q*Fyd*bWq$0kTuIVTS6Udca+Ulk{Cpt!J5)?ouE`04 zUoZQ4H#)DGALCk)AJfmwHK?08f{!Hl;=^!og$z)ZhRx>%}Uvh)U15Voh0HeVGqF3 zX3lHvXg$yB=RMhPBl%u>Hk9gvwaf8=!rq+6M_(!I*UNrI)xx))G!ZaA8(RwdrIOp_ zym@Gq;8b0FbIJ3iuXesnqYN-VxA_Zu${yeDmE(V-9RF#$;akvE!v4l%q8`lou9~_H z%fB;U@IA79=ke>2cE)w=;Xm5UdPtQ0!FHYCW_e>R=A9(jUi?8y@?(x0BWc0O@;`t* zkV)H}{9k&$knZC#{w#awleYPi?~fMwd}qFhYt9!l^HuEl)xA@y@sKl*$zRgUv%a=hv&{X9xPF#AS1 z4w&Qc4|06a=ZkHlME+M6h;o_zq?E=h=BG@qbIf{X&u_BwKD1Nhf1)_dsFeBn=ncWC zd-H9*JWuQ7c}F7`-?oGb|8}_!oG$s5(Zc_^WZ~Fs_o~r?FD({4OxoWp>qj5AOXT=7 zBuChrS&X0eMTxlKbA+E+(mqM%|7&?&5HEQl`Qh`$STR1!%MW8B zUEi_mm(nMNpVrd8AV}21cG+=qC7&5DN$i1d+RX8-rkn5+^||0h zd@nsiO%2D|FWKu&g|7q+2 zYTD*X`+McQHj_PgO`ACmd??40Nuz|lc|J~&(%!$X;2+EN zYA?zE3=;f3Ii8s1T}lHH^V8@f%6negzbNO0KtI9rB=@EBmzB3x&hO?tk|Epuzw7~W z+RS>3lJms}ay@UJM_)<)m^P1`4Hx-o-A?#1>#cE|sE5!L!DaU{#zzajGgWYNd^jWX zL+|JC{Fv*LPO`i+{}S9RZ-lJ>+AV^c_5X$JSM%751+~Q_JQ&|?I7(>N^X~& zhHJikDd*F8yYQ1FvTH1-j=|b}{SN}HlJ@V*`u{>+pVD%gZ!gMnm1*MEZyYOGaUXGwGng44TM`(a$ardwnC}?{|@;BwU*G&$DGbLY74a&~zX!&A>>8DQC ze^>2#e1sh5^W->h=6^#^5%*>GQU-12^-hP;!vEQLDqg4<$CtqqY$E!I}!rs+a*!PwBS=&tTcjgEO z=6LlWRcr^Nhfy_4i>$Dg@2m<_%^+d z;0bd6rEbHw@ifje`#0n|%^b%@(hrO;Y9;b-&I_+zA^1<92><4Jca`+Nl--D+Z3+LE z-Zzr#jbb@owUfL-_AC8*t+uo9A1>Fcrk`3mkmX0e-!)8*|35q-;x3c^Tl5h2zq1#% zXfwy{V!5v9CKrt6{FN=op8*Gi{bl0AxLeMTHS+%3RO$a3IZo>1f5JG?zSqePW%lR$ zM+<)V7Ezz%mT!~gI$_RdBH#^@@0cX)+ukPZt7LxK%lRwNCb)SYW|chePRsL-PIJCJ zKn=v|Emz(rG3Vt}Ij+8aOgJ{{CyLy%e%`C6@P8e}W7}rgF4girUZ%jtVLARgwR(O` z)=#!JKl(?>awQ1=WIf}8hGiY^!V=P%(=%tRUY7!53}Tw~?+%X-OQl=ZVH zO_a-=r@oTyxL#Z5w22EyiaSc_r~-T{zu92$84|9q`=9NDw_43YDKIS!Q3URDqNM~Q%}Uf8ye1{P+&L$0e0nV(Nv3;PG; z_aJ6_otNW`KEHI8`A?GhZzJ3rzxxaSrLsQQ$5$#razps4`vA%RlH*dUJ<=smcAT0k6|0=lY=c6FOZ_%!I zddus(@17C%rP6;3Sr4scJy3J-ZHLABt;iwD^{`x@oBb_%l;FN{-j9`j*fNjR!;B|I z+&{#J@qM7k=Ogm|_MOuH?igYJ!Zgo%UU8+c|FTv%NRsv)<$RhX=TkaO`1Xg)r}@(D zei`>;nV*AlAT-x4<7I#TK(50Rq#u9TA0|x}`7!(Lonu5k+pQKnMB3juP~>xfEa+R3 z50~RlwYI*!mCjRkJ~l*C$EGb;xHImS{r2vT0+{{t-S)!&+uA(xgPc#VmG?_&7~$Lc ze!{*&u2;?R=Yvqe?~?a&#QDj#qzmH ze&5|F!WcixaXw03XAO|{th!j-A07~fH_7=ZLgur#c71(nkjRgIKc}7a(_xJ8V_sh* z%l^|>_8)WHev>A8R<6PFy1>lO578n&``OF;v{AR`+ZA&C{6LBDZ{GKNv#0RW`Gnx+ zJXlTh0`vd#OM+i5{VbMo>stx`=6UyoT({SMCG5?0sa=k5-^=?y<~SVJTIBP$Qeodl z)*HKB!0Pj!34&M3e%?inV@FqtxaK$(M;T-J)bAI)MGHK}zm@kDX}I89w;;iTzZ3qK zN&cAZ4|Q@qL*0OH3uU=RY2}KbY_NQen=1U6=i_VgJX&$P;ATCi$nxGiM&#cdpRbnZ zd$>H`se1YLygaY<`{#XS|4bSq#$kEe$mk*0+nGm1T(h5?vlxf}7%B39E=c5uPIJB; z=`HH#ij%@mqWCZljS}`ja-DD1e=7aJ>bY3E&e|c{>#;k9f79OQ65%IO-heUJ`Q2$? zyZ03HNToK9WXbU}q(L|)w|skoPFPlN&z}{1 zz2qBd0%iPZ?fzabId1sMUqV8|0^jD)dBf`Mm|Vx2C zJb&x7>*C8~|KB>xvmRWs-G9Dc@M7uzpgbR+mgl2cpOa}n>u=d|9yjN~`SN^Emh1dD znV%q9FtBzUEU!msxZ~R+G|@2rg}Sa9KC_-Lk?Z_L zav@`mGc#npT_fvll=O3+&O5elsnMR#NtEZ2^C=Nm-aa#y$^P)3ww{?R$Nzwzg#9hj z|4eE)md}^vb&Z+N%jEgqqgvRT>)4siMSV_^9mE{xAC~P{p|#@y>3`Fw!jI{H85NMV z%X@jEztQbJzV+-W^3$W<(|)QP$8xp%D(})!%+FP$h5c#iXRSQ%D&=_>Cb=(VfR(pY zu8VG$ytypz8ExF_E61^1IgZh^%ePcomoh(gdEaTUt;-mkWPZ%{eJx7lrf z_U;z*%NW^Dy8a;inB#_1j(fRs+@tOxw#3AFSvlE>`30`Lg2Y54as2eji7Bahsq-@O z3sUo@Pad6_lbt%V zrMe1I6AMza^KR&jl`KYTXJq;BJE(uIU}d@xEW#x z5xyt4(A;6TQc}#krRC;jWEZ4~fFhsdF2Zf$;W92H(V52FhNFSov)Iq}&CSbkr6jwk za#g6rnW00y?`j&unZa@&8fj&(g=FUWg;{2ySP9(H!@Dyr!$?-;3`Z-sPcmX2vs^{! z(Jh;JWHo}jo4U%#_Hd=7cUQ2xiL3m=x#-F*JzTk43Ju)O#D2EVqNTj=Y8u0ADJy#| zBr{LVOwCHoE?AtApPHCenCUJ}=uq#wnnvZF!^&6-Y2;IbyYjS9p-reqlr-MYLv3EH z-0qo?G)T!<1m!TDyD4{`W}bnb&e1Bj5RJqDlE@5199HhoCe&-ry;ynX91O&dx^roO zV~W|WLTycsPNX5oET3s+vuC*q+)eC}Ph~Syv(epP2c7FGNKU6A#p4jlcbNU6R$haL z4gCkl=^0`Jlh=%R4G|h)VXM29HJY2V*h^!(sYejAn>INXTLZc&b0chdW6RF#rrbI9 zKq{+w4xp81H_gnM=k{uga7&N8%9#WAx~qV8=B4JRyK>d6Nn@(rm7kw6&m1|uI2)=N zc=$_QSm=Vm#Woayhcnd`BRx&r?u3Qdi#^6&m&2@fHjN~QOuO|FT3~hnEl=_jGcy)I z{e;?d=n+c{G9gWE+}_D#X=+|h6UU)Vs29&RFGe1&^VKv)osDK>C7U|4h2vfiFUgcW zRn4{txAf)(_j-7tew3Ay0c#Q4Fi+&gn_ai?@K})N%FfTtp*c+DE;QT?ZDS;F>KM&J z|Cw3X1XL3kQ1wbIfI{y~8_$;jq}J`5VC(hM49Xl`aQwJ4XCw>kN2-8v;Q zF)`hhndXX%nKscDB1B@IYcbn5QSMA$gnMEZv3)~@X($~yHI6PHSkSDTMXBSbO`IPy zbxK@;Eix~afyBJjG`g(FPEH*^JuxP{U`cK&9TCP3QBFpOQgj(CF4S|Et;w#5W^Brr zO=g2-S!671vU9wBjH|%KB8>^t_aI|z-i{98FeTH;i|la#W!Y(q$jQph zNKBEz5(}ER9c}w3R=o1PdzNh+Nt>ko>5Jz!kPf22dypP%Mrcvm$v5 z<}O({E7?CN6t`355l69R@aEkdRKl*$$oA|kj3_63Hy5T!^UUR z?9I9!t;-2(R&*%p(sSao4IM3R zAi5V+F&PHU^oi4HmiJs{P0oswFVni7-Y zoS4B@ChADJ!^@!&^3ZfV);TfLeQdHQi)rA+hZ#OyW!lT(R+$eC@yvYUT)Nr1fC^4! zXp#w;Nnvw=oY1u>-Qi7%$!wYtxvS~H_HlF*mmNk;sw$4@rlz(KW}8|>i|2xvX_;|0 zsA_gASnTm~Tm(C=Ag?g>0tdrx=D}l070E{Ii~`IdJM}X7TF$dmggfc3bBuH1q73Jv zjAUaFCE+4G=q`x*|`@yOzdxZ;4tP-)SlN+d{@)MY+?L3R$qB^SDvz) zEOMMVO|+cHZZF5%Jq$z!$cgAVP4{{65Fv`h8wBz|8K8RORiznt^C7!1-9&Ze<++xa zCm@}JP5ZEE%y&Xs&No;fp|nNGSPL$=wjEKeV% z9s6#EzN#rm&!Zb&b6qKkyh0MQQx_-Fw=VMv(&=FOri9O9p50!=OU+)Cnwdk_ZqqX8 zhp4E}vpifqyGZenJ|ybcM!-bI7MG&I`ce58JBl0VN9A4YC~n|+RK)l^t9Iobf$#P_ByxOiUe4T9>MBiI zUhJ_l_v8^yD=&P9g|R0d7DwHMk3JXa%iPBESX&71Ocy)aZKNO0`_;t`cN^)4^DcI= z!`()nhYw@tm^|3aDao5onRp&b6G3*0Xmnb3H94E<43CRQ^wPHGVgDy@o(I_?O=ol` zGdI(fOtYAmtXv=y=5hZ=zn+H;W0OC>v11PNv|1Iw(@CDAFF4q7Q9+tkv)YUDXcsX9V#GA8s3g&7k~Cd*44m)Hf;=zqgz+3_+CBm ze##G)W}e~glJfX0T6m(D0t}V$O=C|V!=f`Y=Ut=%cn*}E=kc}>wc}#d;OR!I2s^nc z$Z=(|YY&TdFpuYEp1EM9;P^sSLDj?lA1W>q)e#n+7PE|qN_Bd!2b_z%dlKJ%sNNqr zEj6W(o%iIXNr`wng-*?-1;4qDB1Jbf#J#*6x&bTC2;oIp@w0{%H}9W#H>*q-dpfa& z3)k#ha$}waRv`OgE169h*>s!UNQ}>3Om~VBr=`;KVRY$|IIpNE zF*h|YKZiZ}SFj{;QHWV<5aB|%A9=;NZNq6y%f847#=9-EQw{4rJ~F0=I{!cNAS}K3 zh~SSrV!seq`PqxJyjVwyI~1Znh_5`LCx}~oUJi?h%cTl4FI_|rV8n~5O@q5%(_>ylih;y9yy=c{BH;IZ}9oZ>+Z|8a2||3o>!1T*U40uJZ@Mb zZ#j0?k=?f!6OQLe9U4wWSd>a_DsJWHB(n3E{mdFf9VwcgF9=zdRV0>oPlx|FFYbc4 z`w#n8-?IOh-@lr@Ek;$Gaxjz)f+-nU<|#@2l2js%)=h2s9K@el$w%~nKQdPW*M-T7@Z zw|zdtP0hpN3g=QMgL9MaZF(`A?4tFCIFH>vvt0b1q-UPhpk_9$`?UKf&Eq~x-RHRP&LJw z`^5Cfdkd6~Kd#_;NeYH=O)3mM&PicN<6v?oKM`cZu;(2zh^fp=yO^S;cB#ZPJ6p&! zSN6PA?ts?#|BuLEx4){5zNbKWB7g$MGG~#dzD8^Il4;8;LS%fE3e9_x#o1u?Px!{m!3?f>|KGm z@~bYHlW-^p9cUJU_kYTYEmU4>(ASXs8!0at;z7|&)i86wZrih!LLohal(ERXZOroM zhGxy+(i}4u<9-ssBZKn3J|APm$igRg_f2EbwOHxdxTEqVkF$lcyz%?)ne_Fj{IY`G z_E50tA?%cu&+6xk1*Dg7ra_a#`B$a+IfZ$2NvdwHo7o9rmn(9o`-)WV2x8=-ukc&(vH5yj9=-#xGg%JQKubT47d zi>gOCPQh_%mw6yx7)6S&;n-r0w;fw7(F;QC3kOw~+z`Ih6E^NI_4t15KHR{g;|Lej zlq^t&6JpJS1q!zk6WD^(CHb<0aBpOmn9ZPPuEZ)MyC9u*(O}}~H&Jly2>y-WB3EW1 z6_PJf<+p4qjJeOFo>=X&yqohHx2EP$&e1#Zj!~+?T?lljEr= zu9cV@iU^jT!9!@VUr6geP#%U6 zW7)H+a(Th5(wO!zVJk=L?3m+)$0582%wx~v$sS-v2J=*3I|*`)`tJzFr$S$MJBl7)wV`5mG60Q8+-o~L;@|LRfUz3M^N!qtJ}+B`5mkBVv86}a}V?tC^egvU=MT5CMVt(z1$vQScu&EnJHo|l%1TM$tO00eN&xC-(MxC z=F@7LbsjmM(d`0eMcptyJA)p-9iIZzIlHXm?%Ztd5*qs05o|%p76!D;Szx-Li7_WH zu7KWpoM)&vIn1Zn*{e>pFV&SLuI~9kqs&(s`Ad>wcifWfWO}wBn?2p*K4=V0gn2nj z3~HS`ewUNhmM-)4749oKGgD0a2K#obX<^5j@0Oa`H(e&qoIYN)v04|gsy2shWx>Z_ zW+IP}lN1ookx3cyK}9Kfs=58Rm%J*z8{u<@Kw*^zR6c7c3go_;aJyAkZ60pj6y3V1 zhg)?StGp_LPTdfEx}k?lRs!+(B-Ew2J0+ht$t}!FO^$w2NZyr z4)P_x?25@>}N$3Pbrz_TSJ zGoNe+?;b57%(@6mjP&W9fp@xh2)pRf+e-UTnrPiNBz-3v^k zH*@mnf+C0AvkMKy7r5Cg;tS{jy3|Z(T3lireeWjYOk!V2vH{ZsIC?T0kr5mX>0M0z za=02>)eGP}LmqMHhDb;v{ZCJ}uq@%r*q&Ju8KM{B#dJeRgjzbT_s3sQIvPdGV#ak=^t#p6Oxq)r05z)SiZB;Y>Sx zz1dyQ^iry5O-8-YvqyEZ)m?=TV*wq8o)rV3+>i zjtw=-$ljrq)yGaBbvCHFkUQ~RTeqFN&O-4mTDO6!({S-_>4mB(Oup=jrLwa3=q{o5 zrmvKG$-*!%uZK1@3$wZ3C-i(}Qq_&@+h7PUFlruI2Su0g5~FU(`*MW#0;GkN7oj}*Pz2X^|kxcY)(CG_k3n+S~Ckky%X!?)c$N^AE>cjo4HG zvxOo;n|SyKGo(D8PnD7M-m&N4@;^9dA z3r)tVoxKev7nr13-dS~oHyzO~Hh0Tnv$~|qGp$|zPm{Xv=y~=vY4U$C?VE-2(&PTY zYCs*`q{rioZoNoQ$2XZ<@c0X?9k>J5C=v8>zE-cW`nb@VLb%`!+hpnty~tw2%ztE) zp7=kmK}05abFv=Rq&hCRKrwUS?XU~1Thw7q`}4(?GxAWrrkN&xA=}u(>|}AN>WS5F zSQD4G%G~5J^lk)OSWj}1gY=2~n%2n7n9EpR!QlLy!M0GQ`SV^yMGS<8Fu?A&F~#@N zZ)!^(IFQLmZmfQqmz{kHNuM0HjR}aGGsn}kfFk_Q7;~wDfi~h(Wimipw z()`h&U)M^T=~ZF=gV@J@bmvxhk-x-{jx&GqJW?q7)rZo*IEqDa=R(-{LSES3=1nZj zk~TN!SF`d>9y^ebXp8#fYTn-87w*U0_scdp;DZX*p2t&9}O z3a`kZVWWkSqwIZ*bM!l%O@BQ<|CN4qZF9~{f78HU=ZuWD^jD0Gn>_qA1AonofpQ-^ zzUc5Xls`XXm1G}2vw(){md1Li3;M68;nLUGOqy+h!^LQBj-i23;`gcsYy6b_dxIew zk6$lFS-ZyTMhZVkvpoHqx=7;<&j>%o8n2MLRO98(3VX+J;h#m5zlxlf%L#nA$O`8P zz`y52{F4M+{)qv;BOUlE@RJK%{wWP^Uj)3&JpmcTz}thL65xvy*|11|?I|8xlVQvrN6*jEDY2>Gl6F8_oC_frjg3fR{G-vzuDxcNKanWH-3>%hJq z_;}zAz<&YW2>f{{m!JH*R?Ku9*!u&QzjB|KD*(9s6~~+h0+)YMit}LL>%o5r@O_XU zJMfo)M**J!ejLDm<`Km|PT=xSneZJ6$iaU)@ISy$F7WOUw+OgA<@iy>z4qX1qdhRC|_*%$M5pemd^|^g9@F8Gd z0$lz{O>SQbd>Z7Z47mK08Qi`cco_Jv0DhcXihnABKLY+MN@y;%mlxOC~qCaMIHrlkyk)m~1#7~JN- zBg$=$JPG0=uYtJ8{U9!Ke~2spG&!$lKZuJw9pWOdg}BK5AujR&h%5gz6_4u=agpaj zT;z2S7kL20MIH!oW1&9>KwRWS5Epqp#6=zmaghf@T=}PtczFXMF7jfCi@X8iA`gbR z$U`9R*HGSIh>N@g;v#Q^xX42wE^<4>m4Awemp26BA}@uw$c-3x|3_|zxX7a*?vqen zJH$o43F0F6gSg0}ATDwT#BBy~qaZHwGKh=ZAL1f+KwRWbi2EzAVeyXx;vz4HxX1$_ zE^;TtMVN@e;vx@(xX2SAF7hOZ+YQFa1c-~g65=8chPcR+ATIKBh%5gD z53lGXh>N@m;vx@$xX9BXF7jN6TLkr-4snrJLtNx`h>JWI;vz4CxU(T{F2qG%196c@ zL0sfT5Epqd#Fc+)iq~@y#6?~UagjS9F7jfCi@XHlu7UCvLtNx_5Er=<;vz4BxX4Q( z?sE`#6Y!nT&r2ac$QvL($de#H$TvZLke5MR`KKy*{cnP}$QvOp@^pxcybR(ZFNe6V zLp_v1T;#@R_q>2S7vdr>hq%ZqAnsETw;bXk_k*~|iy$uY3W$rm65=|ceJda?a({@6 zycpsluY|bBt03+!h+7G9kq1CrN@o;v%nwxX9}uuKZKB{QRwjxXA4g z7kN3vMP3JSk=H}q?a;n;5EpqA#6?~KagoN@t z;-*1)8z3%nC&Wcw1#yu#LR{nqT!+>`c^e@v@&t&Byc*&nH{{>lWUi6>LELT-*NAqH z!^o2$F7g_Pi`)<5BKL>50T9;@;v!FnxX5cEE^>c}i#!10-U)I2AujSN@q;vx@# zxX1$`?zPab0w6B(B8ZE;9^xVogt*9qA#MT04TQMJiyW#6|9axX7Il_jRae2gF5Q4snqOKwRWbh>JV{;tqtkPKb-V z0^%YMgt*8PATIJGh&u}EIRWA#uY|bBgCQ>RB#4VV9pd(fxVgaFLp>A$?*P0Q_$2UC z0=yOQO~5mOmjUkyyd3y$h+6{lS68r4f_lpY`zok6Lw*{-5Ar1NgM1VCDTMZF1V6~r!4L8>@bd%Y$AJ5p$aBFD@^bJq7yS5vALK>g z2YChfsRBR#;0Jjz_(5I?e)>TD1b`ppCEy2n75K43egeS{@>1}Fyc+!c&U=pdrv~^! z$Y(9^V-U9v_yFMbz&nDUM&Qqay#e}99s>Li;CA5jIGEa^ zfG-B_0R9+oC-4ox6M!c|+;re0!MF9$zQKz{te9(j=kuK+(!fu8`d zM_z2fE5Xn0P_96*M_yvVtH94u@DmL7$V)AFHTW3><*Egq1H2CSgAlhK_$=Td5Vsxh zDBwH5e<_?t$jdFb1N_**j}v$U#4Q7UKiHQ8-wV6~_%h&?z`q6mRlq+4UJcv@yasqD z@Lvo3QQ&pJPr??r30=yFVC~9~1+&bHW!9Q}l1y8czxfZ;_ zf>&Aa8Vg=$!KCH-Vor;8y{!1pXZGYT$># ze=YFE!0Uk*gCGAXVjaca_!lB2n24vH^yo-82F!}iuu1Gz$3ts9e6$1M*$xU z`EdaM3+$c1bHF|U_blMehLu+Ifv1@=Y2F9Z8x;Qs+$0=yUGzZCclV803Y zK(H?Z-Vf}{fzO3}Rsg>Q>??uK0bT|C9K@{#{sq|A0Dm0f)&jp2?CXGs1Fr`@5BxL$ zzZ>iufnNdk##DEI{t5E$2Rt48_yd0pcmVJa@E-{Ldf>spCxf35;MKtG!25xpDByX( z9l*1}j}!Q9P~HUKrNEPb4+THzz?T5e1^zJPrwDik@M7TCfS(fJQ^39yxDVKG0{%Vl zGT`~(ryTequ&)4q5bB{4_yF)z1>6UCHSmwXPYv)-fY$;qfc(?}-weDScyEZ?0DLU) zM&RE--1J%Q>*5!H2RH?1?N-&#%wUIBh`fwQ_Ww<6%?yMkv?$(S}h1JQm7T3j9vsn}D}A zgVJXiaC)&(ZRNn*C~6oL!0E+FwN(Odr>J360sj-~p&B^7psBVR;O5$p?W_gfL0Qv# zZNNjoe?9Q^P!A2jJA$7^;Fp8F5$o=^w}O8^;9Qc0z3qG zSKxNwmjRCgejf7a0R94SC-5(UCjbus|4G2RgMB*iJAmf`za0D&0S|+ED+YcA*p~pm z5_l=_ZooGI?+&~S_;rxaa^O9{z5@7Fz$<|V0A!0!Pb3_J+xCj|Ib;CA5n zo@o^DM#zr?xDDbufp-R;0DJ)WNdkT~@O0pQkk4G;e}jDy@Ik16~e19_pb2cqrIc0uKXT1)Tp97P+q)cm&wj0MCW;)&h?N`#Ru5f!71? z0r_tLehv6<1a1dEMx49<4+HK8d_VZ{2Yx@q4FG-(_z47F5B9;pi@<*f@ZsRc4tyZ^ zi2}X{xC8j@;NJ=STJWC$d<5_$;MW092YwgC%>~{CcoFcC;HMaP6z~$@(ZEZAFMx7w z0zMk-%Yb)){FDRt16~1q4EU)89s|4z_*mf8!0!eBHNeM#eJyajuU-dy2$Z)TxC8t& z0RIc(HUb|H_QrH~|DOQd5BNmj{=iege*p0FU>^v467XQ)lYxf-zZc4D2QGj020tna zxIfrCfPVt{bON6WaT9<$fhPfP1^&~4$AWz>@M*w{fX4wZ2HqO{mjIs*_NBl#LAf>o zj|clQ;4^@i1HT@41@HvmmB4QRUIqLBj{vU)J`?=c0iOlD9{7#G8-SNW zeKrD*gt$h$yZ@g6KYqYx1NR4hGx!exJ_qarfhPhF2L2A@Cj@vB*xP}-fJXtJ3)}%b z8MqU83h)HrslbzfzYFzIJR1BI178C6CBPRzK1+e)_xPKDzY6wcz?Xvma^N=s zuK?Zx@>vP|8}MHRd?xT};5UQ+8sN)-*8;xj3V1N^+kuAwzXP}(_-f!$z`H{IIDp>?_D^}sg)kArq81AZ^qmjk~Kcm?nc zz$<~Tg8Wwj4+LHfd?Wa&0e(O5THu?2*8zV3cs=k3fj0nu2zVp#hk+Xj?*6|SxF7IG zfcpb~6nFseGT?#0w*U_Y{uuBO;Ew~h1K$cf3iv}%{|?~8fIES=1D*iffc}#Nd>h0~ z2c8Iia)ECL`y${^051kU9paV%zY%yT@N)383HW!w%YZ)#yc{?_H(LRG2gI!e{yFd} z;5)#7HSni^*8u+y@LJ$If!6_l8hAbMmQeo0}liq1^hXP>j3^da3}D~pdJ!{mx2Ez;Jd+3I`9{P z=K`+;UIhFl;KjgS23`XEN8qKvUje=e_#WV8!1n?#2fh)?TLJu4u&)G;&r4SU-v{>9 zz^j1Q0N)S17WmJQpE}^LfqgyjjnIw_z+VUZM&Jj48#CSg|2N=%z|R8r2mUnVGXVG- zU>^v)0eCR*H-U!$uLf=h{ub~k;I}|MIDo$m_Drg1OL{*zcui04g6aJ|8oue+P=phj+H<8Io7mzX=HQ5v8t-T_k4|G z@AD0Pw7<$=_Wb+)(&;h6Z@5J#`P|x zn{&NIX?E-D^oey!x8!=c((I6bSti7m2S;-w9=PyZBx1p*Fj3R<+_8? z?YQ<)x;@vY{x-|&&-F2-J8*qi>5g3QSGp6|yOi$C^%kYOaJ^1x_Q3h+6U&vpjO&F; z2XO6D`f{#gmA-=OXr-^@+NN|ju7i~B&UFW+dvNWe^i^D+I%k$Ykn3Yg2XTE^>7HEg zSGpJ1yOi$D^%kZ3aJ^3HzFaR?x*yjIl@8|GrF4I;W0fAjb+pn0xwa{NHP=B(58}Fm z(u2A7QF;j1r~Wd_AHwx9rEOdvRyvgH{Yr;%y-VqEuD2*1!Sy<&*@L&#{z?z!dZE&G zu3btG<2qL9Yq*Y9dN|iMrLW~WNa+z=cToB|u6>jq$@Qr}&GJWaeN5?St`93citGJK zkLG%p(qp*ZqI3+`>y#eL^>U@halKG!2iGp8$8#O4^aQS>m7d78P3cKo2Pr+7>kdj! z;o3*(sa&7>!z{m(>tjmCa(!6oXAr{i_-C2uTz>mAV}@6^z~dXR62oc zm(n+I9jo+AuA`Nn#kEc88@UcrdN$V`l%B)2kJ5=;pZc#^{v@uCDedC=u+nq6-mi2r z*SnNX;d+bGsa&s9I*se)O3&kZq0(#uI(@>WbOzV4O3&vyTImH`+mz1aI!Ng(t~)55 z&9#rxIb5G=G|QjM^)aOva(!6oJg)aEozL|yr3<*;qI4nG>y%!^^>U>bbG=aMBCcIZ zFX1{?>7`spD}58!Hl=UoI!NhdTz63V7Os7izLo1!XU*~#bA3$d3!{H0tUQ+gfOhm~H>^?s%A<$9OW_i?>N=?z@3Q+gxU%ay*L>xD{h;@YM3 z16;={{UF!TNu9B);@YP4f4B}(dMDQ%lzy6P zAElq+`qUY-{1sduQ+gNIhn0So>-|bU$Mr6ypXYju(!04{r}PV4FIW0St`{m@$+b)A zm$;5q`em-8m41b5o6>u@4pMqA*Bz98m1`fR_i=sdSF`+8Tpv?O25wa zE~O7}y+!GRT(4954X&3f{U+B7m9FO6rSw}|$143c*U?J9!?jK6cexHy`ViM0lzxwE zAEn>t`qXK&{54!3Q~EI1hn4<->-|c9$n`FzKjM0e(nq*nr}W2MFIV~#t`{m@%e71C zPq~g&`ZKPhmHwP-o6=u!9i;S8t~)6GCD%Smf5r8wQ)c<=xIU)zF|H3Q{WaJ7mHvk7 zT}pq;^%kX%bG=UK@3>yB^!HpZRJxvPm(nM=j#c^xuA`Oyk!zdMKXDzT^hvHeDE%|n zK1%<>^{EE4{0&?mQ~DIwhm}6f^?s#)<$9OWXSm*?^lx0RQ~GzVmn(gi>xD`;a_v(3 zzg)*E{Rh|4O8?2VP3gb54pRCY*BzApn`RLf8LFvlqo^{iihh1-YkB#461m6gYh% zT50x+Nlu@zDb0S7)aesJO0x%uPoLl;{7SPI+Nk_WcjS7% z(w(^8r8IltmCCO)dqI!NuXIV=>N?*=(tkPF-9j)}0T-%iH#&wX= z?1geFztZeQ&eJD+l)j4VQzy;xvj^X({7MILeOPJsz#QHGQJVdth|?!_Db0Q{1(jcE z_F@v1U+KPFFIT!B*9(;n=Gvunf39Pd9>8_9((FY|YJa7#<~m4e_CO$&U+KYI`zSqx z>r+3ODyjTRvtL|5n%!0aJ^1x_F@!W|0&IWfeW?2 z(sr(0N)O{YR%!MNS*ZUj%^qx|@+*BU*Fj2;;JSm-*KzHm^hmBx{b-gyitA%avll+7 z{7R4FdcV@6x!$Gp7_PS{9mDlHrN?r;T~ReA!~(MnI`+NSg* zu7i}G%ykE)r*Q3~^i-}-{a}{g$@MX%W4S)8^fa#bE6pB+r}kHxJzz}zU+H+R*C{=N z>*Y#c&-Fs36S#IM&3>U2jeknB7eJ`}m7c}5O=r&*r*=(sQ`>Q96<9Qzy*w zvj^~~|10g{`moY-x!$jIGS|D5PT_is(y3gpQ#y_7t^uzv>p+@#p$_*5Ci@emy@I z^QXtxweIg?^{Lwr(#re)THhBd&u?0Np49YlO&`_t5ltV`^g&JU)%0#n@6_}*O>frp z22HQk^eRm+({!PxvoxKm>Dii&*Yspf$7p(lrXw^xNYi~Z-CfgNG~HIyEi`?ukLUUG zo2E}{`naZ#YWj$#4{7?KruS-kx2AV$dYh&WKG9tdW5DUG(AYueKg%&(_J*(R?{ss zeXf_*{+d3i>EoI{s_7$|KBVb`n%=AF-J0I1>1~?ctmzG!UaRRqLQQ9BI#tuN zH65?%$(oMQ^axEyXnK&Q`)Inmrn_jmt)^RO`dm+~{WX13)5kS^RMST^eMr*>HN98U zyEVO2)7vz?S<@Rdy;jq!G`&pIg__RNbgHIjYdT)jlQkWq=@FWa(DWcp_tA8BO?T0B zTTQpn^tm9d{WX13)5kS^RMST^eMr*>HN98UyEVO2)7vz?S<@Rdy;jq!G`&pIg__RN zbgHIjYdT)jlQkWq=@FWa(DWcp_tA8BO?T0BTTQpn^tnK-{WX13)5kS^RMST^eMr*> zHN98UyEVO2)7vz?S<@Rdy;jq!G`&pIg__RNbgHIjYdT)jlQkWq=@FWa(DWcp_tA8B zO?T0BTTQpn^tr3F_Sf`DO&{0vQB5Dw^dU_j)bw6W@7DBAO>fimW=(I<^jb}?()2P- z7iv07)2W)Ct?778Pu6sdrblQxLeqmZ-AB{iHQhziZ8hCO)8~39v|(rRimwF4T0Erc*UNThsBHo~-E@O^?uYgr)~+ zx{s#2Yr2c3+iJRnrq6ZP+F#QrHGN#uM>Ty!(}y&DP}6%gy<5{eHN8#Kn>D>b(`z-o zO4G|UU8w0SO{Z#lwx;7XJz3K+njWF)2u%;tbRSK3*K`+6x7BnDO`q$gwZEoMYWldQ zk81jerVnZQpr-d~dbg%`YI>WdH*0!>rq^nEm8O?zx=_*8ZA4X==yX5&xy19cx>hW*@^E3%byw_kI1vqxY@O z*R{W@n7(Z{UYDO_{}#88c#VDZ|J$+u?`DqG4UYZwBON|(I1c|^(3u>xkPiIL*QN3A zLCoXX^ZCUicGIy&;lOyu$`R|^(T|SR#|zpx){K}#y#9vo&!4YPp@-eyXt9>G&ultY z#b?K#SWG+F5Ak#y1#5atd41PtX$S;6b>Q}Us5I{252`2af$;`#OAHTNBf;+ILON8J5@1kk^_u#+vws_RqYwI;LS|))}84 zUtx#8g{}PmvG*`}DP?JUJ)axmRx@6VmNhh0%%TngH!@A=2x^BYO`GY%3pX9EXHhPjrTJ zf!9kp)gJ;cX4RTBj2)0;Q^IMyt8c9;Ug~^TNTIi#0Wmg{f zH9|F$6c~KObt`j)@4BjB7kq3E1-4L2(6k6EvDEM_$PP61!DXA=YvrwH#jV}Lc?481 z^!6al`*}SD~_zK`&tO8AH;RMW6K;@DSSC8Jo)pDklv41Q-ih z5cd)!hR2YGh}${xHZ}AA)9H4(+U*3V+ed{#%kFBdN8#ISuW&C7845JO9W?mL$STl` zM{khs-~;UVpv#~t6xa|BY>1lZWC3ge*zyI>XBW|3V&n2+4`TV{%y5WHhnGxC?DLIrC&a1PPJ*_UBB{(!PE1ZkVQ3HZR z&UbEU5E(H$@}Da33kT~)A-Y5cRRA$ZZr_1=f#4V*XzoM|6zN8rnSpSgm6P^z*ca+l z5vWVsdNiD+1S4Bsj2#Q4hwp$GhN48|Os(lOFbdLYpkHqP$szpz|_}@@T@|3hR3WEJ5xC*aL>a zn_Yu{$1;r$45OyOd(e()U_xd!kR2l&$(>Lt&8)`4tSVtv%e-c_Oe$ohVh$^Tx-wfn1vYJT|6SFN0_?T3?J-6QIqxP~KWrUfTI7hZ`EGE$>O6fqa&{hA4FeBq{+7WrRWOur9lWNzdZQT{BT@zSOXls^R-iv&^ z8eL%#99`FQbX`w|(fFp|2uHZab7Ade`x_w7<+j zflNe0<3!<084cS$rJ1wV64Xl_|Aa0SiS#ltzUH`z`B0hh@X0!!vhU`Kv>XCu80*r#PRf@i{!)BZ9B zr9kK3SHOhxmcXCA2F^I<_Yx?tC|}YVJotG^Laf6CQVoD@sG+U1(39o#XDg*$5*g@p zG}wGkBOy}N1UPKg1Zcy)-EwH4vit;ydw5_;*+7B@BM=xW?Z6P}mK6@Hii}MQ<*m}x zuYt0@G}7A2w8kf+g`d+`|ON`gR*{1oNq1JEtOPj*i7K3q#?WC_ztF37esNC|c-1VskkSm)_)g3}*8 zNe~sx`f^1e>urI+=rJ48+y_RGv2lG`V^uh?B*yp}O>l|1RyYsj1>p&5tS8{6w@w3( znRm~TuF>Ty$sROwo|$G;?=);vw7NO5qQoL|2A3(URbdcVbYQ7;DgoR~f#JtsYo1^@ zr!o%1>uiSWj!|Vee4>}(pI~^=A3Zt|W%ve_;Tu$j^OvZTl%e$&^q`n zJ-k2raU1>CZtsC6>9=+e^ofXD)`R%v@>`=fOQ$lPJ=Wh_)4>JETc1Gp!Om1`DD*;j ztb0DeC(;Y+esX)1`zGy!N+rbqG9Ppj_Ue)qTnF@%_d#dX$MZo$aM$aDKH8F!NgM=O z2o>%aAM_^lPW1QR^g%}S?7X%@2p1xO{1QqY(*<4h zPD=d#7sMm@(=je+lGERR(*>=jJMO{vpOq}b90Re<;4R~#2q{l4ac_W8*u z(uc9x-dceGDO}Kgn7s)X)Kd(Qf(vSHo(p;kcfBs? zaZChKh{}saW`MM^V_eW!r@#NE3wj8S)$4+mH?Zy~F6fdLqrU!xhe<4ozrqC-gQYQD z&|D;=De$|mzybX37#Gw4+=M;+H(k&$ICqZ=dJ3G;T+qn2*AA<{6M~8idLFWYC5hT$ z3Aia_R7;tYxpo)Xg4_e&<-waVN)^=AE?nM;JVqT@PA)dXnR?9X zLMWAHRw>pFuLkNdYKP~2V_R7f(00tB8@MXar+f=&r&v2YmWZ~~2wuJV`fDSzD)3OC zbt7d5ti0Pm>Ho~yVgD%pYVGi0!v8f0P2fKe`2RO+ho|A0OR|a^Ahme4!{S$1DPz`;_bqrv)Vjyawl>Gh7jn`1)R~_d*0FD1U(dRE;Fl(LYa}89@Veqj~G$FT}|Kl*Y9nt5LuvyWP`C4y1-btH=y^MjS;`R~9$4Q%tg{hm;rmu(sCzOaK z*ZZ4{v>$^f*sW0rc0zF3$XtK1CXrW0s&G}K%aS?k5 zeTnXPH03XFQLHJlOQ|V$63LpTjNNKv)rUA%!fJB-=Qy_1TsRc>_*2b3Wmk;v(Ucn? zMO{-UR44K&pOBSEO?ij#h#L&MQ|M%y>+N3J{<{Fyg5epKk^GN_fc(!vgV2=mY>xhe z&5fU;##Pjqeaazd?A4U^_>}1wfHdV15Rk~HET|^>O+MuSkz}|-&&6<52A?vMNY*sv zkWY=QdKw3mSZmO?puRU-s>!G9B6Qf9bV;A`DLlPLQ!auOy_(Xt?MTx<5qL)it%6cJ z$w3b=Q=lAtl~u;}Icicj3|xqj)a~tb+Y20Ux~&C-==K27trHs$Ao`^q??zmTtH)!7 zT_bjQ^!Pv2vCHFyJ~j$xADkv*^@4XX4li+lESW_nvCmoa5Ygg0QrE&)scYf<#k3Z_ zDy?xfkqd&*MF_c7po+FKpMZ-b0mzMSqJVW+b%-`eR8)21MLycoP!(%WLseSiiW+PV zK~YjI>8z(6CElGDw1syq?u}ef&tEclND@tXvI+#yRO6zjHBU) z8fR;Fc-;DB7^>k6WScBKLuTjZtoG@%I<2c zd!D61#WMnbC5Agzf2=@Xv-;POMqy)LI@DM*^K%5GX68FfR9pp>yK$17^&o^L%si9v z@i)9@)QHQd@f>Q*X8zGy(g-0N_P-4wl9gb2I1j_G=}h};?KV(bx|R_hnGx#LQRX*| zFxJfPAT#pIafY7tyyI_s3FHOmBZHRVE}7pK=q{f5y|#jAX7htCfL5{H>uI(EkNGWw zRl3Y?HI1tU*3wNzW_KohGaY<`oS1T8RR_NDX~M^2euFXaLR{B2Fg^)&Ny+?Js2B~i zM>0E7LGewur6=DN1Vjc6fy;2Td=gkG6y=#UgrsAO8PR?@4BG*l`{pY{oRRt}aGk^Y zxWmM_PQdP|+Ste7vlxB27#+QPOxK4-F--TpcWj~>aX6zx)QwKJTX+t`M8Ol(SX)RE z(sb)-$hfAv7m_rS>E10MHPam@QE{2>W*j1EeSvTxO!rL2$KT7`cm*}Crp9c#hYI9{ zG_$~Clj)A^{43k;6NH6kyBBXX+CmG?&$QAV?2D=#eWK904R=Wb{lU9QpmAZxrubwwosKRYK>}^+tSi2;cFJmw$$GIk86{9Cw_?Ma${3$9Av6zzf}> z^E+hkn(dBErYbS^DU2X>ax^h5?riQ_3O?qlL>B?9RVAK%+erJE-k^N~VxmyX)jS*V z-)L^!gBl-Bjae-hvQ;P#m$bXmS%0X|jlDnsbR%x1^Xp}{JlvJede$7z-05{V14C8f z9U@tC@eAKFvg$w}XB`P~3+iu%jG+&PN@r~)-*Hc$&cOH{O<4~q>Y9>FrSlM?P-@CP z0)OFy_%Wm^={p-_5_zr2 z;s{>L=Y*6gXspfqSyO;#n=74kJQ7J(9vhRai$E8d%Sd zf6b_KNG!%|&VKx>FFC!P9IP)B79}_y8MHlHDC02jOe$k#thJfF9wG%`WrQeM84qG7 z;!(!!@PDj~vM;HOeqyhtjL-ht!C8FxBn7mqSlU|^3j z{(>AyQ^t|xquN$u?ldq+=+@i)NakG?CKzCER5w)M#**4rRjj5Y$JX`Vha z@;~H}->Y7BgspDd+eY~B?ciSyNnzCn&kX-}9{Ig}A7K^$0Uu!%F(3JT>OQuJ7~A$f z6%!~#F3+~T8(_xXZF{$o*VAlc%u5ct)Nu3OAHXil@wKNR!-~CaZ}I&?^$gqg{6GU{ zqNhUhkkV*o!qLZ@139k!0~$cO_BScni|g8VV&Ao)GmLY#3LU6sBAE393 z;XMo)@=T&t(~?7Kj68aJZ;-tPeG8}1n`dcShWsDaUyt|v0*=n37DbR-N@`I{olCV? zMPtWRiyt2(wiwl7t^n55V!(?=lGpA9lK)L!m(^krhbI3H=Ek>E<9|3VIs=V6RxQS2 z08)z^K|mt4xa1z9UsH=evvfRxbu8S9K`qW9S~a!ETxaCb;ypq3!w|7h3t^9%T1@9) z?h9YSyMydoAYwtbut!ZT{!HP?JB6u&)AOjs@TDoL#hs-8QY~(zd0SkySTA^B znjf4YfHk%7n@PTHH;{ZUs1j7CkWlsl`MPkVq|# z{TqowQ;QHQwZ~iZM6@ud#SuiSrWPMo8Obi$6=at~#DZ*LkD6M{ypwS8c#8+&^gL>D zFs2YGd5a3tf2kIigXQtnLgZscZ!t^&YijYu^G1^AW`X4E!8f56%{=Y#SDPD;qsA9d zV^)h7(70pOVkZnhYHR z6z$ln6X*ZP3eQ0TSW}C)R~t#5x(i4y1>b~PYy{3ZVgEzs#>1)cNlxQO&=`d?RJmx= z!tg*{`7lH^B8TSge3VhVgGr)?cHv->=LixdFH{j_A5^5>va?W=o*;-GZ>E>Uilfr( z6I7z7xBPM&Ju+UGUUnS)KQ|Zur@xXf|sp`dJ{~<|z_*o;V#pE?lCM;U< zV?B!I!bJ))<$j?*!3i(Kb#22!8kALYkrPq;;}e(iV(bOvmH2ZPVF+mvrx9nN>DW3~ zCyvxxDlMW;mO*d0n-Jn%WMY3=^i-Z*1+dt6C%4P3Tj@}#^7^cuKxF~sC8(@nGxIMp zH$IFS4|E#eiN-i^337&@mk$qAsr%%XtNY~mOLU)Hy>m#<@-%+F1GJG9EmmQc=$qhd z7$RA;<_3J^ChFY`>K#Ea2zIts!jYZL=_>4OB}7VgmIk-)-nM41eBAa|;xfwVG^|ju z``Bwx)h-tUqmmV@C#9bw{jNOL1qMzW86?Ne(9e;=Rp{bK^{R4&)C*A{hb1_CDhd=O zozI<$Hg|4l&7Drf7~BsqzC>tH^H!|-O9n`nTDO?BPwwU*fAFVDqx$TtC0v@Q4^g`oN~&72Z6%}KW*gjQ(eH} zDsZbNP*3RzP^h(cm2Sc@d^m*`hv{jjTp`rp5Zd|>ZD6YfrLe3`ie+I}Xg%)=t*@B2 zNt{AUt;jMi!eAtw#lQ}U%1|+wUP^}eco}BSLKZA!jRPrZAW_X71Si%=M}`4Q0+bqc zHR+n;{E6OyD~$B6+X3{}@!Wv=rL|(G?;mbqXR9BsdJE#1!v9b?q(z%Z^Xh8?^g3qZ zn8iz0+snt%XFl3?dz2wKq&5TBK?pdbUx!s}*H{ZLgwt)$uEy2`UqxVhTH`wMAri5H3A?aFtNQ>mF9ul{hdmb9t62w`NRTzPU=vmQhMhu~2md&sh z!LE%3+Y%R+$K4_+gM$}m^2Hs=a$4K_t*7Dq%KLxOGse3zBiJmBo!h)os3yQ?jUboA zhC5pnY|)tuo?_|fR?Te5fL-75{STqMCb!Y^?Y?~b7w%Ap2D>0GEPd?n1{8NPAIIU@ zlvwLv;8YASaYuk{AV;!wR4X|I(iuogIZapE7c>Ibm80b7lxVU2*G_LK#;W$UK5gp@ z;2}C~Ca-iS58U zdJ_V;F#B(B|K04#fgnu%ZuZJioWF(HlYN=Y9`?0fbNRyTZToVW{e6h~=!nh!cFF+S zYxXHd`*UKLJu(g2p|}OL?8|p*bF- zq~O7D8vi$$7e?uMN*M`*bRaz!#-N8rNj*O!MM{B8g+eVq)5dNVUV(&!4B0m{(3~0h z3D=@8S-^-R#RGS^yDc+9@p?+;msKQ_7A3h;kS=EBw#XDT&_X%r*4ZNP>K!zPFbCL{rXL4e&*3#IfDR~s z-Ilg>Prz2--47ID(bj2zF2 zPLsSNz;UN3GlI<(;IYkpr8}0uw+fH9)dX794g@EM23j&Bf5vqngMJAo?%v^Z6S1yE zXC0Y5hN8waA|oHLLe=s+Nm*QL3B8DHBdu8ncR0Vz=KQt_46*eha1UZ;d9CsE#fYFM zmk?^D5YQJ10leP^-qXuVw!Y3EthB9YRDUv#mu7aNlXCMgBgzZ4TZUtfzLRK;d3<9xe59&#bXK4Fcs1S= zWb6+!O!S8t=C+DyEoqJO-D+DWoE!OD43}j)N7%aCrJ5n_(h8iRk3$ojVI9mYn)8vO zpAFGhLe-XGXSb}fEUhSC+8Ue!aryW8(Fcc(Q*z`?8@Ql=fj!H0T%fLG3(zO1G!eCR zJ^gB_dU)3py-1=8@XJUI*0UN^vUKF!eZ%m;Bjtzai!_w;@;i|HfBAtCfs@DC=x+c) zp*uhZy0esZXCj&i-RV{i-NCzeNLzs->CV!qiemG2pa~^Fsm{~@Wj(FI&_I1=1iRj# zD4VDmoCmPWjQqm7(;zR2S0oV%tUFc(-nv7&V`;i$7aO+Nx`P2pcdX%orKCI5TI!Bv z(47Wjf6yKD2i@6HG0lSR2tuJFC`Ur6p(72hu-w~ZE{qEtQtnDt}CE=o7vP@&8>dn&Rs+wmH2P z?XtP|zKYtMsaQoeheFlk6jfw>`p0Xcg{m*`*-)rbss@~qY^UTwUt(&I$;0_TfD#T~ zW_nFEZRu*fef2c;C6i&%&l~ak;j994x$Wij;z75^LP3F1bzZ;;RB^yuRwoC|XARwp zj;O>!FDm6;@deN5Q!dP90I+X(prw4Gth^w@91aZVxFBGF^Z@V7^wdMBxPi+plr@OV zGWuRn9WA`<@==;)H!jkV6Xgv~Qw4IIQ!@t$IYPBwe9MTY?LcWbm4fJ1d~%w@DSAtJ zxk}9T@nNzKpaRR)(e4<-lsXi4( zn%F{VA{A$#0F($TQIpC}n4N4hf@Ux%mdCg-)_9BP0SV(aCzB>_Q>q(7iBXLi3^qzY zT4_2M(*YY3`o4$r9jC(Z!7)G*o2Wu#GZcfT)Urb-#E~naxd>i!Ynl*e+Y*VEFoiUr zuvDWnaD!jf3J>v8=F!z9(JUp(E^YuD-}yaMjxr}yTTfkUr?lH~IhX$PKY@FgI@iJ` zwONGi%KpOlMOC)(Y!ez`l$WMgS!X+)qe97b&Mg0#@jItlrd{tvJu`!BK%Xpjs2FC= zN+^abntOsTyHcGNwN_}VJvZqMTa6+I6^YY!PSDkMB=zf*mRvh4Wu*fXCTycbQrQc9 zn2Y@-_*iXa_43Wbs~@>S8Opk*+`_%Nv*@+WwKZg)K3wwo=FJW6giDxU&09YQ3)-!7 z>2;A?w!SDDw0=4FIE}P#hycKD zk?PNO-mwlnH_qU3eFHAvgSy2XHYXrJH^)&Y*4~)*R$dM8BdF}rjBB85E*fmRCKtYC zp06^SZ#KimL)CA7L>gzZ~%_n3X1xzRYD8cXJ|m zk#Zyt!-N0M>97+K`=^8gUqJ0g0Eu~*ceg(U|B(4~;ET+st124Ou;#uKeZw@F8lxNh z(wYwMqB&%4Yb)+HxwjBLhoXL4@v8l~{dnC7FK5E*kapnJSK{Sj!9g*GdiL_RvtZqJ zhXvsOZqS7IVz>vz7ZbUI+kwL<0}jDBs<3Yi9R3BDqp1S>c~a9YE;WUIvm0F}82z>V z{VDOi8I1oLs*JyI)MC39=^rjr?t^53RdPI>)IxkGJ8@px-3Lfen;3|Yw`=fV)4r9&GxCT3}$CTv$M=o2FD-nJ`AUL;6;2b**UV0l&gu z8g}mGYaoZxTv6WCBiN5$V~ov1w510|jThX1EXx-Wbyx4<%*e=>iirlBm_dGC9k{0HOJlak*kO{qHL~I?P}3`1W3C zfL)Wjd_sRl^uixee;akNUFf(RbfE8r)Hilt1v7IZ^Jk(L63|X1r0n8rc+bKc(tW3d z>L_bQqPFMz;dI|~^XM~LrcEUr@E>rX{}~7R035QZO=QsZA2Sa3;*Y>#^%NV2Z@?Ph z(2Z~)La)L|VP9*|Y7I8D2J2gcb*(|HnpCw0D_euhT7xK~FK!JkY7H)I4K8R6&TD-N zKEJi~DZ)GSB03m{4hXI5p<&2j=7r`L<5tCL`rlVJILu#h|4JedC>|tG{1j_3VAFmM ziu*Vyjs=PyBY*ZXm%`ADquwpYh}au;95b4Bi~9cs*5!mL-M}IZZ9$E0FT;ecFI~s< zABbn1aSG1NWp+K&MY8%&z==-JgnJ;vE*l8r1SyXL7b4|ILCSuDlr!*0aQpN^o7?k= zXgC03P-sT<^XKT;Lt;I6s(I9SB%a4|WD@JH*GQaHCx}Ww+x1XOK^s^7i8c#Zz6m6V zwqpcsBXP_C#1zCIL0c)*UhE}%khl=`55wqTB>acdY}8+jhB5`GG=`Xq8JqYNedogT z=}}gcD4*W~<2-Ds2H!NcWLb}0%0s^$lI6u3>sV+XCgv68+gVE6LCKDik`n|avp!-< z9>yO*$%B(@Nlc&L1PD2S!B?PT^pq%xBXyENR@Zf6qS?OL>j%3p^r%_ zOa~Z6G;}Ru9pcZgK_3h>g#GLOr1eKkCW?4^uJ&zHO{OXsLrti;yA}5^GDDG(!2I3h zF`o#I^&;kokm?uP6Fi zUZD~Z&ePYV3Qbn=C|y9&U-nz!b%=0pqBo9uhr={+Liy?YqVB27rsMeiHFTG3ipSZvGHoby#le9J%~A;KWb@% z^hWWy?eC(ujwkCm3Lobo3sLxJwlt;W9m;c0Gl3VTYdIO1y~-*PM7ePy34kA|wkOwn z!G4Gab#DEJ&^mo?O_?Bof(;lelv|I@O&i>)uQOyhs$U!M7$VA3r-WKCa9{TJ+Q5IO zJQ#O z*bExbiSW2U7N3E%D05{(I&fo-e-T-BBB+fh4(L!>PCkka!<3Wm04!bpk)+8z<0&n$ zv;n4Tje!XRHGtPx=@&o+LNx-@u^D4hrIwVEab`XWI;Mq2fe8rCFa9qri?Z4x?G2^G92j1K2! z(O9b|m~b}vKX9YUi+{ryG8aBc(eb~eH#w2=<_Ta8gr0?A{z*OvtN|V%%c=3+#1U8z zL#J}IB8RL}I#%JLj6*=E7JY$?Ko$OW$KOi)g&hW#84%406FYJ%$MD1QxwTHL8iU%+ zsIe2m<-I4M@RS>>1HLB-QD(dF{XpQRIkdlt0PezdRX~K`NX>Pic{QoA6t=z#SXxHMTWD6A3M>FtL5Yglep#!3J0lPA9k(EY`L0 zK1>mG8L`&DT0?maq1;UrxJX2_ShXl;Wvlqf%o$pD`Kj2nnNIzmJ`QRT-H$=)3XzuA zg>yGSVZ$(ds8VaNC2C`jN>?P;mK*)nE4Tt2al&Ky5VCYCw5>zs#m6k6N6_h7bwg(q zmvAWyyB8$mFM~Ixr%$l8mbNaJbz=yiRS#Ahc*ow9y}PtDZqNFU9p^Z=2%G~KgU!+( zj5^Ehw@E9ay_jP}7_!IUiU(mRlCdkWgmbrvutM_%7rv|)9F3x**b28%G*?E*a%(BC z%%I+wr^;V1_S^^E(B-d=iNzgJ#=yfJh)}$u(@(Hf9hdn*7@`$dyGbi{jNgu$oCLS- zqByR_31qCC2&KkQ2o#V*|F*_nq>`ssiPl0nJW?>OLdZT>vd)Q)zZW_w?MCO@rQEUV z+XUYv7l^vvXL_jfwCN1oT^u0$gKh`iZdz`{hPJ}E*!48P&czaPjTJtFn6L>8le;7B zfeyo)3VB^NUfyBr*V%JF0=6!F=D%ByoO~*Go8%9lwYxWuY5NtOKc{+nj?zxpgFg#| zVL3dsA3Cu>JWW(6gG9Q6*-Gwu&;y%1I|F~~2zCyL!ryT2dUkUhbWaIw92KhNl!2Q3 z0od73mN;XKI+I4}3l_&1B`YVwqgZ3Y6P?j^F^on&ZG3L%Lv_f#KoO@7Ig!HV^G2N? zt@5Qp+hTU7I&C!CC0NqYD0~2RErttU&=S@1RK^-;7|S(E0U0TVq344fKBh889R!g; zR^_?v9HTh#80U~Vn&`-ZHq=Aspjdgeq1?3yd6m#P&Hk)@h@6x=vT{_Ce*G|3kcF3D z1lQtbEy)NL*?@(hjhrx1@=7gRPZM%NU?IcASnNm_P|^myR%h>fh--*LZi%)L%kfEL zxe*?Xw8@3#N2Jp(ET@ZW9f#$kr(+P9b7EfZPA$Dy9&5m|mdt~fkUEH$U6RIf+8>Qr z-X<{8B;K>5u`D2MONy5d$XSC+;>|D=x5Vd&Yc7e~u2f`;@YlxaU1T;nQn+uPB5lJ< z88NjJrKR~})X=R*kR36kAvTGzj1fxhU#!H!_JU)~Q zlz}!S2AqB2k){cGzMeJS$ zx`vF2OeM>$mzf-FyT%&LDAFNsNRMl*K>}k$>s2T`hl7-7)=}T#+`eaob94 zGR@aWN8zrbs%}#rdu%GS(%91bbf{^Y=CC+(84O3%0mRppi>a9FUMcc z6=lqEX%=47`Uaj2!DJ+@%3a&ewy(+(ckysSIbhWU8mK_n=(qMk9so@p9@tV|N*XJR z)AU$OB|UhCss5xl@MPG3s6FVyL}I^PiG=Er)K$6*CYxaB%=|5BFUJ5}AOI_HumK-; z0E%X3&?=c)E4Ri(I(Ne22uP;+3!f-l3!Li6q$Z3SE7^(l5yIsu%70){sGM7(a&FP( zyO9Sh6eWmQ8b)>=%H5#qxb)(jSgms042$^H+@I| zBZ=Ig7~(SIBY9xHAs;cVY;^_O4D><&g6?NOz?KGA4tFdV$#hV37b({{$I>M|hg(Oo zK@$T`7^fMxnn@9^X6q)?g$TeP(IMsM1`K(fF4h{(Hm}tB5=UdMe#1s7an#o9?g4Sp z#7i$(d;@oFpvJ`Ev(Z$=-_F3G`$27MhA3{bNvtJKg#ClgPq+8c;&PUJ9?<&du-4Y= zs>=32i)X0?4#B~rEO_$Yev}0_hoaW71)PL`VHLX>nIfDN7AAJZRrstu=MgBj$2QU{ z4{!m7Lf?+mz8ydsEP@0h_AL1aVYbcWv>;TqZn{atrg(Xw!DFd@qbqw?$Bg%7l~6ra z)Qa;lSaN>DSU~&0>M?}Iz5q5xc9u(w?It#ZAK17L$$`=Zw0i)qZH#c&WdGL1#4_7`DjtV5 zp6r-duAl&hFFuPPll0(bQ3+0}#Tl5X0c5gBaSrjf-yH}BYTFnYZ> z&o5y7*U%(DA8!}KcvqUn8%efJl)9FyyaPkWYpk>2obgxid1MA{S#VacusE(RCGAUL zM8~Rj08^}OTiC+QT>IX}0xK{%SSO1jlm(3r)h_Vtyb~v9m-tZ@|kg5(lHMG>_01xb`++G$2KcD0`31r)9*8x?_k>s2G4p!7Ad@ zB?w;8b-0`m&mQlfX2>vINS5^Vvh7BWf)tzFD@SLWxF%;9-C-_E6X1HV2Uk2IXF#@T zkP+E+M|+X|kRk^MQbN|rVv+exw2@W(fW$S+jVrt6yrtw3LT`bo?{YjMQy`Kpo^*9A`Hw7k&62pg8;C;dFsTz4K-iB0r6uRR%n(eV>!4}zLZ7Lr|9!r^{;rJ8G(WKMz%+bmRQdc+} z69+fa>y_|2D81&C1a|?J4RjW(8sX-0kOM+eT*h;h8k-NNcqFE7Dw{Vm|1>r)B)+&D z<|7pHp)|Ue>TN5GinEd(?m%n_`yUu*%nld8l8^xIgMOhhK4f~gQNy9$9C4)Ggi=M} z)%S&ZKtvTSX+g>p(LomB$C)8Dhb``TGId9(;?eFF?riZ@GHQ`6u90zD%@!XcKjxJv z#Ffn4XCZ9ZF!wnJohn)j4H-%O$;{uLK?*fAhukhl5Y}|I4IgxaBEwUVCWfLS141qi zQUPDU8yRx>V3ladZHb0l?1rOoi?B)4Jqo?KEwEaoJhbpiPV*Hc6Y3F3asc)DR`kc` zBVw4B0gPc^=34Lq#B?4&-)>g~l`_mFDLYctN#uY6Gk!Q98}7vZw;{-7417%kyLG;j zob**%5J!aOTJTUfrGp6U3J`g@`(JCP7z5IIvK2^py{Ow z(&3~V6mhCD1@21@CV|X~H+yJA&L(#(n8;D+< zVoVBNWz);28b=je$tiJYQ7u}+D-gZN!9_k<8xC%e!t3fyCTpM)Y!pEvbD;ga>tdKf z)DK20`kH{vv#LR0q?lFxM7a)Xn*35do>iT~5j63v>I^{2S=C+;g0(B9`R-X&GIRH- z98Ucojxl=Y&klEc=N4+1oOdo~mk00MDYkchkh)9CJ0Hz{4t9j|utd1mlGZGioG0G0 z^}_ZrFW~Sd?Up_}y1plR)-9HyqB}Yg+I)q_=BWqr;x0Xh_F-hUwr%0uEy9I;Ec`Vv zA`O$1XuowFL>$#HuO~uZ1`k$ysK=*%??NA%LuZwDg`u`7T+72ewYnPVaX5UUduoTJAAo<3<4 z&Rz!rM^`HMAyEls)W=~>p>aZW3%tkJtiZluIp3uzt=xZ3S&?3H@E_tG@3Q#g!8+@` z{9QFBKo5-@dU$DE$ja!dskM!(y;y-X z8ZR=8cz|idEVw&Fh3JaVA~-fJqviYl!$v#tz9qq#7cWY4CV{^Lfh#e|9t$ogx}Qn< zD|8!dCO^`|=-?@g%oRAuhEDI9b0P(OJijL;$1aT2NVu?Dk>Kj_*hAH|aFHril>v`(%tqFjagB)LLuNl~uq?MgX-bmVF^BuP6Pho}p?HST(u1#%Rg8CLBM z23J*TBnO!olOckTC&=*-GeMv8V9OeMuw`#(k0D?;>yi`m8e@#2=Vk2YWZt&wA^ljC z-DT`ER-6LXKanjNv91;jgfpQW$T?MJm_7mt7!*U@aO@5awN&==klxGnEP4(OC!s1D z>A-bM2-nRBhr^f>j7P`DDBg|qMpyu!Lj={Xv3@3XXX;x>W^}9*ODu-Ihng4QoZl6+ z9}Mq}t5}l((9qk~1ITnt)?r?&-H3cjZ_OGVOGbJwv3_45th@J@46*%Iam|%Wcx-kT zfw*d2eQlGKDE7k^>vCCo;nQzvc3V@pkQ^FkK07?AGO7yO45?H4Cm$Y*p$` zU;j+_@xGxd`Z}Ax(#EOE=;mEG+zcLu_LgA1#MwO6mFSh5$K3mN0VF7L(66!fp)jIr z3b%1q+UX4@wA~Y!ye3T3(U9mIiXFeQI0!0hiJj`Mz-7!kS_!TrvhL-tL{@Ed6C`?c z6Lbf)Gw+y56m@jpki}xxl9bhZp>LNaTp5P@3p zTXrSG;S9w+u?LyWq!V<8;yN9U`wscZe_N0L0X=}!c&qyfcsz#uAFB@Ycm#h<%;Wi# zWSDuJh>l?hgDyJQUfp!qvWn?2ah$5KrbTfwlT8RnnwN~Jt3CHL>otj+vp1;V0nmuJ zx<3*UmQSdc0IOu%-sbWLl6H=+ar|69v{BI@8;UH^RSI&RrQXjw5_5z-s!eG$PkmGn z>_mN;4QqLTfDii%hH3dW^9?-P2XV^82mAhydq5v(WVF2y!+E8Jbh_72W8c5*9l!4{ zgp=mJJH_hzt0Bg||B;~YYsmy8-v5NlQSg>r$_e=g1=4+b*3=h%fvUuAwT{rK9$3GI ze6SQUG_YY#A5_mkdI_N?Ug{hfwEAZ3nGDs4gBG*lc*F^avp7%KAmM?P>r77CSRicW z$lVxg*L^M+_XV8w3~gE>M7}wI8rth+cu>kekH_XUxn^pyD+~sk>C@=5pYuKnbM+{7 zQcv};8T@r-JSW1lEW~xneVy};k8{g}ye7OJ<-c^MsN8b#08I_Cg1NtM;+` z!)3VP$cW3(pAg97S#EnA3wml{TU9W~O1H?dAP+Jts^&(<&c(DUXH9iW7d+ODnMo6i zVI|4JIiB`rbWVFjBY}qKUE=)}xB-NlVBuhwE6N=2lpV8vH zTQAQRgva&NV`tEc$T<`?H}cJICL6+ z>V3hZU~K+DY;1^6EB`>uXKRRzc>X5r54#u6M3`F_kXsX2L^o>!chMj8USXwSQM;mi zu`hGM1-K>=8yZ+#eh7We3G?xrgzOQ4B^1aPEAL?o(@sI^30oy3B{t+d2}u;khkOKm zf^x?=#Hq)$Scw<*U0n)BFz$ot)LgY6EUK>O!mxvkcx-SOWz;fL9)oaA->ifb6s1I-7( z=P0hBVlDhZ`6Bp*g`vC~Twb7}{D#)d`F|Hiw#a?_^ddpnLWk$Qfet98hHhv3NDQX; z0xjVn-E}d&8!;sbrk8A4$u+k!&y=c*Kob zWen7=?@FlAT^DMX;YKp3ZG`NA+FoJ2)syfE2W*_~$w1eE%242utpy~<(|0Aouc|0t z#e!c2!Jp3)1QPt;uD1nW2~wU%`&R1aF^GFKQ!tw~adbGRH$;yV4UV7b@Qo%Rg@c$) zHql9T%lO2wstEz{ILwPXU@X)KtPu$LA-u7*d_8N#dT7KS;n74Q6k1?w#6P>(8nFq< zm36YnnIIXgr@+pMXY~1Kd~Q70&Jg+mQTuZ*s? zD4k-23oyJhdFe=>vH=J%N0av{uY-w3pHfNOO+u2{U}IdLGS}ZmpK=VNX7DNdCyCm8 zD1q6hG;%tpj9myh{`rf94oy{@x0-^V;{yl?U`eKD62`jg1V?0}Z!V{U{0P3&V9Km}6HW8NegnS9Jj z+(<_JJt3a<`k1SK!C2&Df{1G_A9F30QCLH&=h_J9MQ_1qLr-JD=*DssjAlaMqG0qE$oRcrgopm^dei@yg3*3ZU%i@RC>VW; z>5SeeS}=O|M{fsJTEXZ+1GHT*`nv(TRx21?O3*14jEbmFQ^9B~ZX{#(m52tg`;-et zz83bbS=QbNtgeF5!E8WOH<)y#?GEPsV7r4AaL3kxqMlVrS=*j4nAonN3lOwA!RIKh zp`xh+UFRXd%;-9I#*JiH7z8Z|UFW>-;W`@;{#>rJ1;h*2Irs{jS7(B^A`)E>x~*^F zKV$RfQ!v~52>G#>KNQ{Z%yO`uixGaHV`NYhXiA1bX97hqs3+RiiuxGTp6X+j^wLCN z$lG5J_!b;`3E|4cp|_~^!NFzE**1szV`RahshG7`Z^qRijz3k%iFFPUckrAa&Ra2Z zsLD`5tdCK*+5BA$4!w*U$#AGYL@YT}B{&4P;&2F9GKW?XhaNy&b#Z7p+=k%LH?wRG zWnpB&p_8B+))Qc2Y#q7}{Il+cA?qBX^rVAys2Zpnb!aJWB*URBpqN;P&iw`)n!nw{ zp_{3+!8-JM(B{w=+iVUU4BfD90uy6%=o;|PnuA=}%OR)_RH(fxev5&PkwJgPjbs?~ zCGbz^g%1B3{3%2jbZJlt{EXnw6J<7kUIQHc+^cSqGF%UHp3$7T_71OJzU_^S0IV%DOTLeH;5-o zDg$(4K?{Yk_rC&fcSP8A@pdn&tFZ3nmD;>5fgA*Hmx5>3?qI)*w|;9ExI&Ga6Jd`C zn27eFyrr&H65zJX=H_~QsdV^9yp`WF1aCz{+H)F%w{)I(l)XX8-tTnw1|@sHBhs+- zlNWy90S_=NrinAWG3A{eQzT2I0UF2pFx?89$83O~_K220(Ia+`GbSLE-N&B^hX z@`052d(9V;zd4AkF8=1h`~`n+nQ8O)3CKh6_fv4u8V3=0_?rtqqw$xMwhjs?Gl787 zUrr!RNn*nLJWvLU5+o&eG=aK-h@~!`jG_W2>)gBz@+3fpkw6FAd+R}FDh(fur%Bw!N zIrIc5xAp@QV{>R1iaI+Y2iG_>(yp0!9bjz=8CySKVq{Kt+(?Ew^ME^;LuP(gZ1FcRK-W^YU~ z)RO*2vmUKcw3c*}p#x;S%I2dB4A6Eh=?nw3UQ6mh&?(iD_NP8gwWQ9tk&N9JBkaNM zQ?4bIeT@DoxqAhX$yH0LX9J>I(%w^TcQEi%+Z|kj$&vLRQK#Wr(o3-W*skGOiaIOc za}?JgPACCiJ?|P~*X~{Nk5vSC{}W;#-F0o=Sceou33LV&i!lfq~Oi!ax~H3+`EFxloy19&L(WiOh?UK7`jN-?rE9f@|*e6i=IG(U>Y^kd9F zY4T}L;$3t54TjmKL~t$*dQg&G>|DLd#e=BN>qx+4p7!X4W8bkYQGlu&iBCNUI?wO{RI7~F2mBhEZ>&ijqlsi zTMMaJN0We=fj)9E*1lE8hm*lSR47W4mXnF%Td5CE?j|(fHec?eKyx?DTH#CdxxT`k_2>E`BhJGW>y1{F0K@*%@tF*G1{73(`Y#K1 z*aE_NbzmM4PwpA$pWa!638Fgisz@{r(J4toluj7 zU^SWsmWA`K)74}N)Z`JU2|XTn ztBiICE(V+;Lm`7^Ax?$7cj(ytB!es4n4O;@{fjiQ1KxdkMV4Zn%hxVHK z0EAAL&a~0g&ta5mtj)(NI+NJcdrGiJY%oV>YJr+tXO<(g)pW*V>No~|6TLMIE7JHS z-PTQAEKO&_+@mIccRibYwyIvgs>#o9Vex&}xxFS|h6w2rUmH#S>+gl9K6tnyzQiW~ ztOWafnA_xc6g5K4Q-6z;R})`+lm92=qRQm|YmiBM~lLI@3mzKZa4N zv6_b{I+NJsTfY&i_KXeYnEZc4-AdD$YD^t8o#_yh4^yR*FR{rte=WrK<1ucN-%FI*H1T!X=n`LilP~?<mtpyrP9uf65cnGP}e?i&my zzjhvyPE306NcC?tnJ@T~{rEnZ;fk0@U2L|Hd0GHSyR!&cLrvb~3-@ZMJ!RGQajWxA_3G zz<&(pI_kuAY!fd0>KiMdRIq+yjn2BxvDhc*;AGa|7fI#mSrvG)9Fq5$Pd!YbY zRwl$n+beH@pI#8kTj-g}Cn&fA&Q*f-7F^-~WgASd@88PCcOxb@?UUCAM6!;N*KUkb zjdk5oDaq?yONcqw26F;$5t)Xol9mU5>-EvZdYoAnv_g104V|-@iz7U;=rmv|kaKb7 z{FgmKBPi6l%E4F?+M@VE zOS|OFjuM)WQNoFsho>a8mp&Gjw)rp#<_N77sJVsK_=;O-RQycB(#BOAg$7B((0-fc zsOPt^%oboO&_0>1KzQgF?|3AmRAWu)os!JH-7I8wv<>FS>{LV{x6B5^$$7maI1f9^ zi;=;WKCg?*xScr^DdRdd9|VYUb+M^jeG8TXunu(#FEVias;UUwf<*>#1v$-`>(ndG zT9tN@2J*DD7qGOJgoM))F|nF?<-fC;q^_l5k%B^DtU^BQO*xE$*bSR1La zst{ycQolvaTNY!=fmOCZRXFb!eRXL$O1Z1BOtBQt=tP$(`bhE3JVcQ?cY=8f3K_X( zuTds+ zs)AHFv1Y*<1Le4`fnCCU%m~}39QPrZb&PU+^?{J`_=8hYj+GK@KO4+Zjv=pkmE+(S zlU5E^5Vv-S)d;chn1R0B3y=Sobfo!x){!F2h1#bhb74pwqaznEN;u)JXG%J9hy-gn zP=Yx+vd4O_j`-GgoQ_O}dAM~%Zb+fr;WwKg-LQ#edH-kaO#%M(l1G+poY4O8uCVt> z*@`R^PiWUjup@0SN0vDV_wE#6;2M`Kc|w~++IH#+-T6ak1ofLvkmfR}{uazG+LyQe zQqxgFYkEfr?Xd$=653M|>~b5-5!zo7t=&SKTjdrSu|A0j(k!>oe$8zl)f?HQF8Y+D z)1GZ06A?#T?y-#vM12?~>~P&*kxt?TqVMa4P&e-9c8}juh;!8(K77tC9Xr1=ZUgDL z(ql+y|C?6u^WSDkRbj%{K08{0$l#Jx8=c?!w*0U>P=;qb3Kx;8l~XXI*NdO_=b4#fDTW_~YQbJsHoh>4&d)lg0M+hwb&l z^}yUEwl@0VPu>t>Td=nxw#0t;aS3*Y4d#e#8j476KYY$JF0nbadZQn{63#%K&Hk$U zrEXdN8c9;$3$RvQ=#4VG4h-l_2pwUh)a?4~b zDs|2q6GFKQ;;)NDS2$unJ#d4w3Szg3E^nbWQ}-GRYUq0;D7843>eB;VR(S-~#&l%_ zqf}!(y_+Ja#OcZo5^SCg<_PL8pyn3Tg5_}p70R`+BEq|jqt6cP;Pz>5S)tU|0Z)6x zlb0<)?fH*cLV`4ZYo7!INSh(k%0eQFZpXHjn>DA59t#h#C5Q@yk|Cj{OqQsa`gr1}ExX6^( zLa1kFF+*0c%?l*PwGjJZ9=qI~_cF`-7fb}(SIS(DVB#{!Hm-%Nc}d87^3IC97t?}D z+_jLf1Ut$GbL4#rau2t>2R-hRx0Bi<_Q9nhu}5;Bib@$tcW!+b5F`ph+&L)6vu!X(CS}Md+%lQ?P)ah{ z6=tQ|Nfs9@c{7xMCz=-W#~SOeVFVp~zQiZ~+2@oZ7PaO?cJA8)L`C*IjQ)fNjay6H zudzzdJxI-t1K2|Vb{Bvl2{<${;0^+?m8$09L3J;P5q*gPKPQODo}UQ72@9Z{3cw8lFem}@5(B;=0ACS+8>$73`y~c^NdT@9fV-~|fSX~k3B*w) z0G|(>V9o>qI97~!haB<2#DFIWz~cqrR5{|z#DF6N;4uR5bO|^(G2oE`u$KTl zNuv78(u6ebD**Qpfb*Xd%vqNhu&V$}7l6k~RBuQOm?i*!q_wB;pi(*F_{4xd0r-Oe z{Cc%Ob)Uq5UlK%QPfGwEC{g|JfrQNYL;!vu0PEz4OA`aWDga*;fS;}ssLn_XxK;qJ z7J$b{<{Xn4@L>VCL;&6|QT_R!32FR?0Q{Q(JW>L#O$-k40K|8|4eKk(>b4jt*7RGYQxhW?`2! z#;m7x4m?zkU1&ISya5w2Ooh-@prtj~y)~GH$&jp6)ZNa^&r*tt*kFdY!=s~6E*^u& z`2=`deU|vp92(nMd}^WthAwy^v&U$98`J?mg2-=?jQ0C(>lb(b(kidvAZ~cCj2A%J z??1nD;CBxE&Vk=K@H+>7=fLkA_?-j4bKrLl{LX>jf&;mQ(`J-hkzG(+JY!NpS<#H? z*;5LNiVG)q^-T&+F7Qp7iFbR^&zXhOOJ|gnW={*2mSs;Zm{pivHnlLjv|w6c_VnPi ziwaA!XH3bSTr>^M&`D`_ugTd3(I6-~bwgW@MpQZl1tSa!keA_BTN2*S#!W2PT?mreuAKy}&F0#LxriIR_o z+zI}RgMSiyD-B*$R#GrYu#UTo(_bQfXr0CBCkea?iwlV%M_pc2S_qaDmle${E)x19 z*cXrPM0m#SyaVx`Tr{hQl$9}#({CbtwSG>SuBaA6ma>LTD<~@|n(g8h^klYUDEI_S zV!=BaS~xlTqJpwXQ}H*cK1H_%d9dwrSBm12v5ITx9D*g47j`V=dSbzJvY(Pd!Id)B z{65)d&L}G=Ry=`}VJR#!dwSvJ*`oF288ATtN9U8Kvf)n(mXwfbl@?tIXE24Y$xai= zB^f=vELc#SsGmggEp$2|doC%MUOICI)PBbF;w!RCgEME&C@CwIE}%3!I5WF!M)rUx zOH2Dm&IykY+uxoteO5tn5!elvLv81wQZ#QaevcQB&j0hAPs#^+!n2e7Zu*>#d!6y? z$L|jK?Sfw{!uhZ`>LUcPaP8}cU+gyU?SfzIGVx(6kZ)J~?v7tL-C7e zdwqB&+jlsA@j$)rNckIf6ZZht0T@C{rbocJ@CGm<{zM{UdA=Mazut3FrjF=`hDPuo-RcHbUpvHU~#d7FJB*Ya>m!9{f~3SHVpIIB8O0Y3UTu1TRJ&*y)NAu&%xn^G1&n zf)(FmJ1!Pn{<-~+DJ{Bq+Ki&f`J$(MN+9zuEi9Q{SUh&h`1~TfUH?flN@KP`KmALK zrpLKIVBoZZOAGTS&6r+Rba8M-FlJvxV~fW3&F4{lKJ+`NpcEL!?poqEpdV5$;p6iM zjE;p!kst^Cs_py7Ztvisw(l3Sy=+c+552Om1RN_a0@X2Di*`e1OfO8-jPN~%16sb4 zQN-wWd_Hp!Oth(GJ`NP{W>i!;7tgpjM$bh|SLvjJ;)0U=nKLdoG@BxHhUk|gQ9B8u zsTS*2{<#AOsT9c^DPN>9hW@F0Ad4Sc?gLVRA$>~RH}lKJA9-SIJXCp0dZ~RJdixKR zH_iPj{FCjF&Kbv-{2MS(CT*0|=f`TyOarpeqk))Eq^w2YTYrL!uRYAS!PAU>AfFN> zk{&B336La6{hHf5dNg=i!ED30ar!g1kM>s@oR(65=Js}f0|sK2+$Me!Ts(fZw$4TN zGj}fO*x1kWR;D3)SW1ZnemeQm(njL%RQa8*(N7m-4wT2)Ka>ZWe&z{5QXJ z>h#I8eSN0S>QjU?qAd_|>|f&#k;nO@tMifaJLRtGk7S2)iSlj}e2Hw42%d6tm*^OB zsf4E-oqp^#lsnRoeVyh4e*D^UqI>F(-H-j(h9^1*F4-Ryj>b&($A+`@HmkRw#L?SN zb&itWo~m@-X1)DH_WO$Xd#e9bXspoN8Ew+rK4pco%lg1iCCrm-RZqGxX0Rp5+LP@Y zsZ(iL!Su2LY*hBQzRAJcRK(5VR0&(7X4XsnYXl@Sh^?!?P?C;RD&Gn>BQF61gj1Txrr7i zsHsC)GL2Ot61KPTO=0yz#XUu^C>^N)mJ}8jIOWU~0ow%Up1`Ftrp?f}%R?#)6E=)8 zEn!n#^zjy`6uuajolMJAD%L5iZs`h~3ZP5y$s-EDwi2kZFcoB@f;ZwR*4RTHd2Yh=dKJG$d@Q zVI9A<#IsH7u!|IcPu$d@ESbhC5eeH*F1#psasHGN z>_hkUAt&R$P@r8&eYH8MH8q=S|ASWc@_GAJNm+hT{~0rj@~8L?PJ)ty1yp|NwElhb zOD4_CFPWY{WkCP@(&8DHW4rv+6gwF(JD+Aa=w$kgDP>nuSMBd1iVP{CrYs5BpFgtz zn!UHcd1gRBT-YG zj(DwUFHvfH`fLfM3>TGITRkfVSioe&e@NXiC^HYTOB%YP7^{O`c0G48&iWw&j#uYlW_)0iyIJ4WiMR6co{$ z%0h9HtyOxH>R!{&;wddgpI~8xovT~F!?d!zRjrMh;@~`bZAvmPAOm4@7 z)S#lgOK~_=w^9dHd6XJ7Y%8f{b=#FB)i}z9<_ddlFO(Ry+^B5 zu?kG>?s-Jt}J}Pr`Olh9uL}p*@+#Dw&DegQB+V+(-pRp)^IXDAlO|x^_euS7VGT z`1+2BG5Vd9dNGdQ2Yf=P@GAb27m-ZR=q$Htz$-{ZIFKJlTK4WY5YGtBrIL@kTqJ~kX z#@&JAXkTpXRP95n{6YN&o3t->2dsVO4KPLl+0)-tHrlA#m%>m0i!v(3@pM;|{Gg-^ z=GJayh}%k`IDU5;jd7dW$`EH^NaFkhXhIjO&%{{0J z)22>Ha}w-0vTY|(yC_2vcB$)6g7MKVN$==x%L;KxjPOv1JDde)Aj zeY3p(Dz2yVDR1Js{r@32#vxrhsX^RX%-88eIQ{1fzCK33h;LJYS2~{_r{AjO>$&(6}sO#LFhjllsX>3lk$ev4k9m+s5cFXGz>95R^BC;92OZsO~tz536NIFK-% zPyN%c=kxUm`bB*EimxBzxqdnhHB9%N?YaITUmxtbJ{|`kru)wETt5&8@1^4$5ckjD z`1-l-%f0{(AW8QP^<005ub=0+-eosxf2rsCgM59M=Xw_$0F%xqBI>v9!u6iG-dEZu z*)9ESme*&(p6Dk_9;U)^2R)vv1VR#&OJTHTi0QomcWjn$Uz*pt|iFpoHZP)n9%3CnhKS|}x9 zP{}rom@t?SLj)li#bNRpCJ%>6xF#47uJka~07_mAWIAimr5 zx0-zZ(R^Pp=@(7<*T5lVNWH2qosY_Q>jln>UI#%JQrCDN%-aX#`$FCKSva~3sZDj? zKQ7;Q)P3Izr`AWlFj(^2;>5vpNGE4W?rWQ{A=LvfnLaFUhjUdm~zfR&Sx{8{I8%Kd`2M8 zvl*Y9M>+WPK%QqaK6#EXpF1J9vl*Y)?tZU8Zu0p)6W9F%BYGjAFYhW zoafCwf6m4Bac|0i)p7v zOuD8TYSE;t+ldV&)GvnNwUE!n=9e?_LlVQE74!SlrOEGDO@8MgKls$~;U_eMUTpoI ztIbbmSD)Wn9dWVwas1_f&YSkEtDh0E7n|P~YVuoLDZkB^rhYdWxnb*g-n`V+?-J3I z`r7}jnBSm=x>$OS!Vd61E9ST6(&Tr%Hb3Z1K6UgQ(0~_Pzc}ofs%IsBnblAin_sym zKReEHw`dPcZuD#}89{N3D z^3S}?^c%-`jE|kiDPAJ_9XILf)bV#E{+rhjEAhpJ@?YE-?3RAku3agg3+2B>lh1-aL)?|4mqi)P$hCqF)LiR>RUpJD4~w9nay#wAs{E&g?J0HTk?|^sT0smISQ{6FrzVjix`wn>b9q@~YA0rIl`Ob&%?mOV! zcfj9+c$+@o`4HZH2fX_Z_`4Br)8{)M!n^N)ci#bj7vgRDeCI=W_Z{%=JK*;t-lorY zK7@DQ0q?#8{!YZV5C-+-J0HTk?|^sT0sj`nTl({z58>T+z`O5&=PaR3pYMDK@4f@x zeFywr#3u=Z^6;Gx;oWz@yYGO%1Mz9X0DgwRcnI&l1KxcH%(Mm(82248?mJ+ZCLgBb zz5~X6N4|RyZ}XiHXbtba1Kxc{n&Ung_Z=|qJ7Ab5ALi@61IB$vzPk}`^Swf#HN5){ zc=sJ?j`?8Rcfh#sfMJ?^n6LW|8226d?n3-p!T_G{dIt4ZL0XAiVnyc=sLfoYJ%Do9R8FkNXaI_Z{$DeX;O%C5Gv{?|^sT z0e>6dZNKI_AHuuufOp>k&($ECzFiq2y!#G#_Z{%B2fU>x-}w;UeFwbz4*1t0-trIM z`4FD(MZN>xeFyw&@y){Xoe$ypUgSIA-FLuqwXB`r(Y|a4K7@DQ0q?#8CZa(E#(f8j z`wp0}55|25jQb84yGqBr+;_mZ@5#7|A)SRlrwZjeRk(*-%R6PzJI=Se;Q*=Ek@nC zM%bdhx`%ngjxQYI?U_DWy)@dzHSMw}2k1qfx>$NiTOb{#N1j^<^rJrgZhdt5y|%sB z@}EH-ivbMx+4jh@L-Nca&zzg*#pdyTq{(qT)9;e>FNR+Y&5;h%>$ybfv5ZVFe~HpN zVbbf9^l(2>euGQT7%dbZ&qb*%ec=fA{rPw_T*Jem8Xnd>>-Fy!?(dp@6|g<)yGqJe z&0{)!kH;<~{j;Y21xf$n+ne)UEbr<|R^E0a|1>?eLPYvy3H48H-}}Mn;?WIBNIPmd ztMgf`$!A#dsg_w$-hNxf55=l*m^b?R;(xGmdHlg*k-u!Y$onVTJQ*;8@Iv5+U#3N0 znP{&r1WM>fCAS}G+3wN%Qqv`awpw6&f$!Ey{+GU=6e$;dI(PDd^YQ3_B&ynXE5kMU z%;M99@pmmBRrjOXd^TKgJ}+FKlSr5Ca^(f5Ut15aKagH$P5M_$`ainz`x+_VrRPgq z|9MG&8vSnC?RPTL^~s+YN1(oNRP>kEevuk^ruCT@e{uTE*smz(*6!-Qbe)v_;&jgm z^qceSJK(h&Uct1u=@Qf8$6rCTI0K`T?RU#1rok3T??+cQyjsfl^2i4A6CapmG9G;R`P!oZ(;?598m8U&R;Z8UfUgJd9t$@7niYclC!qxbsUV|7!Z% z|Ibg2Jbzi&^*?#{{oj39LpA?4DbS=qlLAc&G%3)eK$8MZ3N$Iuq(GAbO$szA(4;_< z0!<1uDbS=qlLAc&G%3)eK$8MZ3N$Iuq(GAbO$szA(4;_<0!<1uDbS=qlLAc&G%3)e zK$8MZ3N$Iuq(GAbO$szA(4;_<0!<2BC zoqiD)*px+ z^-Mtxm7@{mOgv5(9_q7#kCR5vpHgR$Khk6T`~K+h-<9J@NUvCqMv?y}$^Xf;iZ4Z# z#1S9x#bYs^RPx0~6hBD%pl9*ENJQs9k3&Jo|3C!jl}+;Pb7_?+hw)koET3JDvHaO` z81$+fSw0E+BivF}Vfd;GEY55C-w666B|Mrw!7>gakzPFigZ8~1^cffWoz(5)03A_L zwm-^yBjV=~7W{u;eZtz03eN|jyh8^-A60o=8V%(I{i49Hr~rtL@(elQFzYAr$nRRD zrypG&(;k|h&metWAJC@~A%5J6gjpVWfdB3#6%<*aScuf1BkS?v;|60&TKe~NLZ{gn+=v@Lol6|BTk}JU1O5d3;#QA8Pe_J3zpC_5UKjpW4L=kYe%mf}M1Jx6ef;<}ghMgi-zFK)ixb!jdXD}WIuE(Fjc#K_B$Ae$vVFW8~`s9DzKTtmKf2JG_hv*E_^gr~sC@JRe zZ~APdu1~hy7Fk|CqNIPKyrl!J@H6Vs3;j9F4lC)QeY52lPVQAUEkBR`hj89$X(j!a z`G@^sN&6fU2VBzs5-57*an^zIVzrY2tPrBuM z2K~iRNfFs+zApThEhk71Wy=#Q$5~pEKg{xFomdOurG8T0u*mlW+ne=u0FU^1Qp=M( zK0}O{5CViJrRwTY(S zTv8Ese|gaUlsqW(I=!W#Kk0za|+gq}v8%_9t|2)%0cO?CZi$^(7P@|^#USDt)1vP>UG z!mlXL4X}q6$e?U*gel(`A0eY!tcc}x4}q%gSVKj=HQ=Mi;E_g{o59}xbg*gI~2nbGNO zPDo|k`jDUW*=zC>>$403FCU3Mf&Pd5H@~L_9{Hb@#@6s|{!f3K{g3)r=l?YFM}56` zln?ga<`3KSz*3;+jUR@+Cg~xp^9KVg!@hw$06lE^og2{qS)RPb*X?m*9%q$(<+0^o zJ_-5a#~)s~{uSv!x;~I6l&4xgp!^6Ybr|vmM}C4lFO`!b zFOx>zX3CN95o8^C7V)r$O3+_OZ#hbP9p!<(V0~Qq5MPRkZ$Z&7gRt+x&t$K-fgd{1 z!t!sD{!IJSX^rvdIzC!?5#>>JsjtX~7Ah#8JZWjIhEF6I_T){pm1*-EICy9{l5eL zN(ixanEZ*towfZ9{E76F4BGKdBY3|(K=@}-Jb?%OvluThagi775AUhT59NK-m*2H^ z{AKt9_O+(hNu+Wv-JgLzFXC1B0qs5HM0EVTj(7R@)^8d89)tYx>=QS=iokpPeyg^h zOHM-Mt>rJ+E93`lADou)84XO>mRkPg9c^EU(noz`PFVUAAOhq+l!x}1TD%qg8R0zi zA-kW*TZCSu;w4>LWXukcGmZnm=gl69h5 zztP@e`$GTB176x6_Fh%{Ncz&BkBB{F;4?mW^e^^TCH<>Gf42Ikp}sfzaPd4DSnGS_ z&+b!+-lWU-g7_$pE^kGb&!ktp(DV%c*?;uPQ#wF#40&wr6~Fz0_GzTwPx$fu@Mym( zc!XCQ`-AMk_7nTZhtKwr>MY+Mq7#c8!tQek!Y?@g0eP4y2LfkF0V8i;{`nBfYdmH? zg!a+6bTF;&JblXX(H6`fM5U_<7lVF9*aP=}cW{%0^>{d1x$*NiD#|gGFHa}K1DM~4$G({U(@iqSp}a?U@9KxAq!S_iA@u+MF@0)8!t4)_R~0ES%7^}g z@m!QmY3-GKIqBI;fG;_?egPQ{VVn)`PHe;5ArCcF^_M>iz`ikMW1ZKa238SXe!8!h^^!ZuAY)NB&jv67|Uk6Jk$j`M(9>)H2QY zl-SD;#`CbB7&h`gkPjwfn10mbUD`8oC9IV&_D3lmOR+tZ41@nlq4)%UQ+fzKqu+TM+WJQd~XbF>&@MP5K2)QDQ^2KmY9~=Un;pP2R1eh(;mJn)c;{PO+rt2yh$`VV!6BmeaiYhUG$|J6^d{UZs39*oDko^K`m z2ei4xGO#}kIWd`U6)DbeXN>m9Ch61+$HiWIT;M?;&hH8A3iEp}x4x~_-%b77)W5nv zHTAD*>R;8=zyG%MFXnHw{-wQ(@kr50c={Li1m^3Mt6veW)yL4cGM|e10;6xkHTpJE zqi@3r>RZu&)Rzy16V$g6SKmew)VE%IG(mkUVXbdzZ#*tTF0F49Vo$HEZvp?x(YG(I ze_2sH!}v8^|Cd<3KCjpR6)#({^I>)C{|7_M+#g`aGZFBEJhZZ5pTHgs(LS^I98W$f zz9li9{2KWq6q5Bb%|95w1u+;g>0!J8{y+w_$Dlu8&kDY%RE_VZ?~4}$+RUdhp8O~H zv~RIq#}Zomt>j?!4`IE2f_Nw8<^Oq|pMQSw(}#h*0B{a!z&}S1MI=3p=UceHLE_(R z;!*yihoWBmxi;2E+m}1BUjg_XPD~dUDC(Il@eU{Hg+ae4#v@YQhl(;D;d+!%n3Gf( z&u7ZP@b__5ydBTl`6QXollg#<^gsHi=Osi{^u>CS9e;;7o_S(D!g@Vu*^C!D9Cj@8 zpgto`D8vpF|2@t#rUc!J+^N-=Ha9MF(;hQGicLJy&z z@LQ{%iw6xfzeT+;+CRFKL|%AYeoK1m2f}ahq|0wu&x|Z3WUGtEU;gzI@>iSmCmF8_ ze?{74J%RNHKa5~~o9z*2{ZT$Ue4#nZlKi4!_CGvk{W~QS$|>ou56XOZQKBIqIbZie z@(ts~66V|d^p*Wv3+R7L?*;hLiWWUtWr8G6E z=3k7@jxM!jP%|CI{OHl8){F=^&exApK1VXbpbYP1`khhAXD|J(klOA~KY^QpOnQ3* zz>ni%F@HGL>83C7EuCKa0m%2fmcLBnw8#syUai;rDBq?2X8%Ella;L$1lRJt(-$rv zy$;T7>gSmUA@5Tv%Mj%~)zgz;EX|gjdO(OAB1`Q8D*EbUw1+a1zt+!$?~-sk z;h)&Ta2M$#_K@`VOnaAPk9f-Sa(lO%ANK!rI9*=-A>Xr3_cFJ(h`j;$5vMbgXPEqn zFv}z6FzqwYKkKBE0&nJ1OHRI%_0N;P(f({N<}dWwBlOQQJ-c76bS$5zKFex)XR7E2 zdS#AvcDniwMA`l$PNtoAd}R!MI-H!`NhT1$M|paLKdH|lf3i;BGWpri zH(qkuCI6hk&pK^A5>I+c`Z;lZZ|PsfFP~kH^7iNYkzPCL4fra4!FmAl?_m7m{PTLe z`fPtc>Di&_tuoK%axQ&q=|g+FoT`HN(gVH0UomGO#q>S;rZQ~rhV&ynA0CkU4j6iq ze&iRSAIgvP`XH(BRAm2chf`2Gj8B@L1Dc-y9AbTKe<_fDn%^@_Pw0g(`fH|$u$J%S z|I{j%{}GS+bTQuAUlH4X$=}o1e{#H7Bz?Lpy^d!xF1?t4x!oU6`^%(<_CA+J`>vAu zrUYN?-6H9;xPbORePT|so%!RD{YNDyy^8Y0j%ScwtCnwCerCZhVvo{ZV|zWV^^@HX z0r))o|KbAZbDHC|Ri6Bk{IcyF3gR*OX?)9HS*LYC(l`B?@?7LCF_7PglT5L{Ck?-5 z_WI}{{F9dYuCnyH&DVb=UXNGv1|Q|^XMdCOf}Th}>9!xrTe`1P(i3^eu#*ybBkd{t z(%G#Q7WyONkzOzHMZV~;miIfMA0+=oC+$&dk7sW4@xO-G^xg^jAq;wCW&A~Yke`mF zWc|8IGfiwo#KgnwGgmyCgrIemRx4i)~d zDv#DzRrMo&xwoo3G4v;GAlUYe_i?-{>z9DX`cstg%V)nW@;Q^MDZdlXaQtq^!`^uc zQ(j}AhoWr(mOtk73cZ1%Nc*)P#&YdXgZVqb#fU$;(AL3JcIZN zJtU;*v$y7akc18WMgaE+<7IYAUogz}K8*7TAbF7bQ_mkp-1$TD=V6_`K3}w+_PrAN z?pYLhSUL>)phz;l{M;7ChiP9gG9L4Pb@PvhG5Wy#q|g)T2eBS3C1iV$zYk-25%?mX zAaBqZkMfE2!ThFTeFPutvuHny|8loq?R@S_4cGJkMYR9Ui)`Np z?H9y+vD;q4A835rp1Qr{Jejm#a+{&xOH>~9JC=9eH}FkgZRX|o>>@&)#v@P|#G>r3-A zpw0Rb);HTYza{&HD4&eICh3E|z&Goc-u%@2WywU(Pr3WCeCtyuqyfBmazxeomavG9 zueYCYd<5SQxhV?Yl0ueoa3VHKMOz)F= zy)yi-MoyL7sz?u<$eH zclnTipH-Y+mEI@eL1{m|zT#UCWAVN9FpOy=0sN8m*Qgi9{?h1Do96F_ir`nyN4Dwt zP?vv`q=#OAmG!Wc=5Kes)i)o&`5(JqUg*Cb;YgUKs-2(UdY+t>VVTgrBTgzsaUt^q zBG2^xW^a8gB^#gK1AxH`CGPXiaZ5>H~G2A&p%v#j@kaO0zZdiK7NLN zh}814oPS@LpSAq(-J&o~ql=ju4Z~d((3f$T!`5JlNDfWr1SBN|x z@YcUDUvP9O+bQP_TzQ@Dl=BD-k8*u1+Nr(;mBH}N0PNRpv1jc1$MC_N$bSio{BM=z zAT9rcz)$MwJ~#bTx0fH|<2f%(eUa#P^M97{v78rXd`q`mp3xA(Z8HC?(C@6A6-faM*RUj*WtAL=8GSMzR>fx zx_zjh`nA4d`eJYOiN2Njb+M0n`k0lcUvgggM89-(as8!}^T(oJbo$+HSg+^E{43%| zupY+pdHSYD<7@i>_3Kx%e!)yf2WUTKn7-97S*Ig{meu+Q{82jA(}U?EW#?-%<=%dm z9{PMo50kR`1N7=5yqzD8EcNO2LbHB`{5rJ#s^|A(SU=HJ(JVx zWi~y#{@3Bnr|b0*+V7};7W9|$+x3En^Fr@D^`X$OTf%*!A9edk{%jxE4=fMY>+l%+ zs-s)<9qD%(^l_8nX5Bop#JHw0fnx2_*hu1!T4twSiJ>M_ue<&~3J7v8@>z`b&76Ih-zbxC^ z>u&=dze)LYdS3hXt0I%*BmFVk!Su6kdk#qcc73np==PQJNcnVq2Mqj(lg(DQuWx-^ z+P6Q;{^@JqE`}HBKMCn!J+&zIs9CQGe*#epl7Lqtd?ptpDPI<`=YQ zpKecWPiD$}S+~8!e(USg{9KXs$;@bNd+PN*UwfMIlukeE40z)q()Uzt`=wX0eWX2n z?P=)`{uYX$o@(SB>dW$!GoE~t z{PlXC*vAO7y=dQi<>?W8e3JEWj1RKR-_CD){SoV(Tn{a`%X+ize~8b!^k)8_^sUDu zjQDQmXWJX$qK4P&b1|n~`a3{Go*~}whvbL)bjkX)v_JT@bV{$cOa7=&4EjOjKk=FW zWBUAdw;3&k1*Oyllod(4VayKb@BR z2grYTWIO?U!+$P6n)*YYOZ#+-KLlGo?7!$^``YnR%xP_P`4jcyej6N@)Z<6whx9c+ zay?Y)ACdHtgw!vSYi)J;9r2hC!hjwA0yewrf1Na^jJ?M6%y~IezK6BF7BTe0`F1%v z=*lZ6EbIM350nS{dD}Svw&`L2pz$Xq`X-G1yPPxv|0DNT$a)RJH$qvR{Wfc;*AE-)FN|^hMaql(F+UY^*8|0# zGv+VtH?bG>e6ZMeSYN_^SXnQ!`6Zo%_^VPy*rQKx;e4<7ulZrk_tvh@)X)DK=`jqV zL0;CMH`nuX(tdjVO!k-A`N`V(&tN%*hQ&i6A@kqZeuwobp^u*bi;~_F7W(S-O`)gI zw|0FJ`M=3)PtaG-2VEe2>+8?L;#sl&I@(u%y*|kOM-A7H5;gV5Y#Hk7%_oVwCwKjo z^cvY3xkl&({>FKGkgo!!Mfy){fxkLV2S4`5QVP(ki1`k@{{&`*g}(H^^19dkzbDQ7 zNtpAyz&~Q_8JzzX|7^?l{5Sk>$^5W7gY?1g87ImEo)v=+`#l0dg{QM9_R;DCZ2!vR z{6zT_%yIU)azx4tc}xFJ-`SYCMh1D%pA0|we?Rw!vI=X$ue&ULjf9ckP+8pTO@k^j ze@pmJe_`z(`QcOl=VlFW@CTQVRK=G+@rkNKE7kN2royy`Jkgf_s7M6_fZr9aKN)U+||0f`0#$3V>b+uXb7tyx@bMA!q_>lxgC_m3K9Dc) zPdekJpEC3Xh?J*={`)F|AYqxm)nTO1`nmJV5})w!+1`%e3p)EV!VwoB(_nv{}%cW3;yp1zVkEUuLSUM_k0eZpugz)9IP@3g5%?`-<<&d}P7Ig1+=2Kr7;izp4^3+R6dptQZ~RlpUP)fBFS+=Siau4@ zTnj-fVy{Eq(mzYZ!hcP-pOl^S#{3KPv9=efk0GB>J}g*4K$`g&&i~I_e5_{yUhf~E zeL(qe5a+WfZ~379STB?HC4?e8e~>-K@>E=ZO{)(gJIZ4g0{Hazv%moX*O#9aegu2L zelYrl>(Aph9_43$VZ1uU@=7=?;m?99pcm_Vp2?Z@Y8U=#z%w4{EmX&Ad>aq^lm2*H zU;0-g|Cd=kwx81a#M1kD%!jajP@js$uc8yIsUYZyaPTn11?(5GFA&!9+u9#meoKqf zK7l^+=xgi`EzjoxW9*M84pbWZ1MQtJ$2|X=2pj)qetRV9+9Qyk^j{_VfcC&F(0bwyW@X~aH>=m8et5f6P6m+G_EW4^rTh=N8W!?ZUN zVcH+)CT^JahYbVXtp5WZ^?|WtK>SNdSm0ygFjvAz5B8fVJVUQI><96`W8>jZQR2nO zC4Ye5nV)nr;=>;kf|@@OM!eW3!o!lE*r$>l+BY1t=een$Fh36ezU0S>0HFM^S0b*x zBIS?C&`;u@#(dN<_@{kgdj!e(?U882Z!g5Gz3ti~Nhzf0Z{ z9-;k2c-lYGKU>TM8NU4h_DHRMT+RC$-T+|-D}(7Zuuo9n7qm~%6jk;KQbKrT`$WTA z`{O-z_KC)~@nEczC>qj~^PXb=vHs(v*9F)oF8y5lg!#RYeZu&QvrkxFK8@@XTOP4T zU3+hnqCJB4sI^xvwS7`MUwNs+FTVd%BnupygFZ4Js;xV<2I0NF`O!gyp}+0<0a*{x z^P`IAdBuJPy-J5O8J<7T^M~uXe<7o`BYvgx0|z^oX&}q`3;M4;KPX|yJIG%-FX^|> zD>4Am{Fp5VJ2{u-?GM%KA$C0{i1p@W#M}8A%#={K?fFID_#l5stQ)09kjEg_-+&K>t=dm|0%6;K-1%n0lb-P3PTl3!pXVhOOn%%y za2Ur^Kwmlk0Q!I*Wj{EN`&%ogMa zcOuAN)+f&uRK^KLWj)jS&#Wu2uY9Mad@>$bxqL=ky6X$o<->Ys2gz&4_beaj*^Zw= z`P}<+480rBPxh-w`JX9dO68#JSCcTx*MOccYk5ltvR?Z_KcGKkNnihdo=3&N)BIdp zzs%uKhwy{=zf5cYHslAR&s~4c=e0iiFzi3lYf}^u=#St>SwC{uyAFqhpEZ5pKl3og z$0(1X7x_!h=Va}Eh{Jsmm;X?Ho=>ZY$k63=^Z$y?KeDNS{Bd4Q^2dIWPHZRA^EK!{ znR^p`!tZCXYz+Gg{M$!;D*0hO_2`muKu`Yo81|*y+n1<@$MTW~kNrTsK}7(Bk)cXVkWD70dH3I1g7T_X;Fab7S@dm(eLo{wC925wV9ucJ#jre?-#LBx-U zeYyP1k%ZO|uVbvLDpqA6@Fv<~+-Dj^z!D!xlZcc z1$s#Qa8MO(`Yd0}v={0x@fZ~Famyd~_)E$+5>zD{&-4>Azb*6^e6hC;KI2=w_Lcax zK{aaehY;WDwV%ZM&T|SqFAu6Q3(xYl8UC^L@yXM@EV3C1{=@)cxjJ5f`3I&UG2*c@|^4<@*MUe^lu0EuZe$L;V&OQ zFpTy6d4b1yUg*CJ&M$-*e&&dr&mD4dxje$651!b9@m)_(FT!LI^~8e=uR=|<&jRc( zLwNO|7Y+`Y{SmSU0_8!tP(-}!SBhhN{SePbSIGUN{!J9AWA7 zl72YM=8NGc^BeHr&#@GF-h}0TA}`#YzQV?%e^LHEDh|5!`hMnc*wYWBFV;heEApQ6 zkAOc-SXjbjlA~vPwCHHf-U_22<4UK|F2v5_&W2U!;{V!tVjjgYl-g5Kv(F^`oL|<9?IhX>z zk6uz9Uw;}UJ(8aM9PLBcOlL8D*{_S^F;+f~UXFOP{3r3mhrH&7ASn;>!+KB=LFh;A zzucUM2u0rqeN^!7hoC-}{@Bq$0!#k|KUn{D;@{^S_W$|$1@J+~w;l`lA?!C|U-0^C zXAk!i$sSwjuc;J^ufy!GsTTHc_6gk|qEYs5UpUPM6D217DZ&LrU${Na>QV=Aya#(r z`p07TqMonEcz2!#rrRH~9{-`dwf#ZpPjJ#t>}kkHax}x|upV`9ILjdX4;g;S>g`I5 zhrJE{=+OGe@^94W56i!pKPQ0+zlQcm%;*oxPiSAuzdIQpui;-GKS+5J9{<+rkG13n zpS(wULryYFdV1}XjF8q^UQwPVWxY~|**@(F(p$o3(!r|n%pk`XLC@X=KgjuSjQ=vU zCo#VwVcM?}mhpo4my+{fSU=)?zm(+y=lg5N&-LS9c0N25;xgW)cEPh6KM70jXbXh* za{S9kJX9h(IsV1?7LOj^%K6Al_4v2Fo#S7r-{;no|Jq+@{F^@{gIRa{SUT8MGalCW zIrzrIvj5r}Z@0HHeesv`7X?{w8kZ#9@i81?fj>{`^&)#egf~9UzUcTkoTh!?0D8+Uc_7b%8qBjp;rGi9M9&Yb+tZZc`tT6+t1u|d9^*} zjfc>lnR~}ZHNEY47XJS%ydH0Q_1#JP@rui|{#~}?*)b2F_S~x4@$3~YJopdu2cy;L zulDRQ8L#$w?JxRvaLk2oIG)9L)7on&AMA0D{(`@ToUiLI<*yyjV*F?AIf<_w&tg1U zwD^t2v)F{ehxUe)&)TOPKXSZ{@p8$=HyY3S>}AQnc04=k8_yEIc0B8okJ4Tk|B_-- z9>V*_vuq!qJlf0lc(L)U&;Av7*w0ntS)ctb@z@`=!Q{VJ&X;-gk@(v2Y&J^+z&D=;FC2}oMcA{)BM~;A&mNCcAg~W<`#mE2 z=e)3LyTZn!e>E7-divpsA80)5^`D00Szmidf5!PpKIFGYD6cI4TYDVm9Vs9f|5e8O z#%F@x<;fS3r*-35uRO3f{_T%vp|98>NUvAKcs84*fL(b!>*E*n-v;AZIe&>HjJ@{K z$FuC;UjK={$aq%Oj%W4y!HVNqpFRP<){kc~-sB_vFaC8sey4qUuWx)R`l!vbHwLMH zedANn$G-8X^beoB{xzoO8;?qS+OsEiGCg1ak^Fr6LCWKs|B-l~e3$X8e|{DAwBC;+ zRn+#hJ};x?59MhjP62P^5#*^#%K0v0U_4vp?*;WM9fmz=#|Mx{SRV=Ue1_fM#qp|S zc8SKTwd2=|4Zq<2`$6nS<9>LIAMN^R5dJlnSs@Yy?Ug?yl+xp4!1Mkw>LcktXb*S% zzXl(7%KB%xh*!~v60gbQjh_*TN5)@v{dj@!di_Y|2Lc#Ab3CcX&*DE$){jvh>Wgui zJjHkd_sP2JWrIi*<*V|S2>AQ%4Rf;#ADIuVub*f(T|bn+9sEN9zdVZj)1>?|ldInq}c|qu8`LTbebhxL7^8r#H`X4!r`N#$1PgwXX*F*XX zJm#zNhdA4XXElCE596s$nLp6}b{P)p@HfOCD#AV5Ki>h_Zxh6Q5pMg5{Ob3@NI&?9 z&tD?wgYlx2w^!&b^BG~5Z*k8zr2U6~Eu3@5k33KDt1^7i^5bFf&vBo>KaBr|KNQV# z{A2y|9sT)!&pyX^IDg2T)^yt|i3L2+n@}ikzTDTx=9Mt!i!x54kM@L>*Y5}aft+d+ z*AKCOspVeP&GuTH{6_Sr3n?+3n#2R{rs z9R=KAA7uGk zS95xcFo+LIdsIxk)UOS-;C&qukNiPD4!GR>(O&b2bLo*DqNo$2AN^VATQTsGUL03u zYH`T^{=gy7n;rp}p2R1Hu|-PQMbi`QSuyb_3HFPwBL5P`qbHIe{yEks(N9}I$CI9D z|E>5S46whM zc+eC4z|4s+{0;tDz`IMY%xdz6fk%1JA1Vf3=+Qpt@ek=m`h(kCdbJOE{KNPR@T=mB zYxKM}I$w$#Zh>dO<~y@lt2 zP72GQJUp)>`>VNs_v`k&4j7i>_+lRsFWWuA{a5_zH$ zWB!}pBb9gS&-~r!-*Mlz>`%1!O<{Z$I)Vhl&+@)& z_=hM**q@i}dKJzmus*uKqkM6kAE5pbgAMWk_rr^QY~&y00k1o&Y@*KM`YY@!@h2!O zBmF%p=S!u&5>D|z4;qv3)GzuxsRtjAQ+Ijskrt*c`s{Nx{R{Y2{R#1?Z;0$K{wELy zJ#gWcp$Fj6e?s3!yxjL?){koT=O!e6vmP(r0R3?S@TjAV2RDm%Q4=PAf;eYwe*=Fa zeeo|U{K5Xw0RDUp{Ie*Y<-D&QkHLS5^e3b)<4cq$Mqa^vp?bV#_GgK_1wY#UzDRk6 z^m*Qlj(f`pNsE#?Wit=%Nw>G^;-th{`!A~7HjOwd8TdbJ9~d+nEe&~DOgUbB2~1HzT*Oa^2O~<&A-7% zefDMPU`oz^N&i9mRrVypv^N*UB+&Z4bP(n_=xO_#wNFz<9#-i`)VJ0it+h9+>`~B< z`s@!d@2p zrD=b1zNKnEDC_~TKS7@#Rr|Aa5cVAA;>8|d*6ll2Wm9mDYl`tM`2%LfaJ4B z^?aw;i!#4aGvCSctdbw3Chg6GFgFqQ>{ac5M8dW9Cc;BzKd`oU#lH3CM~pq1c|m(r zk01T^Xzl#xOSMN&`0P<@f7|18``6==Y8?8H>>75ZF<%|t+F@8KK0Fyp02S! z5npAmYWwu1&W9TN^M^JcYV6My=R-~Wznl3`&;De8XwHXxJvU;cpep(m=+d+GC`3qJe{nGY@c@y+?khV_SjTy5+JPM_2M;C!g| z&;746A6mcvPpqjISbx{^mA>_NegEqU>+dgq|E}+R5%yQWK9&1??Rr-6F!tYzJwK22 zb@3mr{e7)4fB919kL&9%lKTbfulED^>i@&@{|$W$|8X*(DytltJ}=3?`X!nF-VOP- zLi?AR_dn_m#r@R6-_83UIaZ2u{vG4H5%^c+bhwQ7ApdZFym|j4=+V6Y5&Ih;-*}&k z?A-%Rn)g3qJni0pWA8I*-v1~I;_mu;^FCA3L$4>|JVEn5(@T7xY4iR^_kKm}S7_e< z*u4LdQ}5n=fzA6Ld4542%&(6)&HEp@pU>=Hgg$EC|H$(zqMyMZaokTS=Ox&Qn)g3~ zKdSHFY~KIay#JB%qj~=$=1VXhens8?i1On841GTf8}S0>Ctqy;?#0Hx(&v9cUp@bO zf%Ls3^S`@ZX#N-O)EDe{l>1Q3>0wciBJMWIw)yv0juZx1?qJg+8AZ#`&2v(-Z$<;7^>7Z(;fD`Hdc)_rjxs zI3I=hSUKXIM?g4pxHT;KF}>4FKb@BC77Rn5m&)-LrZ4Lk2xrT~t=v0e{p;ENH=h4E z`1i8)m-|5BzXIneB_8`nFyABhS4sJpzT@*>Z~XZ!%3)|t55Ny|{O|gwuY%X-Pl|Hb z$H14>u+aBb&mY^kY-aJ|IX~;Aml685xbrRD5>^a9t<#^k`fzdS>DZGWT)PyhN(kG`*)=Sx8U4#&0b<^1gk?z`i8 zLRsIF`RtVTKS+AWe0DO$@)`f-IG>V{`pA7~h)4Z9c=lJ$2a+EhmC^Lj^Lv?cM*R0l zeE=VGGICxRC~`igbS#-<`|10)(;48myX}wkvS@$OQ|=?y=_ko9_C8cEf4L72{M+HB z2YSVvq?8XR`u^`+Cd2$(d;XJ}eo3Jh`16#ecgEa@4E%OE+ic1kbGqB5y%-jL*6?YU zpW2h~7NGq_o_Ub{GvoG8!sC7dE~wc49V@52{)zII%2}b8`11q3h7TsQ>|f$PPtIG2 zy9nc-0?T{{;g1#l&5QT>XFV50dnfh%@yL_+*??Z%ett#!paOWbd;xskci;GyIQdyS zfB-)DDZ11{?*K}<{M(~_cabQkE41 zYxoxjAlRSn{IvG(i%)W2o8{NM;Sb>Vx%?{U9aF-8CO(Y#B*B1~UFr&pn9zx>Lp@jKFif zxrlgdS&;L(R^Bv}kCf;C>ipcH1s2yIF9|f z+2=2|x<43x*80rS2lj)=fAI$_{G1hiCG|o7cyWH#>50B4KbI8@BKA+uemE-v-`Edz z{66v`^1HiMes@!Tk5C?o{1$g9uKafU6VLmy{$+iCjnoI>OnE@$a|h)!^uzGM+(31D zxg211dgvdxUq)}&B|PwvzV;8!{SA_SXN|ro%6&BAp9%592YZXv>38+I>F-QP`mVj$ zFnymsqkp6FnhfDnZutk-RHxrL=%vs3_+&Trnb1r8`3?+Jhl|DP@R~Jl809&-)Ge(i z_L=y<=z-wCL-`@$e9;RJ0ls&Q8y-vqI-CLR+lL=d7oZRO^!ygXPi}$!8`Sc{)qjKH z`pdK5h6bzkUrG%Tm--Lq4Lh8EagWMR>X`=-E_&&Uel3YB8qu#iao*k7ciNt#eOE|| zejq=IKJ?pv)PF_Mk44H8#E(F~6p)p>f6M640au?6Xn2NC3w)CL*3+j0NfIz+?8o?0 zN!+JOeMi^Pp6FxzBJcAR{Z+_vKG5EmTRJ{4Kzpmt=*vtwRdVSA{s8^}>m&US_y`xY z{?+yz?Om3~(l1jkh<^3yzk+V?irABxxV9&)ekXltkIC~)I*9aFX?+cQo%YznKKl>h ztg~ua%1`~vc&Fge2jwd{LlTzuI9Gr^U2QsaMf^qB_7eJmo`4_fN9JPBAU&1`oS+|s z$xmz_X>Z|=lE)v2XZw+Vy!O-i7v1*L_E5=d?9_8XG+9CF*w$FHLUq<25zz4j}v zra6x$xsUhoGNa|UxGNFPsBx9>=j^!M6l zjmIz2{=WVt?N`$MoheBBjjAHwjXYp`F@3MS)=2r*nD(*uYsvJtl2g+0raggQ)Bj4z z64LYf->6)tqx+wh!yzkBUhf){Ogi1ONRw5I=+SpOB;PsW@6{)hHhQ{8^a zn)a*ie}IR>f27axIodnxbS+Ey-1hVJKj~k(J-zn2Oza^fVa8h{BSi$f_8M*2-opRV zUTf54fR_jMbXEHx9_7or^+CAgj7ocq*0k3ewvUGQx2@ z;WwmzSxtFUMb>A*%3sa@BELYNtdqTr{nw7yfN$z=$M;1qf52zTeV)8I&GO`OqHo#1 zb^T2H+VNbU89;;nLXVWBk1Q=cI;6atq0f+h$?0Cj`&_iWDfR7Qf5W5s@o6o8wLU<3 zHuUwm@=N2x0Vd}Ejr>J@a3Q4W4=69{yDas!{oz!f@So^Qln?orIDWMFF(;Lk{>c1s zA9c*h;(`}>q&^(aKX1qLl7D-O2Astge?;Q10{$uBPo#7pgp)G!CkW#{ogc$^Q$KpV`SGPx znDRe$Z2|Y;$$Xl~KRsW!1@+hRA;td0`zhh?<^=0!{JquQH+ysm_t)?-`&BdLwl=pt zXsm75s^pe zFIE0#;hzHjcLhP_TeZJf@xN>LBbOcsdF?~`zZ?6_L4WIS?dbigs=l8-Z1U$UCE649 z#eq1(ZwTZ5#7(3}xEuEuLf^u_sH6{nHz~ryU#QHUs+$1xY>mG?@Qe7vLKOPD%wGdv z9L~WXZi@c6G9L|U|FZi2L-oLiO?oIZ(|7ad{fWr`fe)MeMi>uzBtBryE;9Wy1$FG_ z{pCs5`TujvpVj%3zZYzKq5W$80m5G?$)PV)llG~h??JQ&_UnhpZ{TnGKNf<>e*J=a zsr+%V>pL%oU(bJ1yBFC1uj76D|79N8@>?yvN(YqJ} z2dz(<>mTSJ&Gioup}GEn^{eLk2kk37VeQ}VV%PJU>mRh&S6mRDQ{sDfF`Tc)i>mOV{`}ey3fi*3?p4nXgFzYWbzWxzwu7A9^|NQ3q$1C~# z2i6mD9%iwF2BcX}Y@YvUp8v4+0rn3x&wn(}e{lY{dHw_T6ZTIu&wpI(d8X$15AMIj z(>(vtJpY09+UEL)^oHj8NAvs#^lS6{NAvuL(T`Q~TXNgrX{ zZy@kteLfui$|5c9`7-q1OgSp&-F^NrUg7-Qynmp1{{T7w{UPmyA3vvg{{USlTK|Am z{t02vM3(xLIN&n=qoBXEe+qa0E#K*`=ZU`WkfAP;4Y)$PU~ zR1E$(36DqnLwYO{A-whPA9Ff8>F)=R@h{ZgOZlhh?}_sCX)S*xeJy_@^e-gxpYcw( zvlU@u55oRy&ARd%@)P+F70yoMjxO)-LyZ&A2ZV{cUP4!>!ka5L#Xwy6mwcz z_4(0?q?aiNgRcJ|?c<2?7nOCQ%Thk;56KC)b3R$HB@f|C9JD zG1Eh4y#n)xQThut_`Ki2+&_C^^kuVL%)`Fp&bRv#nm6yCGS0nh6yg03`$i-D?-;!Q zQSw8}EzMOQs7v(%P1R?0a$bkevh(60%&qvb@9{A*RJ|Md)Ru|yCw_u|0cQ@U>rg$c z@y(}QhFKu||JXJiI1tnS+d7)n9TyT!$)^r~Y`2au@j{~?$ER9+ao*Pe|Cqt&vI!sN z&nng8e^yt%F8{L)@NaH_U(v*`!{4w6z<4V1SLc5V-fQ#Up$Sum|Cqtw>cjuZ2KcLV zf_3=uJ2c_}AO4mG_~%!^zt`X|`0!bl+VVfHg?C;4UpDyu^1r$P{(Gn_8noZoUiq&4 z_~hT({BPF$RhR$04e(#n0KcF+L>>N<2LFUEhVaj88{l_SSv4sC{(bUYE$IIQ|8;n; zE&u0Mz<<)<`|JPu2Ka|{|E?>4|66p%rW%s}Pc^{bOaauO{0}t1XJ4u<|A3x6tIPlI z4gN!tyypL(Zh*g*#!`d)D|gCw<;REGsLMamCUAB6ckb8t75$&o|BZOBE&o$1;6G&W z{q&S~ZClec#|M>>^6p6L?J2Q>(ciko5l|TPCH^5)3jnlgNKWXq6bTU%^?G5ms zTLFLUZjI>2=X-7WzoPX+UH!@tRV+3LgJ)d2rpH2xd3|8axw&z~x#w){`%!A4#FuYIee zTP6Rf5^C|^sTFx0{>Kdd@91QN{uIfz_*?a4PaXbkPJRC5#ajHdOm5Y;-y;V9gic2C zr%0>C|NaX2H!W$z1s^^|N-h4+!+y~s)Bu|Q9x?bQw0@TSi5oZZ8T0>R_@5?#0W)8f zy6K>P)#P_+)te{X&MKiMGv{|JIK!hh@?8vkBj{*COH+hBh* z!k<2(@ykB^tqt-YhW*h9|JN%T-%tNnH^Be974VDq)#Ep^Uw$3-OQZZBy1yR3k^S;t z!O)HH`|01vei=wM#_vB?U;ak+%U`d6|Ni&X=ikVF z`8C)tjmqEhv-S9m?3YipG{!&nUXAbP|3>!9FT;Lmlz--Z_4tkKmpj`U;~#%e<3HeQ zzee`UOu8|C|IgLezmfg&{T1+!f1n<}k^S;n2+~IN?=*&+pMM(JFYjw_jQ{=**5}{I ze%YICjNkGL_4o_;R4adPhyBs0{Kt-Ke1HGh(ExuG_D3W9;zu;Tzy3QL;NJ!NqY?f` ze^KK<#7OfrvR{T^e>B1$|0Rv&mtVW^skZ)mV1G2i|JW~Ue1HCp?3aEh$VT`ZPSoeW zr$PSzumb+C|4M!SjqI0?!G39!|NO7k=fAf>{yzu%qY?h^d`#o}+rN?h@_N`Gjqq>& zHI46Y|3>!9B*2IdHB$XNVuI)&e4u6(yCJXd0KFlBTf2hKM@7aKeM)|8R86ZaDAwJ}PyuyI* z`TLddzifa|K0d_14cj^@4EUat=nxI_|7#6Y@#PQsUzh*;jr~zu|H}V1nfUM_|LgGI z1jI)9SI*YsL;goA=Cl2;i1uxW|CM@t$p56q!b^U+8u^(#jE3c}@-_rnp z+Sm`Z`0A#5{5u-pKfeNgrK?WG(-wqXyq6 zKQX0LCqG_o?2o$q@2}7Q?gsfkZ0wJ^{6A8k|2+-z|8ZkK)aCzCjc+ecW&IZ$Lr$PJwg|R z=g$mc(fbPhQD^@>hw{B3J|oQo`B8`eb+q3Piod4;{vpus2gTpp0H1V&3RWk-YU+!3 z|MQ3m*#DUqA3_t)&hzk_|EZY%YybD-Nn2U^`jPzaMIG?)0)eggl$+PzJMceW!{~WG zp+Cvr_M$I<*V#LDy>6=hjQeu~>f8P64(@iA_T6)LadzEwa@~O)&KU?M#D3jy(z5ysx_oI}O!ZRo%LCX2&+uAXjJnVs!w*13ANU40`kJ6#-hc3nL&am`i9WODzm zcd5a3Z_BAIsNx&P1Kn3_yJ}t)Cg)YIFfori*{1N8M=Ts7IZdtCCB* z4=k>`V|P)#HZi<7F*dBe)iHThavtx+oz9Np=FQ{V5l-HI@W76B2ku#S$NqbE?%1CV z0R1z&CXn&l_OJVyeY&O{?H*gDM|XYt+EDK9!y9Q-ibHwyI#48k_>qrr|vk?^coajr&!8>fPYN zaHJ2c8^|5luiD=@s50ALudba!xi$v=%auuI_kpXD??{e@1I4N7Wbn6kI`{0@b^DH` z1G~F}v0vJ{?=|Dw@o(R&!s+YP>b6_M>6_KM!o9bC8cn<|`F@jF%UHKD@fohsIWY7I_qds{Nn5elAH z*_r8J@|tAF#P6&0>k&KsdeuMmdKLI^Ft}R9eoD2>g*ql*7eHGD6k071&dne+{Sdai z&OD@g5g48w+Z-63nR$I+?Z&BI6~9a6W`8P>zDor<=I%oMw!I}BOoUWXh3{2;8~=MC zH@7X2eiLFhyh-)Xy-9_)1-dtEQ-Mc=ExQ5`znxcX41qqok>7^h$ZzBBKw-m2k>AFT zsyqT}HWPkz;IbLK_RjUH*j;LL641y{b0)=iEl0z)%nt5xJ3DtRg}Iytc?kVg8U zzinC1_wbt8$!#jVCy*JQxbnG3``b>0BmWQ#o{RL(o{Iz$YGCpsf%TL46B?PElGiiP z=YJbo%h2%j*t-Lxvv^xOJ0q3tnf?g#4tC7mrTV8~W?!ETN?k9X82?D1X9ls`6wZK# zu9%$JriLJgRTSA@8&ajY@l;^reWbj-idc3Uh92 zHjIB!^>6qEHMLYn`)_;vCIYy=us;oWNehKU=PAnt^)rn6#Thh_&KZj7y)+13jI6~Am^=DNt> ztX4zQ6M<;>Z`2htS8fdCmNo{uXRi!I&P98tuMBieM*|&@mEpfkkKVoeFVme<6RK-w zLa9GVxBO2PEOeyT2lMIQ%SJki!&Q(UUxf_V z98DGGPO8w-_8sMYOI!Ec*Ky6hefyrurLR)Cwl_!78HNiJ*Qxadu=4sneVz)5$rF^P$`b+ufe~UwTm4oUZyVCz(SEw-Ynq;VZ;;UWB zFXz=ly!5u6eW}vq*f;YVovEqIRmapX=1V)uV`|MbbXej&LJ9 z?R=nr`iqHIO>Lj~rQYB#s_=Wlea{H7O%vm@wC|ue10AsM`ZxSyZwFZ7ipi_K-HYq%12-m<&$nl%ru))Uy^+G~WN#w< zy?kU~?$`3Y8^4#2y|yF2;Wet|BvjV7zyRalZ0X*xr8|`it(lq5g%X3|H8V3~z3Vs5 zAJ$FidnbG@PTPj;%2nc3^|vGsxBxv_IekE6!KAw*kHXhHn#O}9jwSIQuL4JBX7cWd~r>dIzV7_&GpfGn>1>#SI z0#^s3I|BWazmbnTmv5Kuj{-wNg@KA=WsXJ*x=+3Bl7 zz2SAUGojw}hw@#iQ16<#vEjhF+1an<)Bic2ElmICd@}u=VBqq(@$&!A-k--eRps&j z__<5gCb?~zv`xAsKvyU`Kjc|3ZbH}~~E@3Y)<&pr2? zb5o*CjYpe4lvCySoI~L@A;sq<7T-)R)93mkrJ%oa8WQd~Xo;J~sjE~4vZvNU#Jf_o z*C>RCEGCD^qDJxfuHvMrgcBd&^dF~_k1Sff)#@$7J$xOOiGF6`EL{Koa;-MWm1l{& zn#=J!uI4mH)j59WT#HmF5?+;(?mw={l><`{rogJ!j?BpRyZTrQ{PG4|Y0jxcmF(`F zqTOsom9!JZe4??X7WlnzulUI#PT}+lZL;%6i*5>+?4QDE4kC`gbG*eLr^*aCcG(O1 zyQW(d1p&u2i$c1}LX>a!R!p-fr>15HhP^88UR$!Tb=pVspUn#_RrJ3EQvl(sm9 z7A?!8R^71Wm35C=lMVfe;ZZB@V|KuR+u5Vm#fBY3nw-RiEwqMV91f-%MpL91N-0(q zmX!EPaQk`GYPKO23+cur;dX)}OB3;h9!N=WH%ZcBt4ey^I!g7ZmAsmOZG!u#6;?WG z5lD2GE9g$hixN$EDx~@Jh{4c8-6t4Ow;#0{(E#Hqg0{G{Nv27T(IUf`mo~ZL2RM<- z3?E8pk=5`RkHa=J&l8CBm^CK@4RDBZh}juvYVkcg1CJsh`WXEox@^wJkY3s9x^|Lh zp40c3mE$LMuv)zRNV@lNdhy6413M=4d8Eg!saDV9R-6BEtAaQ3VgAvQ7SMCL-zDpB zP%A5w;gtsDuhQt>w;0_!G&au>s~OeBOH^-Z(($Fh={jOfqKA{L*d*dF4S)R8c$W5X zv=uyJO~nD5^MVy!EqY zrMCGiT}1d4kE($uTwI2;jKnot`a1)UCFVO_qg?}?PS;G-Puem)$)jZw=2&c==d3E5 zyOKjoCO-$?-&mw93ukhe;d&`trYXqxW9w`!s9yr01+j!da0+y@zMqXZLYv7oW2#ll@*q=fl*X$(4k7rn=QqrwWk- z5Xob%YzG>=0&id}4wn&9;B$_3E3$p^QkTtlkv4p;I%0UvDh=fn1wJ?Y2^=rIVBr+! zEr^KcGl(I{o8p^>r&*_4ZSxIqa|$Wc()qhuI>VQ)B!|-~Hn=rcTWn6+6dA3;B#x%k z=6^xW75H87oaDxXzu$FMRuuT%pU4VR{>QR@8EV)vtLix`#svyI+QWTvT~0K8f%$G# zx~l_ipTEkQ>G$4}s@zV}opg4_thx84WdXZZvS>9E*4$RsVnj%^N&(B;34lxCo!Kns83( zc}2^9zL(i|K-8CWikAa%<(zuZXgvHsZ&eNOmS=N#Bz3RItL1A#Rd7XrVdB>9rBC#dLR)*pb{+Q3$qAt& zQfQkO+UAC~^)zUj&AT>rA>D-A!nLVO6Y+%}NTHk6A=3=H8%Xq`<-?AdZo)0!+SK{< zOW1jK$aVmlxwP(fPg;vq(uj3NrFVJyXVn}6n#C&>N;?9Zh82K)m;?Dsbm z)1XP{cl&;A*a^-d<bvIeH5B;qe5Co(x$M85_nhqhDcHd(-UftzN4mY~KI z*l7=moZ(I*4vwW?^g-U`UR1qkIR;bLFI}@}9)<;9OETfvg_GV%itoBp>c2_C1G)5b z(rQBmCn=M23`;r52(0F$jO1*?Mo#LR+-P%d*?g27BvQZRnKtK5oArC-z?ub>)s?Fg zd8OekFS-UM!*-DzAkq+$if6`+_|2s1RB0nINcczPHvcNr-cekd`dvY}nwKVVY0|T( zq$MOnx*?TFBZ*m>E8b{xX85w{CQVT^P11_EQW`{(gL1Z=@;zNKMNRAm zG`WfDUWOUrZz#${^{qBnoVZ3xQcs{e7HLCHvcZd)(;N*RLbzcXqL+ToI$ zBp8;6(qKGzItQc77t;M?n+y9kH{E!W3pxG!B8IY^{}KoI0w3UkE@!kxx|0|`9O5!6sk6(z4;nG0K%nZ3{#2NC8{{dr830z;rnD&{k&_iXP*4GK$5%`H~qkm!+8EF8+meI=E~JZ1zxT1QzL&r zOtpgaarjpd^|n2|JoEYvlTvY4_4(4&I{hcAJcoal8w(ePc7-&VG5CJ$Yntk@PP`s>VM&+4aGCi*qI6k7`o=%5lO+!-bCZ(y6f| z(S79VbVoZ;^Md9>jF)j7#^q|fAL2OF<&kkUoAI%@{(~ zhSF*dEWq{P`3ljv9SevaJMSo{lP-&7d9@&YO?B5AL603=43WC>Z2{8>-{2YD{f?jq zw~1lBAggMxh+zH&C=HhviBOyt(%s*3oIAq9O_Pb@U16Bh>xE$UyeIVc&{T#>v?~pY z_h8`*z#_}zK`OEHWP}ftx#E3c66Wua1x|5BNT<$@ZR|=X3e@a9BM8Q6`d&owudu$Y zUzbjc6NiV%P9KfKixQ)8{FoPQ@c}Gc4hUxN2SPeEeWLIGTDV=*hmV#fc6`wm$?~^K ziF)vTAoTacrjI-7d?;ufUVq2~ii~=2>Z9UNKA{@schcfi9#v8O&F&Zs9|&1~C(dYq z7PHxpI0C;?njZ+0Tn--u4$<`wz*o3_c9%C&;gUl+g^wW^R9Z;uXCK0kC1f2P`pydA zkj-^a!)FJa3yJ zJSX~=No7~#?xD+%Bj}6ebZAkK;7{DjYO7c4mg7MC{LCf&?x7sM=di<-6IV;yK zcGyT60~n~Nv=0(i=!SdZ5}A{&!k9kCG-Jydc7bN`m&sk zsxs-@6p|&E-(%q4#}iM&fVg{I^DAo?ELpU0-fNO%NF0EBy~SW2fLmXV4bu)zgN5$f zZ7%HF+;roi8}C{Rm+p9MPr7$9?!cLaX5p02Wg-)Hm=@<^2yK5e5vS?Np`ULC8x z!HPO_y;Z}TGSmIPgT-S*HKrlT+wHkd|Lrr-YSO8c4Md|Ipq6f)F66r$+ol_xRh7~$ z(+R$8pWe&q^`%PRPs3bf#jqq_K9NkxbarkbFV0Yj$wiuEF2z;PrTO>Mxhho0I|tk3 zEJ0~x0@bqA zB+rK$n|t@Ibc#1^n5djOR67pcza<=BNpyWkvVYtpX$7aVc~@{sVeIYF>FHvD(|gl& zqB)+-zdK#Of=l+D9tIOm^=UGEY@b2YD>#d91rof1i!%iC!nqcevq~fM)bAT^a1c0-j^uD2X$9b)4bm z;o{djG0O4vPV~y|+DViftctI9>f^J;v4iy0l6JIJ{0ry6ME?fD|ELiEI0%CeKTqSt zt-GY-0%odQ|DC4)W48EkS%&*is`eTy|E*PeU?;|dHesmJ``}LfCSvw&B8C@JHFsEf zO{#QmFmDS`F|dEIZWGCJIX2FCSD^ddP+oYQWA5Z<{OKqDm0{KHcW~Y?`9L z^q``@><`Powuco2-$(}KGr1PWO$i!he6}+`E+Ng=Xy9LzROjO)WqgiJR>ogz_Vv}L z`jqiWt`99Xmoi>SH+^k>_0{5= zZz~s<9B7*WMXPcr0e{Pl!)Taz2|co#ezEkAGRb%?T5mZ$dn}mJ{OZ zZGrLe!R-n;UU@$kcb6cIix*A4$5rZa@xs?DgnI;y(90(OA}Yn3rX}UqZxxGv%`YVHvF7`1%3HQiMp^>;cdhG1;} zfqFr*4B-vqIBC`J^bB7Kkya14XShNfU|7WA>#9_F&Jac4zh?L!Ijk!mVv|oB;%u%> zLo$`y$7ITmIBl_`%2l!>jy$AvtlqdZ6U$Xdie|o3`cf&n991J;ixa?xJ#V8<~rHx?cj1N7uL!A0)iC|66Qzv>9TSZ8JJb$wv#+Le-y#v5=_>~{76#n zs46P)*OP2#0ILA%$q$@XvwA_zfXbSh)hp`;RMl?0DURS31Upg9C+a(8tRBg6dDh2S z+*Z13$g9>oA27PIg=Hh$BZilFAB^WSxlKh*e~F`H$(s~(_6O&20dmBme*Km1m-z+R=Np@)ukZg}*J7;nK$fK=wlRS^ZiX}9^lk+@I z?6{u7j=Y2r4nSp5HfI@wVor?{R)zTwg0)pC5g%>O|0=+`7yr<{jrK7w1^HX{BsN28YcBB#ZD z1D>j^MC0ajyuQJ5iuV%*U({dUEGBvm-E_aB%I7Qb`rXAPzY}u2{)dzU0**bR_?}bo zulRm7p81@Hmo*o7vA3*S<4lpAM104o#cj@oyXXO@X6KMp-)BZ`1rh${RAIZG%UQj` zD2q>H&CU^mNqe0Ww?rq_w4@XD`_2^4yGpFqnTWv}Ge)uBz02wJ2V{@S<$Z9qh$_+F z`7b4JDYrN`@R^wVk|*Ix@x8RB?^uAJu|KHODw zLd_`?W!1v%M$cBI*eXb+GMDV0CTl+7xbGgwsw#R%&<;UTh7d)btU5{*V`TMFGIUX$ zd{(IxWmLxDGCz-~7x7q@q!}lxHT;<&v&|XctNe{?^4(iY%C&eOs_>VM=0(Utgb8 zbx%P-m3$t94L4&-@S4=BeQJUsrN0&MFBXWZYg7vaWxhP5ZsqFFwjb*+JKa9F-{p1t z$D)V0B-Q2N_~%dq&yfB#&Ij;ddxGGt6^H6ZoyFU~)$O0>wqh;Lz$%`9HbH;?TK>VI zeFDDyBw_yhM&ZDYWOv*q+l5 z9G}i{*C%Ms5R>l=7InpsFs-dcPpls&DpFGUVZ<;|%nNw`u1s2iiNQ21StHu#x!#Vd zle@D^N=n^W6j+arO+6lmckA-1+;rzNw{+b)-s)e>sc&3It}~e2Usev`IAspkFEH?X z+BKywgKZ(n&MhQOE+uBiO{AYZutntO>TyHU6Mi?JE6Y`!@IB~0D+o`e+vFd34I1?y zc%6E|D6UQl)X1J((GtkWP$qDgu+&!R4ZC=B9Ti+TW_G>nIHe-~M&-ANP9De;Qpefk z)jVgeo`@r^Zv;JYsI?2Q7>lT`63udAo@RtXJ($aL=I0Wwze4X6vt(x*nvUswhVMS2 zU&isfC2e0`WfAykSlyu{x&$%Jv4s;7!~!`W-ONchOjPypmn#E`*D=?soSjI9BudKJ z8T59goE^Gdg~t_@vqR&GyUkPMMw{vFDQ>nUL&j|>Zg%+g6gPW-v8&YZfwvrM>!th6L)3R-HB3^PcBi?(>Dz^$55wneX{Wijw8NYzZKntRZ3}ESr_v2> za+1XFFx%uE<~v}&)2vPn>@;Jn*KETdCp_QJ=G$$)O4?-}%BHheS!$$naX7rYJ(&ZaF&A9sJL7dh31IoKZK!DyJ7xAwCLtPWVCs{48YX! zbYGMB$R5K5A|*d$F?3SIqZg2DlaDQ4KS_4XbH8zQS(4|CtK&v0gike6mPs&sGf08E zs9uMJ#}}_1b|- z)hxpMmCHst9j-vh3-h?ms+~n69UhNE_SF)_cAsL|TMa3ol5yc}$jGQYZ zdxjA0Mm*u`H;u%tBjCaJ@(bPMH z2p@6=8?Vftt8#LAvcI1>8Mi{6lT&kZg%zCU0?BsCMTzJbKHwE{ocX15QwQDR<#$Zc ze}W|t4WiX`)!?|h)zT`$X$z8!S`3d5wASdg8nw;+mRcd0la21TURQpl;x>_t8s(p* z`8C>1fB7>>{2}!0ztGKdFIu&>Qh$Al!HJoO8if?VLL_H^WEmEu8v;c7cYTu0HMVA` z!H;p|nsmc(PAUX2Eivy3=@zxk@u5sxKo9Aw|LOlAIH1K{*?Y7?(hvm98 zm_E#ZPms2xEAgONqw4P~U#n9#XKELT>PHOHBWhyEEi}3Jp{~_n=Kbs@eQ%d{2rpS1 z^9{Hv(;6*!;ttrH6UjiEOgApNabsg>o(?I`=Ec6$w-JwpeH#l6FY{7pB`>*w^StB*I(W$sCV-T&DJg9L7wu2joV>_fVxvwk_Z8Xapzr!N& z#y**{oSO`xs4)|}BO0+=+?X^@mgiwI-{dbQ(x}GX#rPT+PNXr7LF!~|V{kGst}!?j z7~hyIo>%agJ($=?n+*+@CMvQ;skAXjG#?r<4s7gY^KWh(1>5>Y{R^^x<%&gX=2b7f zdeL-SlEFo!a+?x!KiP%_Ho8_~sSV=_dvDNN5^SFAWToL4kuq$(=wgaYTZ43NyT&Hp zoTQMR;|29AD$Nvw)mtOyuBp8sl7C=bR(bwPa*~(s-Y!krxM=mtlYE*1vr+%xRhWCo zF`VKJgNgJuKhuDP1n=-R!%IZ^Cx5HW{ZD?X&G80rkb>Lv-@2T&t1DNmsac7sDQ9sj zZ=_QrCwV=c8qveWP#QK{_-B$$9&&3xgPxO8vw9nnILRUOGGuU)EbO&;{@XAu$=N?k z`lw;3kR{nX>x2}We27yT)(e;?Of=L9(l9J9-#}rt$-_uLX}=xMTu%uGEPgnVnS2Pz>q{(%HE%0L{ zT`v4%rO2nXnDYnEwmB}OsHI(%SckB)GQlSQ^;Dv^;dx8GR$-H$!Xovj#a!(wqC7=J zj7V)fX-;0V@ubap&B-j`2hryJK}@l^e#ff}KZ=qcvuPJajJ@|Z{6vvT-!;gmuNmZH za{b)|{U8NqUuTx z6t|uitbTrwLYk>X3vo`NvU$!alvQ^n-^C1t?4#jrqY(={ZE>hsnp84T}u_CDNxKq#Hy|`XC|IpyQSAfr6gCnR>YJG-N|xbE*d*FSN0hmBNv!nJI;1Izq}W(8o5e3 zq!rMi*mYL4Ied)@396~efQ+;vwgH!hZr=L-3<_ultR^0XHf z59emO{8)|naIVe&=Uiof7q-%uE#aNG|FoFouUb;&uUmxjw=_3$!d82#dpHr!=EQx| zGQjT{jzMdz>F337a;0xu6#ZQOn@$(_0(k9TF^8y&;&{!Al5|E?&d=4_mfO6&c~1ZP zGs(W=Im$`Bgxea0L5;Xn@N}H5`#Fbi!8+^rHT!y715=4Mn>RTJV*$y)AJ0fS>sicp zTsSGDJO6S<|4MeUC!vg&vO*X{=QdddYs>%RDFQLrYll< zb8)$7VsklVhCi3n{8)oUC_mOH%>Iit((t~DzOo~iQ|&rc;BjrpLFeUxJe6TX4qqYg z&ukLrlMI(K6U$GeTr87uIvR5L{<60rNBGwZ`nt4ahli8i(`psSz#BP+#qSDF=GYvT zGb%K|b2-)6%mR;BMBH@kp8DJ>NqId{Z{3Z@uj6@2xZItm49NZ5j}UcU9tNBDDK@z!P}+J;NMgc;LOh{rqv#H=C4uXF;}p;U&vOP+Hs4x?@rIuX-b13lUlf1yVz zNmFa*>Krb|ygLNx$-~+L-=HxrjMk0DCUCP%QmRhg!;f|;|Flx58{gkG=$f(OT_beY zj8(oswA!M6vO~2=lBZ*x)h8p2TGaM83)H&eMEBOdSIKfoz~4O6y>!4D>qKG4yThtv z=Yu0jqC7q&#ql<7p02cF#U5A5h=8M{cz9Z|I#YJrIel93O4o3_v*t}JMpr0vy)LbI zkL(*EyWAr@vzzev;+l9s^WJRj!<@f*(b`vMC&&+T8{NJUBi%l~&yekMeD|q&KGu(q zC5qhH;$QdV$*!jkiD~DO{kHLnqlYvZ<4FUruF(^ZXmKPH?%y1@TGX`rk^@ISQ!bB7 znD=+#aQEAGTfCY5y<_=fG%|EolS9-<0 zac-^VniImP1q%kOJ$PS|d%fKlcrj(vf&ojGK1fu3@V?L!O1EZBZN5VM1Oox(V-HF9 zoW-2OY@LX)UF{$jmh-F5p0})+XUz{dA3R6&oTfJ3iJ~<$as^j{aIJhxY(KIH)M+G=n#!cn^)Sc+>`I%a<4lcHR0P z(d=3FWlr_N8wo$xBkr50OC{pP5BM5c)q!>4Bk}fx^Whu#bSHOV3@U{GscREG*Zp4g z5m7y(&vUwNps$$hCj4@rau+(+y}SdQuA4p*(!WUey^nW{{O^-Ie|a@gf1%S<6QQpc zYa-}dBzNyHd*4vx_y@lJlIanA(G6u~YYzpAzgPp#s z7F2H>Fkkx-{|V8)NvD$2K}_g7KCuS2-<{@f=hbhBNkHakqG~ZloIH@Fv+u9vIPD*BNMNVekPnFSfSiHS9@Ef>e=KJZ{b;O&42l8 z7E0psk&^3X{( zk9i^wPqT<-!=wV2ckm>2vU}7*q2zF!ic^;BhByNQ7ccBpH1c-z0UHnV>f4EC{K5gQ zKuv}2c8oy|CW@IDlX49F=R<16LO#yPlLvH$R~H&8C+Ttzn|!Y<6s|?(8YoRwKF{mF z75Sf%ll;%~s`1*svU?ZFaQ^2OrDKu0afNgr!P{fVsx)RK!5=-2c@M;% zZ?1`tYU{c{Fh zHWe~4jDRaV?$r$bCP9nj$(#{Lb~%i?$>W4;Z8Y(O3|=gaF=EX~r4a+5G%+Uk3kfu7 z>4=y{DRdTVXe^aRtXD)@lf8emS!ld8w`Ld&bIuDKnn<+E6)DsRN5paGamP3E{LuxP z1;!kI{;NjKQj+h*qF14>(H!VcRDlF^^x_k=HIJ$cKNooYi;P(0P-2f;WVE>!88yj) z@Q%smUS#A|S1-acCmhb8hn=vsc@`OUit;hAPxe(2-8ka_pZC3=;?{qUyRCMyF~h&u zs4D&$?|B$_<)WYCmKc+rcM;VN+Ly0fy6^^L5*-QHY82^6fML6l4g%D8lhS=d zmr1L<*+RR=5G0b`n3UlvBGMXfZv$rVYQ4KAx#W57q|JKi$Loga_N_4db&;fdT^o1w zEld8elj?p(?xmTkTNHidc?C+YY^Id3fmoe`*DBv!Cg_$*->uUt)1ck9$> z$&3ERw=dB=kzjMSuTy8^nVcwl|4n!QW;_|UuZvd(Usi72$nQ2{T%>)SPE~E}yS93# zQApgZ3sfst)k@#4#*dsg zrkD^eRT#8IPsk(q8G+HBj$jk^KOff-d<9fb>*3us|XKPzmD*r@$U!^?g2YM z{Tm%YJAS|AIRNXImA26z`(2fhL}f*s&7u<$GB@IN1{19yUr z;Pc=y@GN*5{29CmrhScc;de($KtH$%oB`eg)`72pjo{DVF>v5F2oF|+7r{Hh6#Q<> zE1(}d2hIR9zeRYk1Z)Irz+>Q6@HBV~ya>Jrrr<{u#CC)St>6rB8dwL`fsNn+@ECX; zJPm#WUIgvmAv{)ZdO<(91e^ih2G)Vc!A9^0@EB-6hw$JW@FKVyOi3rC8T5nifiu8% zunwGa9^t`vz+>Q)3kVOs0bT@0e~<7PguDd$LGKR;4{inPz-Pfm@EmvyO!^Vw!8-6F z_#T*omqLp#B0P8xI0O6{tOKX~gz(^5@EExFXM_hkz>8q+e-Yk}_5k#Q3p)@Vd=RVy z+rUQ9)QRw55qKKh23`c822*+w@-gTKmA@c7SOwOBEnp*f7CZ*3g9s091uueaU`lT~ zc+wg4gG<0UU>jHuj^{dq2f=3W70|?Y2EPL9!1zA69)-@J8$1rq0GElK!A)SiqBGb8 z9tDqsy_KE8v)~p`#8QUm!A$TRSPV9*I)h8Vq3X`yPVg+)3~twS22X(wZD;Tz_%dkE zBBV49;lMY*CE!pU!hwgt!{86#39v+u^n%MkT{hwc^TCh7@!()XXRro52JQuiN}a)D z;M3q4urR(eNOBMk%mfP)I)kI>cd!~PGIj%p>=&fxvvaqtA_wsZ#D z!Pmf)zNn8@oKNrwI0GD;+8L|^r=)cT8^JT+F>pmX@(Gd*#Mck;f|=kH8}bRJ*-^e= zGq@AHp%==RvNz5PcoMuo*{3sT$|t06Ch7w?A1nh8g06__QqZ4|^9tS% zHh>qwW^i=>&R{F}G}sR24?y?=T(_VDj4wcV&<)mr)4&F>9&85h1Y5yR!FEvVKzIis z6`&hD0+xZN!CG*j6Zr;rgGa!2@D!+%QEp%^Xma9wfo`w@ECV-zwO}K-7kmdi0*Wq# z2W9XAco%4rk#Eoq=DHCctOIMo!{A=f;6Zq>7(7LfgBQS0L6ZyjZEt6AFn9o*1HJ)n z0w?-VKf$A53-~422FiZa7dQF`U?%u8SPV`K;Jkqw!7boD;6d;;@HqGbcoxhbi1Uv7 zWCfTB?go#8P2gGZB-OoWC#en&7zF>|3~(X19ozyQ0Y3-d0CRn)H-k}c!Dok{-hyX_ zqTYf(79sy&)+p3Fa2$9Myk#7&FF*Q};9&6UN$?NqCwB(7fw@=Vyn~~sq1?b_(-98b zIioX}79ix|nYb>&H)bImm^KIL0l)m- z2KfXJfgRupFl{i-A2=9XxEARJe+O;>cdW;C4SoQg0E;%Det@e%(-18Dz6RmIx4=1| z=URjV7l8-CrJE5BJPv*Z=3b9*Lvg-9H#p};gae-hH-XNZ5Du&YkApLAMmX>xP&W+c z=@x_oXMttl+uIQily-Cm_k-JaA{=-eJPQunjc|qNPlEa2`8@~+<~HCu0Kd2u;Xv1I zoxx{8kq?+_0BH+Tjd^LvCVLVRE*cz!>^ zfw_0%z5rV9ML2Ldcm#Z_3E@EQ4>%v-q5}vwf{;7FV(_OwA{>}^5cL&2@F#==-v&>C z3-3cX@DE^GF(xD)Kse9?E(E`S5aGa~58*t52M!|~coMt-dLKr(ktlC)F!rgJ;2s$58KZoou5lA)${c48m>1J56=*T&hl`Zs5>X<9HhWnsV@L zmV_Nce2F1tEau(R^<yrEl!gYoVXRvuiAUFZ8Qc#7^H4=sQ=%PTwi$x7WrFe*yYy>pOyd>G|r) ze=q)0`b{`~YC~+@4gG^_V(a+bHuCwkvE#3WZo9rC_+C``zSUK}d!fI(r6agGN?+Hd zAA#=Q))B-6c|`gj@6t~}KeZ!v`Yu5KU{`G21izo}=?EsnuDg5;UFGA3{^6}1!Cg`M zmM*;v`uM#)4Z7~xj^Mb+YZYk`hJ-FxLenCu z(3f~VwhgT_xv-hlf_qtHTKJp0&dF%#uR-4vRj0Og)u|=W1IIBQ78QO(SNLtv7eOB# zsc#OK1&#lH=)0iLi`4mPUEyDWeggX4QTpC4{d4GJ{@M{d8>PSBrK@rC)ngoSPLw{i zOV5Qq270fka!%0 z%YPa4-N;8(R6ff?`K0Nug?@^T!AAP$_k<##`d;YcUyH3DfqoqN)luQ+cZEL%U3wj3 zm{EGT0I2^9(AV^!(|@e{Pw2Nqg|F}O?}lE8_IY=dep8oT2K`~^=cDv5yYyP<*Q5Q8 zRhRcdZ-G8HD*RPl;g3N7r3d{K^tOMXu11ER5egov9~Yp1483opUfHeF-`@WUIv&Zp z>&KFA-3|R0=vk5aoNm1gx`_H0>h@mhUrp%NYoY(#L;ChY@8}_YN1!X=ACK1E>8tMc ze+qgU^qBdFo&h~({&lGT&|~Hw`atM-RO^l(lUKN`LgiZqeO3?QYoS;75dU82>v{-( z1p2lf!k>bETMywcKtBjQR{5Cpc-HR0zZ-fp^jPP&jD?R?eznk#^$`DF=uh_${}JfN zdx-xO3qLsW{Diwd;rb8#ogU&h88GJ6L;P;&ANLTx4Eos~!q-Co270XW*$bVZeT`MV zN1)U8Jy!Tr(A5YZtA1aAZiF5yyh+0IZx6Z~y0-`aWzY+u$149?=yQ7TzZd!n=&|bO z5$H8N#D5C79a%s4aHopiA58HWt3vn31J|Gm(2{~2390{sT)wNc^2H)tCE6!dqXPm0n@y8K^&t~}il^hWB- zyW1~Q0@|@2bo@9wk)ijA3_q(od>M2v^vXz`58r@j`f8!q^`P&C{xtMuQQ^ZkC>s6< zbn)Ge;H)S;Jb)6)KlGu{XGiH%y7LcxBlHdggl_!AGL> zLtT0q^drz0N9l9B^jhd0(ECN|;qFVQecB6MjXC02{evUWM?-%%l|3p;+c+M zNu+*lNDjAu(ECDwBT!1wA)XpVzG)fqnpbtonBf`XT5Qk>UC2AqTX6T!8*5^qEn5X_sy?VchOt zvHiQDXZE0%LC=LA_58=zcKNS`z7=|``1e9@iQ(UlG#`O}9QuOD^wa0da2JtKo71on zKft_cq|Kd8@?4%p0qO;kHb{e-J($dBEOF;Lj_6jw29+J3`k32f|G$Iz1> zx;CN$Hish9Ow-dCKKC@u&9M3SH*8M9W?@^;e(3*K`Sdqz(r^P=g0&X0(nIS~A#A>c z&DD`<=DVLoDxiC@E@OU_KC`PX)I+a>j%ktZYb|^$2wkgm?D;n6u1K9-i*_7qfqn>j ztTa-av#`m*-t5K9`3=0KC{-s#(^~`cVfTm`E|@LVW)Vb zg5T18uJ^<49@uS@gIrH*BthO;3Jiz~&v; z-1NVG)?PYyn_%O|TBE2juyA@-T(W6`&26x`{(t?1t`eG_FKbQ3l!+t_{FzTlaHiN(F2(H9_ zm!Ggr=q^TW>R_`CHmm;ECe+cPZOeYxyaSt9b>>;9lM`082Uiyv96gp&~Jbqh)f?F-=z851bsjBUu)Y!`G?*NJyt*HS?KN1 zWA&qGwA{wh@u8VL)$h}%6Ox*WigjszkG4f;5dK?G+=Cb+&N=^4=+X#&AnZ0`zbj3A_w&IK=yZ)*RC~rR z4keg|KZQ*h^r-fg-_@mGfc_eERb)Nqf9cXq>FEDrjapQ@%U>VT?eOo0o`yAPw_(34 zeYCBKXuD^?X8Uj0Y=X^2*c`mv&z+&DLS^3so4WHI!9QPab8py&=Jz;k_zNAuS0ios zmag=kg?gp#%jZwp!bCyD?csJ$3u^G-)w_k4?R|S(Mr|` zeNj|?!-JVLAEQOgl|m2Y_b<4CUdl&Ae&K?Q#%6p+M{phXyKHD)BgRduVRH{`VzswB zp}zq=R(pCFdQ#_Q?GRtpb?#0;uYw+{-S`T63-nm+hOQUx|G&geAAVt!6!xHxhh77H zHNodnIq~5KJ6boZp}*9Fz7x6$YdB+tKMcL92mJ)}W6)#WFTaBREOekdeG%j0y53lO z20hle8GhiByaGKbGW>+@Yim3UUmB_Nqq@#dHS}*{q|c5pJE5P0epO_6T8F|eLf3aQ zY%;N)HCBCZg+2p%tTJeaJ_q{z$T;}!=MlON_MRU6JD{KJA$%!x6W1BUt+~65!$U8j z^w&U-hfeE5R~-pYvxW2q=(--lH$zwV5WW?<*hBbs=%k16@wn(akiVgk=?nE(!|kgB zx)tGL^{Gms=k}o2K=(qAb&WSbzYaQ0W>@({jIA|8zX5uzv9VU@C!oh_W7?s2K#$eu zipQi*Jl`2}?9Ksw3iMd*bt&{E&|~GV26{8}U+WVhX`})AYtS1b^MC2tFw{c|pNC_x z8z{ssr_<1fLjSeCp&h<2LLUo#b7VZ>Ha#>;PwQt|HtyxHiItbZ(7%E1j`R^RPdW!W zFLnk;Me4Yo>FwB#W1FD&h2B#dnqadvh7X#?7U&N{kJTQuLH`)~ua!gS8q7f+pvS6j zcIfAN2ww>Ommc&A=vqbW`ce+p!i^qRYECU=+77U)B-?~FO0dl33P2%iyk zj?7)>=s5KKs3WbB{`tRkrBP=<-wf|&*b!cgm%7s+B!_&^cEo_DX&3fIZR%iCo!%MjP0M;&q>Uw1*3@P{Y?k5OjF8P$zi!h4nY*b(r0w(cIa>Q!Fw)I z`t+{6EQ!Zj+$@&Pqx$Z1Ep+F1Cu}6VD-z1@AAa4Y88-SHyzfHGZD*v7)D;(<*W>Um zNvQnSN7{6sOZvWLCu~CXcz&c!^trTS4y-+&#WyL^t~1yl`?%Afi#+@+jPN5gMy2N%{K$A8W_F|> zKK!6T(?b{LFNYpVPkw}-%hE%TkOzjvPS0rgc?xU4NFq&%)8M>kpmR+Xp>VpU-F_$~ihc&G7TZnAqu|^LoEP zzlPSsgOPrusQPmeekPCW{MC6kT7L@hkbgDwke}+jq*+V&_vqr3dv4?p9ehy0Yq_H#e{{7@F#PdohN z;Qh8xIj2SXi9YY`@Us!`zYUB`Pk3lO)c*LplGBft|PUrO|Ko6bw+6X^F^mXwH{M?K8_B@gKVrwF4eWvqzN1%tU z)4`E`qOa3pcoR)Q-`Gx3Tib3z%@PZRtE z=Ee3y=k>-w5Bb>}+t26lb5liZKXhL24(J4k6<#6|s)sG`b1n2xzP?h#ET0zmd1_(oe5o-I_bT)j znjZS%#HI5kltrb7z86pCrkpY6~?>Dd(F=d$x&3O_F|j-4L*`w8zr z5BVvM?WYlbj7wwtX@j5M&_jN7vHi5c&s@BBirZm#-dTTx_K%1kYYm~_h^pLOHXBtm z<5Z{b)4qlGQD;!wOJ&D*uN5nVuEBe%SyUg+-sP(Z6&tbRPz7`Y^s-2ug{Oz&@wWvw zb+AE~uIm#Fa)y4$)|kLij=afX4@dNFJn@t&R*)Onfv`!@%N+Ue|AtB2om}5tJZa%( zhGI!L%DpCHR7g2wo0>eLS`<3is|-nah0DI6BA@WKdNtW4(AUgPD#xJlzgM*(%4mz4 z8Z}U(voQKtIc7ig^nYOV|4WiCvx(cs_1nXfmsl=-<7;%5f5Z4TF8d~)9J}N>I_$3& zSKMVm{!hNf7p+C#p7(zwkPmp;j^NDyANkWA=XP_adGaA2t~R=7`Q2QrK>jJP?+-(N zi6^gK(%*pou|U4~e@g$Y_%GN0iuAn8leSCg`K|Alhu?wp-z#9OmgVnme1B9R$6|f| zE&rF7&*ky`R(W0?{_^jCL4N?cs40OfQE%_CXk{#|P7np-&1UrfEYTKhr~V zia3H52%*pSzl2{KY!AC5VJ}!4HZ`nchB)-0|2Khtu!1J^d4Q!4E0;o_OYsTpJEj3c zd(17Nzk^OcrZ5Pd&ZM8@Fi7i%;hjszU%5a0{ZcyV{4f1rg-YmiDZXEV;q-rgU-&rw z+2u;QcPSt3?E91hVf`oe{gU7C(JR|4CGh`W3AB&9RMVovJJ|JF$XLo)!C1ps&)C4& z#MsQ(!r02##@NnC62k35Jfoe_!C1&x%2>fz!&uMQz}Upt%-F)%%Gk!(&Pa?benvZ^ zgRzjYl(B-bhOwTpfw76PnX!ejm9dSnoslH6_!;et4#q;pQpO6#8pe9Y2F51FX2urA zR>n5Qc1FBR7CQ0qjCMu`Vlqsun;4rJTNqmz z+Zfv!N$ABG#LsAFbTAe&mNHf_)-cvHHZV3ZHZ!&`wlcOcwlfmC$Qqw`MmwW}v5>Kp zv4XLNv7WJkv5B#nv4ydfv5m2vkyu##jCMu`VY-Ma?Y-c2?EPh5iql2-Kv6Qibv4*jp zv4OFPv6-=jv6Zonv7M2mvG^J7j1I;^#!|)##u~n5Qc1A*9cA?ilqn**g zSjbq)SixAsSkKtN*u>b(*uvP#*v8n-NJ2mOi})Gsj1I;^#!|)##u~fz!&uMQz}Upt%-F)%%Gk!(&PXy@{ET)+2V)^)DPsj=4P!lH17j0oGh+*5D`OjD zJ0rP9gKyHrHmDfHH`I)4UA2U&5SLKt&DAq?TjRw#m{JGbTAe&mNHf_)-cvH zHZV3ZHZxYtj_zMSI%0kaeWq-GB>ZIA%yfF3rsV#|ET)!r8Z ziY!CIX3&TU3bN=p-P5xKSu#r}88Vq^re{c49ydKbJ((u6(bJO+L_|a$A8JHI^h1pZ z7<_`bAqpb!DFy{zK=iSRK@<@YqXI@;_|EdLd+S!;8v^?J2f5wRXyI#u^h zN5?U9j~E!79}ALwnL8?{GfIm0{1Gx-<0iW!vMFnO0K zKHFP>d`uKS*o#|F&hXyk&FqP*m$E&`vqbsK@!WC4_h)!-@g|+|r1&A;##o$QPUC#I zmv;01__70goTu8=tL?`+;2WL(od?`K2gGTfUA#~pV-*Vpuhr)m_?UA>^iw``mYew0 z_RAH7$H{+{&>!M$f*wqM=vKnHT}LiZNlZTNe?V{ikD>oF;q=69jUE0|=tKRR{1q=F z+KFlZ|i{H>EIU%eu%djcB0(q*&NV+AN0n1 z+yH#DgZ~itJr2GV_!bAJg;O_?hD+liY{kF4oax|i0)Bvlr-09P@GjtU9Q;3k&vo!A;BR;Ei-8~E;9mg#P6z)s@CFCJ7kI|O zp9J3O;C};ttb^ClG{pK@;NV99?{x6@0AK9jCj;+x@UwvD9Q+dC{SN*);L9ESTfk3u z@cV(UaPZ#&A9C=20v~bkeQ9Ha^)Tw-hXJ2-@Z*429Xtnot%DB(KgYq(1%94`Uk>~N z2mc!IiyZt1z%Oy|hkfZyZby}-9P zcp3PE4t_T9haCJe;Ey@@jlj1$_`Se?<={^Pf7-$Kpp7`z!?O;481QWlegg349sG3Q z+Z}v8@V`6w<-ol!M*aT{z;|=-yMXWE;J*OAw}Za~e5Ql*s{^>b2RQgUfzNjECBWx6 z_%QIf4t_E4w>$W?z>je7JAuE`!G8n1!NGTRzw(0lEO7Am z1MhTje$@ukFLv;61Mhb5hk@rD{I9_K9sEFAcrgFv4qgxZbO-MOzQVzWfe$%2zdnWe zk2v_pfR8%(HNYnw{9C}Q4t_WAwGRFW@N*pex4_SH@E3t!;NY|9zyQmCk%J!!{1OM} z*XS_)1_#drzs$jhfnVX^>wsVB;Fkj5=-^iYzuLjS3Vf4;ZwCHl2mb}|8y);b;NNiY zJ!yf;?Y-H-j{<(HgD(VryMvz!{QC}m7VymuehKh<9Q@0`w>bFsfIsNq4*-A2!G8_> zF$bSP>ql`120_N#NTZ{072#eRuT_bcMAP z>$t~(o9Btl`ImnX9=9$&l2S#SPuSwD*RsGLa@PB+2QyE{dR}`IC?n%^YKonfSEoNHppi^@I4&- zKHxJQe6RhL&uj1`H z_=kZ%?chHGzRkh+qLeH1+3w))A)Nij0~hOfm-lth2q*$K&pj^3xL~gq=daEe{s(x^ z$K;vg=r<6(7n8^Sg7B%$1K$XI#2LT$5YFtcEeHl6(c!@|Ed zkNr!*!~7@I;5{ce{o~}ZccX)f`;k2VbByZKZ1)`{xG5c8k0bn0qW}BdI$zAgtzPiC zFr$3t>TPcg@b`XOfwMr*f7^iTZ97%zPXm4@_@D8J(w_qS3E-DLr1%lQw+kQqurB-2 zyZ%|uA3dS;*W%s+gG2rl@D&EXU+Lcl{EB5PXYFS_(UXg=`CLRew<~{^w(D$V=WPc6 zWB;VUA5ikoLSMti+m~E?T)l@7&h2{kG38T4(Z#?s_bI*(_!#iRVQ)>q&jCJXAC;5T zD4$Od9#`+xp#Rc8&>&nk6F!gLbUG8~^T_SU=jRE0ekXXHciq`qZv*IO(H2Nty>BI) z+xr~s=bfNG8u)fjbkpZp;3osWZBONI`f((I|60N8yfyDtKJ@+@`FtpW{!^g;@fsyG z`uuJJ{lf|PQwjJV!T(d=Py2|&4>#_m^$r4WBb@cW;5dy3jNj`Me5TiWozhcS zE}tUdae6+BaMtss&uG0LRB~?<;mrSvF6B>7Sw6QC9>?dd1bmC&b>2y3m1jNp>^G13 z)Ola}km5tYy9Af2BLAEL`YTr|y%}GhC7k7dXuC4F8m0Je0Wf~$pB0|~zWbrtE)RNJ z0DOPoN5GybjFit>MGCU9EiNZ-UdE0^!X6hHZ);4f@rx@)1^k|Umj9iO{@?NT_?%J2ZCEDD-vS^(Ezh z4EUS|`Uiff{O1852mbdNN?!v0Vc=K(Sm}+PuL1r#tm6!R3*kIo?yE%WeG>F9{#^N7 z3;uQQQ2Ir^ihp1?y>T$`O;;&C4Eo~*#}Dh$4f@B&wce9(?;OHeKO26ld@cn3HPBy= z{^j><@#jU*kMFL0dKLEOP~Z`#|3<=@e?9cp4*F#Y^s7LB+f3zu1?ayh^mX1FRw!=f zm%9`AJduFE06wd4)CPSNc7FJg@$xq$;HMEjkMmTRNA`rAWx~0=hrX!%hWT=}^2 zG2>!Ca$G!)b~{gT<5%Ab{McO7PTB}({ha=U(wp%<1pKI%6u%pFe-8K|Pb+Tp|8?M( zAl@+eeZVh&oz`pSkrxPOIlnnyakGwiGX-pn|A{yH>61sj{e-ieZ~DE~>+8kaAoL)z zE}sGYIP5Bidz-<3p0( zemVGm8uM=zd~N~$efW(Q;J-}Z^9Rsx#QMe9+iv9WS)cB@>>$G9%w<=k{*jMe&zG-w%At>lHVCdmMan|E2tw=xy(#pdY?h@p<60k#LE7opsGOfNwu> zccC=voz39C_ZsB`alJ=?&+1g%*ypba7yHM0Hj8>+2K_od*h8Pwfxopu^>F&<6+Z{~ z62h6!8Sp>GU#%8;5LuV^6F$rOZN7dYf&N5A9V2M6&ggzf5ob@)c zSoJf43buiMs6z>jog7FE8DfgKt@|FZ=8Clc^K zfzL~SPz*`&jo%P;!lHLLwKAXZcD)LBAmz3RwwRy1pNO5zi}+;^=Lq`{FgiN zz-+>~A0PU>Hgp*D#|dr<52-UpLGD#Amc#J}1W?AN1!r^Tj%$R~q}@3qk)F@{>i} z{0iY5Cr=_y-WT|t!iVyZd9By1YwiV~J+R)t9Q1V*0I>Yt&y?Qef8Iqn>tWLA$Kyf& z82a%b@L2`?zTYSxGmb7Gob~X%g{tS1^tSh3z)$Q~d^f##e-#`|t;;^G@pgD{0)8~% z+^%c?TPrki<4M5XdHh_$xxJg6dG_PNzs_qcb zG~;oX`6}mwKUFy?td-9^;G5weMu9ITJZ?NLOTf<*yv{rBy(+))haUs~uRWmko(cZn z0sTkMPK^m72|h%0{x!vQTnCJmEOc% z2Ley?fn@rayv=~%Duey+O2XL=S2*MD9PqjNH05vdNFOJh^* zLSN?{kG!1mSAR_4^AhNH#lF}{;L}VFneB7sCT-Wbz)uq#Kdj3b;jD*N*a1kK)z6KNf0#{pT>l;>c%63{KUhH@<0lsq z&h5IDAJCwW$^Wb-ob~f2%*)0;FDIPo-Sx{2pnv9z%ICeri$6bEp!AO*KWyU3KLeiw zJ)FwK^qJSD^xnQIr->ilLpaOx0r)X;zw#*xjvvU8Gi{x4qj{{*UaZ@>94gpF;^}J~v{WX!taP{&M)e!$5xq=r=m`RzN?A zyp4%7uN3;(-i8x&ydaJ1eU)(5TLacXCcff7XfFN!pvwOh)SK#1`t8U!eg^n5!STbo zoC*5pn|BqopzOTM31|N8e^h|N2Kn4Y_$)eKaVyRzAl>fWpTPeY;Qzt~<+Jw;<+F`& z=0EvM#f|>wcC!4npN!yj-uV|Qy~!J731>cU`i|n1zLU>M348`Y|0Bn)&QGBKL;`+Y z0)A5hen$fSBf?q!y)hve|NLvh+1^&b-u6bjUI6`e7{Y4cGqcKPIp(Q@fFBL~0_+!< zemo8Mudv>k1NybVFUPvl_~(xi&UQGLAK;>oiJ!kkILkQ~c0LPy?h$(Yur7ZFpVhS@fA|FOV_sHy+Cjey3BdBZaos_{&qtoew5yr$IQw4!`VCmG%>)0F z31|Iphn?&NyaamJf38p9|Iq~ebAs1-pSnofWyalC6X@?CoZCBzeN39Z<@1LGdj8|q zOy6*j_E!q@2T0tGAJ*kvgfsm{#3zRSqy+jw!D;`!Px+hp`BT8}pU`%_4}R&E1U_3p zeEC64hK>y(c{F4dz z#svJ<1pL1Rr}HzP)CQf0cKrf)^$rCv{d<2RoZIW>!DgPQ`q^tt^>64;7ra&v6NEFL z=U-C(9|y@M;Df(X{952Y1^-3=Pw^*#KLvU>Zhrx|Yd;4sQ8_!3MHmH)Q3 z-G$4fs?Td6oaK4jq0xTtC7kund`5a3))AyXL?K%zksy4=JKWBpe*GDTo|NUqF zd|Gh*ZvXqm1pJ$Xvpm<%QaAM_fp*0!#@-F>_ZC&mj46T*I2L3_WFUpwL^hF zf&UqT+Y;*cDbW9;sP$6%S3aL7oaMX$Mwtk3HZec_-Dbt6YMPo=4Zzz8=keR>tY=Rmob`Mo))7AFM}-f5SeFYD@GAwc^SYO+{!O0b zHt>HM{qi>O`bh$x#}n}9!T*PsXuC|D!+&@-u3z@=Q$0NMDWy020_}uzdym0-(CB{| z@LQd8S(CtTepLC4LjEhj|8X1`H1d2=a7$tMdN<*-YGLo^3G`1T;D08Zi zt$A(l@&Bp%H}TJ@g4b3&K{)fd0db%i7at{D>;&gRSD@%spg#w8(g^%E;5qEK8hzd; zd}w`yb;nH5{}S|fB0i_^Qa-zq!(n-H$dea3Cr?iTJ}P*f z*St>onDy7W3G|l|&U*VA)*VLwn-b`61^s=_{>THs&-1jszeKyX0e8;{y!qt#cBKes zc`8_+n(>}ZpkF3;&N(}Q{y!7&PbT1B2LD0i87_z3ekk-bZ(gkZXzcTMpg#k7 zLYfxk^9D+g@;p8cKWXGVl5m!PHP-bW}aF{ zIJftpC$+!K`sKO={uUg~(Q*TipgC^5_W&vZxSIfihSrw;za z^w%=r_0IX0mBI(gvMyE7&%yq}B<^hxdWr*^wB8#;8To8S<_x*R|OcAQ-QPmH6t zqs(I9?sHztfVV%bdZVyJK9j<~&KtnNx)plc+ax%Mtjkxy=VYu0&3t`#0{t`Kzt3`Q zubE%=ToK={xe0iRaA_~{5ATCarvsk}12^-|+5|oyOu#<@KJGcFTY#@edkz0Bz}@rV zPbBc)r=arhhWX2kqjwM<*T2^zKlztc+98k3@Z^(9pkEID?)vnC1o|1|(0JeHW@q2$ zO7N*UHwb(+5`N?!<@56d{?8@gGY8|_d#K=b-uDp*Yytmc2jFA4OoD{5S^6>-J> z;QtoFxn1sg-Nl5ne<&btc{u3L0R8OWX@%zk-yrVb5vkuy|r1Ud!uFu3XhY%h&?v@B%=N)^# zw(B_5yE=jX0`Pb9rB@`-Z%V+woq*pfc%9d=K>G#Lp7&@1{p*M0+chTvKSJ<2Z2i`9tIXk0{66+cAW5d)2Xm{i(N{sq(nzKRbbMeOURMcygR@wyQ%26*vCmJi!CC1YbT0 z`oWc2?=iUZJ>cE#iktQ0lL>rYNWc$X6|d)a2~Ouo5nq|}Bxe)O`akG5DnFT%d_Dnu z{d0=11OB4~{*Qv*eZKy6l-Okb)FFRp;=02H2UF{kC7k2%EAX5(mQUWP3H0OOKZf(M z#%^yQoZB_Hi|W(lF*Yag@rsq|RCRiIxW4EOl`5r`<*90^;#bG~;^@S9Y0C5cp$UKG z=)_=Q)E}x&RHppG^jfbtF*Z3`s+NZ8TjsYk#1r|$6vj&a z(Dc~YIx1rQ@`A~{&;==gQXo*-bI52Lc zZZD4)D(eQSYYwN(YQv08fczf(w1x7M+Doi=fi1 z6p0-|#F&4Gm{e$VVvUj#XCX2T9VxFIsky`2Yb=b8P817dLe$YKiJZXZ^2+gv3iTxa zXeBL9uptPm`D9^bf2ulEp75tirB(D#!j0l|CB8&ye28zf=Cj!)em>hD{^%I!3x8yL zI_XDG2}PLBuBBFUMdG$TOJbE}pl3->@3Nk-EO_<}^mMfKTTK>$dixjK+EN{Mk+S-1 z4${aRslThouCI}W?dtLK{cZgNmT1bQ&FLShuWe~>rzHa@sT=z&RjI`7ZJqwYu5OE3 zw9wv0RuXo4hGppJ?KwHym+$JegoeuUwsxDCxN{)qceeGn^<`^radm!nc~`#QZjF?Z zThvx+XIQ(`5tUJ8@9J6D>o3V#J(vGhXpEmKzpJ-z*YK1wYHO4RvWTZhd>1Y>-lV{u%!<$CQN z7F#%Jptrxxl3C2DFH7Ct+tY2UPI&h%ZR37k*xl7pV=8#Y{|=%+9|Cdj`7gPHbrFImOTS4)obg$VA;r3)A%5j%)rvPiJ-^RoZE(Bh#Lz;p_L(Fj-6* zw(qFz&^RUw>ay;%^!9aTX>3`vjS4NKz9ctj-DvKlG1J%8KG2`_+u7`_do9+zj#E0Q z|E!x$*3BLozTvHOgLSKOVRt|ELB!j&uxWR6x8?IBdw+IO#E)gdn`BY8Qmta--E-a9 z~OhOo!Dj#?2V!Eg|z-4qLzVVr*@xK%(Ad9NJZGM) z(w;I?O}Mdepr9Vydd%4GYxXy`{7nsQ4Ay8F9G&`>!|< znY+j-a6r--^|7hOfu1gMfcdN+odR0PMf=gXLZB#Bs*%aaDdl|PW6eQLi~8Dfi>-Ea zwApiAD#bC6-$DIjhhKpEI-SW*sh9Od;rYU&uXCBD(sYA>oz$)5c6)pLj_zJNI!QIl zeG-s8y3XX{rdDlWGz+d5&?(huEqYAD!!_NMI@5@+nyFiKH_duR($tshQJNd-04dt>2tEM8;gMODkM2q`Md2pK7?Yz9Lj!mu|uKEr1$)rjR@`ZoAt4UVY z^}M_uE~T&p?oQK!aD27YTN?7m3gd;9bgjs;tWodj+ImFR=l(D)^&Bc%EfR;R!s-$) zx@kEcOPb5ACT*q~xc;TGZ0ss$=ku>@r7}^;<@)_GBA~jpl~)e%>$Rl5M0FJJpnGW;wbG)8oaFB|_I# zv{kXB-_Hz{#~Sk)pK2V-rBWRO)QA%4VTwEE)Mi>%M+IoiumFRViNa8^K-r#TVRGes zgHQjJAyxR>h<9p+)E8x+PESpbCA3fYl!L-e^J@yX#25ApYs*ut8WEw)pXjfEXPiJ| zg|)9;p!mZ7qCn;Ge?g$w!mmQ07?WiGu(Ku^Yf{YM-(*s?R?(O#jh4nJhbXzXvFXvB zm$iw;kz=3pr;3Hq0=cFHK4fKNT>NifR6F1@R9+n;%`jVMS^wQ37bu;%Kr9~bflFX6 zt}L0_7Fk?&x)+Ave*(p4YQ1Ns zbk;QGplfUAl444730SmHV%_PxW@TaPJBeL59Vq+ocTho7jd7$~d0W9BEw3si$SEcf z+Ejj@B({8k6~e zQOGG%Gr6am3R6>*Ro}T5HP={iV`i*CJwy8&)$+>eiRq~%MrG-3u}J+I^m&|7PsPqz zQd<7mVc{b6b-WEarV=wEnn@XB6BOC)6oc+YvR4XFCDbGhRM+)kN34Uw(7{4+6$M#V zXt^`w94@BG{-&yB4py19VcIr2O3|dhu#b|VJ}1&>yU1VEGvH?_*(u=SPTHvOho&ZI zlSTHtI+xW`SX=M=E7z{|Crg#73EFfbIemJOU7=c9%eyDM@8x$dL?V&V+|4X2l&b{G?h@}X6-kd1le&LbEKPFEto$+^6JulZ^R$U1E1qm_ z$?9_7lAgTZ%3Fukb(5tmudr+tbfmJmR9aZCWR>{BjfD%!G&9$z1cU3UCF1ih%}Yj( z`CYWpTAWpGtcqs=tiJ>hw76 z)vb!^qN#6+l##3l>4l)7sm2akg;Y!a>Qa&R{76@#ELp6Op3Xe;Oe1r1UizvfPu*56 ztjv~03b84_w6@5bx!NPG9V93>rj>iRIx#tsCD)T1%%|9f`@5v0Lk(nw4~?;vh3ZI& z*L6EF_ehz(Qc;(=R3=}|43jYq^8g4cC{;wo+}*J%25Gu?GPAH!Dv`ahVp)8@C|dC< zrKyp^q?rQsHnq%ONiCys#s-morP*A2vNT3Jqwzi2H9kB++wGIWzN^CB$9x-Or#@#0 zs>;zf^I74=3HqbuS!W0kzy6agFTkmb{(J-&FBV7J&1#cwTf?h0Pew@_=d0POa!bmz zigozsR+sZFKAR4z2@FiTx1E=;(!M;J*Hkmtr{~AJq#(++Vr11^xk?o96?cipLykvPgIpa66u9hTz#T;lew2=&r!B?~pc3FX+07MA*PjK zpPj8W@^ZP4rfkS5Mr?)JI|$i{d)&~C50l_X>?NLQpmCr)xs^D1Db}BE+Ue2r?+noc zm6>R$xuY;%ERAA*;)&myf5LG~!$)T&42EE-o0KdVCb#HlAQ&;+&+`{m3X>x|hc+%P zOwue)u0PF_xYTL;9oss>Qa}U8zN3YE&}gxf=B~jVL@{C1Nu|;$YuZ4^(}6G(s8Vms zxMk1fV^qAV%tutPmPHb!#CoWl%#E!+J1HvF^slZTgPKE!mTKZ5Tq%uBP{_uuflVN& z7Q4`xIxHM?q7{$vT9Cpq^%vB~{&fa1aP#R@M0^bi9${)@2G@SDwXb&xsFcT(ejM-RO{e~ z9+7XP8dkDd>tI+}HD@G>;0$QBOy@YMatSCJeHw^4UgIF7$kmMEG}#3QbTkjEY=OKw z!?*8XbXjs>7G-WgrMOu`Hq_>b!ht6%L~#^YcVMQVIpm!SL2>2?-NFU7xOR@+ejtEsaMp_ATPLcI^j6x~QVb5_@{ZXyP%+GIjA7vD_zV zCYWzSHOK5=2Aa1mw75E#YLUzcML}3o_?(ELEzTw<9oPKDm!ulV z@^tC{N|UMx;Y=?^h{!4?R5hhn`j#ZG{>3g-p)xtrnsAjimh!9NW~A-ZsILO+6ZZ>e z2=CrX=h!8`#lzQNCLxkF_8$&sFWhD34&kUPmFb$2T$Y>~AR`j(w>Sx7TP7-*ZjN6` z)XbBnmifGF^LwU8N0Ud6*w>^?s9_?Gi>;Zcj1I{J!(4dK_mSodYuliNd96ixVWyc% znVf+!cXfU7y9cJj5!Oj7$O*|ZBxDs@BJ&cpBE&DyvCt*y25FB*O8(^Z;3yr`;9-ZH zNa&bF6gx3tB1}8GV8_vR<2%hD4IRh$>eA@MWJw0Bak=tCLwpuCx*)}(4dt?Z9JlFz zI{ZoZuoEiVn0fUZyfIZ8FNQU65cr1)slr;?91131XWk5&O-oiu8*vqE*=9Xs9D({T zabTbjmq3G7oQEhv5E-1DZ_DP2eLVcq>9ORk3R7L!i^hMdYRzzLVK+>tGN?9|MB-%n zImJ;l*U_A`$6}8?xNeyYwVajEXn_XeVrOJCjIiQMBpaL(+bnZZP*Y}-kenJR4~tZp zuot2WV{2s8SP}d6CgD4TtT?>7Jv1U(5_uXsrzfqx2o;)YX5C>OB1v>8F|c%c?51Sq zNztz8)Of@oHy<>g2Xb;U+%!n6p{1RkRw|b&B3U|x5os@S!wxdbKV5SK+htU(y(yTy zwaD^`)r20Unbt(K13`USl$DQYO7aCU8(XiG9M&yaHfiGo?FmGB2h(I{kd?7|B*^U0 zltK9+N)n7>!bmqKxq?tR!NSQj(wYjb#iiPxJmY2Ac7{W>IK!6t5suW1_^cO>*FdE@ zv`xep&kCcYCQ^?&N%0Swh>|mDFtad_Mjr-q9*>8(V9`=-=y-ytAZ?Ak=>BhH?A4x( zNx#K#N%P-y4?bc)8% zbK*>UJCYZntaT(nSvsya2$K+?WyCijbPD(5qz{Z+l5)mzxamh@!C|Xg%(6Wg^fX4< z6Uh#Q1X$iMS>!y9v!eDuIDn+NsilY=m`c=-p@a&RZuz+@m+uFPQ6#e-s){F*I zymMjd)(AH;AM;Gd!^fVcM5nN6}*VQ*LFWn!&zLyS0GpJ<#b;AVN zzNFc{n|AQzTyICqfGN?CwFXO zb*Zaw$(h;So_sac%wYzdM5Y53^yD+mab5j>Hp6C|?aeoGIyz40_WI54|Hw|DnR`Tv zg~$b%-k^MjNpz)m(1Y4Ry6B^arAs9$&j%wCkLQ>{ksFr0U8*Tu3q=MERydk)mRgK? z&>pXv2UlgMzmW-dbEDiMCx}TQ<3w7=URlI#A(^UClcqoE_yIj4Zj@u3k~n8JE1rbb z(4h@g2V#1}H1?WECJqy`*%Qa)v^{T#)l(jb>U+(n3T^F#D{gy5%+Vofzs3v5u;6@A zbW25t52wrRNI%RrNEpH^h^(g9joJerk!o^s+#Ezwu57e#lmZ=5oj9Iww93z-9!8*{ zuRFd*`MS3*;h74QY6{Z{5nGCJz%)#`mt@$*$6lcsMsU>miWDUeTt?-a)tSPRjy9&vOLF zZ-Qwd(4gc#Y8Qf}JcR23WhhW+-6>V8*(kz66Km-bl5+95C^%dMh^{aZzT%nqpmziL zf@4eSl_>nd<|OS;@_`E~VV|F|7Rur32mX~e^eI4{rqfMjdOZ>qY);tqC6h6}I~Y$K zl){3scWh2#$lA~z8d*=$X1WO(x{|R1J>f%33_au;=2Sv+NHv5LWP(jl#=1C3#|z}x zAF|l)<_tAWPDzo!(=8Wv*_=AVK_)SBou4HTh@OyRfjIyTmzl(Gh};$`#aTlp6Nkjj z;vagnMZWUzr9$ph)YQSRFp&`tGt?Dzgj9E;EK=n{b2P=Gx{{;gfp&Pz!y`qD)QDS& z=@b%(JJGD98@R;O#B_!B9>TCiB;vF^d!2HQkD3IBF`Y=$mq@0q^YK_t74X?-(^wNF z$Y6FC*xUlP7jqhj3dKdy@sD|h2g1EqJZcsOHA7Svom${DO+sz9xrXzzRU7R>wYn$8 zZM$}xu3PJoU0!ZDhy7-`q4p@_TT|_;3!~E{sT|&;Tr<;iOd&@rB&HBZJhFVFObjiT z`6M%)T_}#Mq9-QHqoF6x7@1_c#xYZ8b>PVZiqWMSELH@QO%!{*O+5c?&+||cgwD-| z0`mr_87{#w=-|v)@ks?r%0^AvO4aJ z^~I(t>jh4$1nmhXHXiPn*mQe^QbO`hC^{8M&r`_P@K~g+c{t0^b%&`aJIb5A&x&zyYE5CX!49jpxz6TXAyIaW+mVZg5ZGW`yFKRE&tpkneaICjii4_Qlxcq z)4T&o-pvX6jRZm9q({A+6g0mcaEhMfd8I70IDv=@3zbUmy3uUZNH zcL^Q0y`jqKX)`g7QE2Dx0`WpM-iO#3E^(38k(FQ~jO?nIj>9s**?mtwv`xt~`>{Ro zcQl>SYYnuUYE#AZra)3%G}>TI@bg}w9{6DxQkwKoAhCvL{KzAny_%9Ge>pdrW(QAWG2<1vmf1T#>ERfO%zk=4FWWb24mk&zI>~jg!P%ik z~=U_>d{F40|na8Nuo5Ak#@NJ>gf|)DCttFPMD`jq2s#WtH+C zYy82&5MC!cURvYRqHAS!#NL&xZw-9{@50M3K5m9D{M?+ZDyhAQF{Q;>@kH!sV}%L_ zYP0UH#La;?c_20Q(?g8pi}45>2h#GCoi{v5ONVOh;z=q9)1g+{ijHMOT&A4|m4z<% zTqvLVJbs3fcA(Zto+Ksjg-M>~!YDn})IsxxHC5>$m72%ncR0jUsBr&_KzJ)V1X* zt+NtBN8<&}Qq!zWY0IKlUeAnRz|iUAqGZWUIwsz9 zyc2IUlHZ|As_A6QR)Pcda;iDup;_lZ{UEkY-L}ek@gnB~*$I;6g=vMbb6Ps|nLWB4 zJ(tEq!c5y?WEa>2c?k`v&4OE^>fDl1X(#QD@KW8Q+0b@#En)%Akz~sP#PHHPG1U{F z|E;&1umOrEW%G{RU`)3lxQ}~bk#Pbhl(G6Yczz@_1)6yAz@(Xnc&24HgWrfHtx+#} z!(wf*nT?g)yo94FYeXP@SYdn1S)@lZiJ z+Z@eYS$@gMN1I%=S6Vyi4$si`8)@AhJ6#327w$pb#dMzP<@WG&7Q{X>JK&i;YGY7P zGH7upDkGX`TFB>jbYCqfYNA4R9}|-tvXCbmrq|k3oyqg$Wc#!dwNt`Fl zrSi(dS{1M?YWoytVE>j9(4<1iK!*#lK>gCaP4XUwxNLkt7pS5!KRrlRBWn?-CGrre zb->$NXtA*jb0-tSR3;1??VJQn4uLMsK_=3vW)1zZQo+0yNys!YC(bemJ7@YkI_O9o}&2Yej!vJA4 zszLAc)W}YG5Buc5wP(SDCw#{zKV&GQ%!FMm0h^ggm5AFC3J&25@=QQr*Blwj3Y!Ud zOxT;Bn8-AycNf#7x|CSxV9fz!Bh5>UxdPkICwXQoeI4u zU2 z8ee6H({+>66?)TDxl*OqA4a1d_HPN^jSkaVnhwo2=Gd6@t_+Z10Me)pY>zia5E`=W zapNdjFp2&k!7De!`oo8Gk?=8ID6qI-^<=_QvnN3nSiu^mum6=-W(K3cB$H)~#ch?V z2Q>~X@&>lZY60FiJlG@}sSRIvu$b3r?#P&!F6ACai;O9R1VPtHNQ1SJhH)MITB}-* zIJ~Ap!`4mB*6t6!idWU~$dT1QKWHAVv^6;3+@E`9N$xc3FxTOoIJ(s&)V6dQFufPf zYdeZw(rG8tW6xN+{Zhgo$*R>=5VVAKS<6ul@##c|ZaCMzoy(p@qS-QA1mkto(t=ph zN`Thngg9D8|KSXwS3k9GtXdePE2Y!rI%2+!(~D2)SB_8D(`gxcuVQtbS06xw)8)~j zca?`cAs8u4jd=A#>&A(hUg_MC+@W(UQ{ROfD7r?j4(W+O^Cv&`d`dYto z0$-*|MX#RT;YdH_6^`|li6QE}HvBek(mUq52mWBaso#`0?SDVYw`(zP%P;hcxjr;ce{*@Z6}SBV zA&hnMI{IHeGoc||xs3fWav1DF`eK)#i*n|=@gPH~S9CA^1z7#?-`%v!-+s8(Z?1EV zVnA=ond_&W@>`G4^5(kX73%+jQ-1OUEpM*7_Ums{AINQfU+0vseq04MSD3#c5A4VQ zUqShxnH%v9*Jt-NlzKJoHt4sU^4q?m<<0dW^9M#z!1TWWv{IPs6|JTwy}I)M)+xVzt`1mp?Q$xN_NM3kjmq=i zfMy@UpSkbR@^g>Ys~Kl{Hyl4kj@J`I){!aSit?=}Z|IWB?@#4fCR2U^$}d3q8=Ve! z`OT%mJa$d_ZA(<{ZKfSitfHa(hV3V)tX==i9;KXlJoxZU>tpIS*CXk^UEa%~rUiOk z!Oq?Km~!TtcFM0f4du`dBd1|u%9-D-PWefcpG5fvLx`(ezKhB;X7slay3}K%EhmIR%`b-v0+PdCDsQ literal 0 HcmV?d00001 diff --git a/build/lib/norch/nn/__init__.py b/build/lib/norch/nn/__init__.py new file mode 100644 index 0000000..8f7997b --- /dev/null +++ b/build/lib/norch/nn/__init__.py @@ -0,0 +1,4 @@ +from .modules import * +from .activation import * +from .loss import * +from .functional import * \ No newline at end of file diff --git a/build/lib/norch/nn/activation.py b/build/lib/norch/nn/activation.py new file mode 100644 index 0000000..a1361f6 --- /dev/null +++ b/build/lib/norch/nn/activation.py @@ -0,0 +1,30 @@ +from .module import Module +from . import functional as F +import math + +class Activation(Module): + """ + Abstract classes for activations + """ + def __init__(self): + super(Activation, self).__init__() + + def forward(self, x): + raise NotImplementedError + + +class Sigmoid(Activation): + def __init__(self): + super().__init__() + + def forward(self, x): + return F.sigmoid(x) + +class Softmax(Activation): + def __init__(self, dim): + super(Softmax, self).__init__() + + self.dim = dim + + def forward(self, x): + return F.softmax(x, self.dim) \ No newline at end of file diff --git a/build/lib/norch/nn/functional.py b/build/lib/norch/nn/functional.py new file mode 100644 index 0000000..29fb68d --- /dev/null +++ b/build/lib/norch/nn/functional.py @@ -0,0 +1,32 @@ +import math +import norch +import numpy as np +from norch.autograd.functions import * + +def sigmoid(x): + z = x.sigmoid() + return z + +def softmax(x, dim=None): + if dim is not None and dim < 0: + dim = x.ndim + dim + + x_max = x.max(axis=dim, keepdim=True) + exp_x = math.e ** (x - x_max) + + if dim is not None: + sum_exp_x = exp_x.sum(axis=dim, keepdim=True) + exp_x.zeros_like() + return exp_x / sum_exp_x + else: + sum_exp_x = exp_x.sum() + return exp_x / sum_exp_x + +def one_hot_encode(x, num_classes): + one_hot = [[0] * num_classes for _ in range(x.numel)] + + # Set the appropriate elements to 1 + for i in range(x.numel): + target_idx = int(x[i]) + one_hot[i][target_idx] = 1 + + return norch.Tensor(one_hot) \ No newline at end of file diff --git a/build/lib/norch/nn/loss.py b/build/lib/norch/nn/loss.py new file mode 100644 index 0000000..6c0abea --- /dev/null +++ b/build/lib/norch/nn/loss.py @@ -0,0 +1,93 @@ +from .module import Module +from norch.autograd.functions import * +import norch +from abc import ABC + +class Loss(Module, ABC): + "Abstract class for loss functions" + + def __init__(self): + super().__init__() + + def forward(self, predictions, labels): + raise NotImplementedError + + def __call__(self, *inputs): + return self.forward(*inputs) + + +class MSELoss(Loss): + def __init__(self): + super().__init__() + + def forward(self, predictions, target): + assert target.shape == predictions.shape, \ + "Labels and predictions shape does not match: {} and {}".format(target.shape, predictions.shape) + + cost = ((predictions - target) ** 2).sum() / predictions.numel + return cost + + +class CrossEntropyLoss(Loss): + def __init__(self): + super().__init__() + + def forward(self, input, target): + assert isinstance(input, norch.Tensor), \ + "Cross entropy argument 'input' must be Tensor, not {}".format(type(input)) + + assert isinstance(target, norch.Tensor), \ + "Cross entropy argument 'target' must be Tensor, not {}".format(type(target)) + if input.ndim > 2: + input = input.squeeze(-1) + + if input.ndim == 1: + if target.numel == 1: + num_classes = input.shape[0] + target = norch.one_hot_encode(target, num_classes).to(target.device) + + logits = norch.softmax(input, dim=0) + target = target.reshape(logits.shape) + cost = -(logits.log() * target).sum() + + else: + # target -> class probabilities (one-hot encoded) + assert target.shape == input.shape, \ + "Input and target shape does not match: {} and {}".format(input.shape, target.shape) + logits = norch.softmax(input, dim=0) + target = target.reshape(logits.shape) + cost = -(logits.log() * target).sum() + + + elif input.ndim == 2: + if target.ndim > 1: + target = target.squeeze(-1) + # batched + if target.ndim == 1: + # target -> Ground truth class indices: + num_classes = input.shape[1] + + target = norch.one_hot_encode(target, num_classes).to(target.device) + + batch_size = input.shape[0] + logits = norch.softmax(input, dim=1) + target = target.reshape(logits.shape) + cost = -(logits.log() * target).sum() / batch_size + + else: + # target -> class probabilities (one-hot encoded) + assert target.shape == input.shape, \ + "Input and target shape does not match: {} and {}".format(input.shape, target.shape) + + batch_size = input.shape[0] + logits = norch.softmax(input, dim=1) + target = target.reshape(logits.shape) + cost = -(logits.log() * target).sum() / batch_size + + if input.requires_grad: + cost.grad_fn = CrossEntropyLossBackward(input, target) + + return cost + + + diff --git a/build/lib/norch/nn/module.py b/build/lib/norch/nn/module.py new file mode 100644 index 0000000..0135bb6 --- /dev/null +++ b/build/lib/norch/nn/module.py @@ -0,0 +1,107 @@ +from .parameter import Parameter +from collections import OrderedDict +from abc import ABC +import pickle +import json +import inspect +import warnings + +class Module(ABC): + """ + Abstract class for modules + """ + def __init__(self): + self._modules = OrderedDict() + self._params = OrderedDict() + self._grads = OrderedDict() + self.training = True + + def forward(self, *inputs, **kwargs): + raise NotImplementedError + + def __call__(self, *inputs, **kwargs): + return self.forward(*inputs, **kwargs) + + def train(self): + self.training = True + for param in self.parameters(): + param.requires_grad = True + + def eval(self): + self.training = False + for param in self.parameters(): + param.requires_grad = False + + def parameters(self): + for name, value in inspect.getmembers(self): + if isinstance(value, Parameter): + yield self, name, value + elif isinstance(value, Module): + yield from value.parameters() + + def modules(self): + yield from self._modules.values() + + def gradients(self): + for module in self.modules(): + yield module._grads + + def zero_grad(self): + for _, _, parameter in self.parameters(): + parameter.zero_grad() + + def to(self, device): + for module, name, _ in self.parameters(): + parameter = getattr(module, name) + parameter = parameter.to(device) + setattr(module, name, parameter) + + return self + + def state_dict(self): + state = OrderedDict() + for i, param in enumerate(self.parameters()): + state[f'param{i}'] = param.tolist() + return state + + def load_state(self, state_dict): + for i, param in self.parameters(): + data = state_dict[f'param{i}'] + if param.shape != data.shape: + warnings.warn(f"The 'state_dict' shape does not match model's parameter shape. " + f"Got {data.shape}, expected {param.shape}.") + param.data = Parameter(data=data) + + def save(self, filename='model.pickle'): + with open(filename, 'wb') as f: + pickle.dump(self, f) + + def save_dict(self, filename='state_dict.json'): + state = self.state_dict() + with open(filename, 'w') as f: + json.dump(state, f) + + def inner_repr(self): + return "" + + def __repr__(self): + string = f"{self.get_name()}(" + tab = " " + modules = self._modules + if modules == {}: + string += f'\n{tab}(parameters): {self.inner_repr()}' + else: + for key, module in modules.items(): + string += f"\n{tab}({key}): {module.get_name()}({module.inner_repr()})" + return f'{string}\n)' + + def get_name(self): + return self.__class__.__name__ + + def __setattr__(self, key, value): + self.__dict__[key] = value + + if isinstance(value, Module): + self._modules[key] = value + elif isinstance(value, Parameter): + self._params[key] = value \ No newline at end of file diff --git a/build/lib/norch/nn/modules/__init__.py b/build/lib/norch/nn/modules/__init__.py new file mode 100644 index 0000000..5c73a2e --- /dev/null +++ b/build/lib/norch/nn/modules/__init__.py @@ -0,0 +1 @@ +from .linear import * \ No newline at end of file diff --git a/build/lib/norch/nn/modules/linear.py b/build/lib/norch/nn/modules/linear.py new file mode 100644 index 0000000..b90d845 --- /dev/null +++ b/build/lib/norch/nn/modules/linear.py @@ -0,0 +1,26 @@ +from ..module import Module +from ..parameter import Parameter + +class Linear(Module): + def __init__(self, input_dim, output_dim, bias=True): + super().__init__() + self.input_dim = input_dim + self.output_dim = output_dim + self.weight = Parameter(shape=[self.output_dim, self.input_dim]) + + if bias: + self.bias = Parameter(shape=[self.output_dim, 1]) + else: + self.bias = None + + def forward(self, x): + if self.bias: + z = self.weight @ x + self.bias + else: + z = self.weight @ x + + return z + + def inner_repr(self): + return f"input_dim={self.input_dim}, output_dim={self.output_dim}, " \ + f"bias={True if self.bias is not None else False}" \ No newline at end of file diff --git a/build/lib/norch/nn/parameter.py b/build/lib/norch/nn/parameter.py new file mode 100644 index 0000000..2324a1a --- /dev/null +++ b/build/lib/norch/nn/parameter.py @@ -0,0 +1,11 @@ +from norch.tensor import Tensor +from norch.utils import utils +import random + +class Parameter(Tensor): + """ + A parameter is a trainable tensor. + """ + def __init__(self, shape): + data = utils.generate_random_list(shape=shape) + super().__init__(data, requires_grad=True) \ No newline at end of file diff --git a/build/lib/norch/norchvision/__init__.py b/build/lib/norch/norchvision/__init__.py new file mode 100644 index 0000000..a6ccfac --- /dev/null +++ b/build/lib/norch/norchvision/__init__.py @@ -0,0 +1 @@ +from .datasets import * \ No newline at end of file diff --git a/build/lib/norch/norchvision/datasets/__init__.py b/build/lib/norch/norchvision/datasets/__init__.py new file mode 100644 index 0000000..fe34b11 --- /dev/null +++ b/build/lib/norch/norchvision/datasets/__init__.py @@ -0,0 +1 @@ +from .mnist import * \ No newline at end of file diff --git a/build/lib/norch/norchvision/datasets/mnist.py b/build/lib/norch/norchvision/datasets/mnist.py new file mode 100644 index 0000000..ff11edf --- /dev/null +++ b/build/lib/norch/norchvision/datasets/mnist.py @@ -0,0 +1,90 @@ +import gzip +import os +import norch +from norch.utils.data import Dataset +import numpy as np + +class MNIST(Dataset): + """ + Loads training, validation, and test partitions of the mnist dataset + (http://yann.lecun.com/exdb/mnist/). If the data is not already contained in data_dir, it will + try to download it. + + This dataset contains 60000 training examples, and 10000 test examples of handwritten digits + in {0, ..., 9} and corresponding labels. Each handwritten image has an "original" dimension of + 28x28x1, and is stored row-wise as a string of 784x1 bytes. Pixel values are in range 0 to 255 + (inclusive). + + Args: + data_dir: String. Relative or absolute path of the dataset. + devel_size: Integer. Size of the development (validation) dataset partition. + + Returns: + X_train: float64 numpy array with shape [784, 60000-devel_size] with values in [0, 1]. + Y_train: uint8 numpy array with shape [60000-devel_size]. Labels. + X_devel: float64 numpy array with shape [784, devel_size] with values in [0, 1]. + Y_devel: uint8 numpy array with shape [devel_size]. Labels. + X_test: float64 numpy array with shape [784, 10000] with values in [0, 1]. + Y_test: uint8 numpy array with shape [10000]. Labels. + """ + + urls = ['https://ossci-datasets.s3.amazonaws.com/mnist/train-images-idx3-ubyte.gz', + 'https://ossci-datasets.s3.amazonaws.com/mnist/train-labels-idx1-ubyte.gz', + 'https://ossci-datasets.s3.amazonaws.com/mnist/t10k-images-idx3-ubyte.gz', + 'https://ossci-datasets.s3.amazonaws.com/mnist/t10k-labels-idx1-ubyte.gz',] + name = 'mnist-data-py' + dirname = 'mnist' + + def __init__(self, path_data, path_label, transform=None, target_transform=None): + self.data = self._load_mnist(path_data, header_size=16).reshape((-1, 28, 28)) + self.labels = self._load_mnist(path_label, header_size=8) + self.transform = transform + self.target_transform = target_transform + + def _load_mnist(self, path, header_size): + with gzip.open(path, 'rb') as f: + data = np.frombuffer(f.read(), np.uint8, offset=header_size) + return np.asarray(data, dtype=np.uint8) + + + @classmethod + def splits(cls, root='.data', train_data='train-images-idx3-ubyte.gz', train_label='train-labels-idx1-ubyte.gz', + test_data='t10k-images-idx3-ubyte.gz', test_label='t10k-labels-idx1-ubyte.gz', **kwargs): + r""" + Loads training and test partitions of the [mnist dataset](https://www.cs.toronto.edu/~kriz/cifar.html). If + the data is not already contained in the ``root`` folder, it will download it. + + Args: + root (str): relative or absolute path of the dataset. + + Returns: + tuple(Dataset): training and testing datasets + """ + path = os.path.join(root, cls.dirname, cls.name) + if not os.path.isdir(path): + path = cls.download(root) + train_data = os.path.join(path, train_data) + train_label = os.path.join(path, train_label) + test_data = os.path.join(path, test_data) + test_label = os.path.join(path, test_label) + return MNIST(train_data, train_label, **kwargs), MNIST(test_data, test_label, **kwargs) + + def __getitem__(self, item): + data = self.data[item].tolist() + label = self.labels[item].tolist() + + if self.transform is not None: + data = self.transform(data) + + if self.target_transform is not None: + label = self.target_transform(label) + + + + return data, label + + def __setitem__(self, key, value): + self.data[key], self.labels[key] = value + + def __len__(self): + return len(self.data) diff --git a/build/lib/norch/norchvision/transforms.py b/build/lib/norch/norchvision/transforms.py new file mode 100644 index 0000000..03dafad --- /dev/null +++ b/build/lib/norch/norchvision/transforms.py @@ -0,0 +1,22 @@ +import norch + +class ToTensor: + def __call__(self, x): + return norch.Tensor(x) + +class Reshape: + def __init__(self, shape): + self.shape = shape + + def __call__(self, x): + return x.reshape(self.shape) + +class Sequential: + def __init__(self, transforms): + self.transforms = transforms + + def __call__(self, x): + for transform in self.transforms: + x = transform(x) + return x + diff --git a/build/lib/norch/optim/__init__.py b/build/lib/norch/optim/__init__.py new file mode 100644 index 0000000..c5baa22 --- /dev/null +++ b/build/lib/norch/optim/__init__.py @@ -0,0 +1 @@ +from .optimizers import * \ No newline at end of file diff --git a/build/lib/norch/optim/optimizer.py b/build/lib/norch/optim/optimizer.py new file mode 100644 index 0000000..fbbd537 --- /dev/null +++ b/build/lib/norch/optim/optimizer.py @@ -0,0 +1,22 @@ +from abc import ABC +from norch.tensor import Tensor + +class Optimizer(ABC): + """ + Abstract class for optimizers + """ + + def __init__(self, parameters): + if isinstance(parameters, Tensor): + raise TypeError("parameters should be an iterable but got {}".format(type(parameters))) + elif isinstance(parameters, dict): + parameters = parameters.values() + + self.parameters = list(parameters) + + def step(self): + raise NotImplementedError + + def zero_grad(self): + for module, name, parameter in self.parameters: + parameter.zero_grad() \ No newline at end of file diff --git a/build/lib/norch/optim/optimizers/__init__.py b/build/lib/norch/optim/optimizers/__init__.py new file mode 100644 index 0000000..bd076ac --- /dev/null +++ b/build/lib/norch/optim/optimizers/__init__.py @@ -0,0 +1 @@ +from .sgd import * \ No newline at end of file diff --git a/build/lib/norch/optim/optimizers/sgd.py b/build/lib/norch/optim/optimizers/sgd.py new file mode 100644 index 0000000..1d785c7 --- /dev/null +++ b/build/lib/norch/optim/optimizers/sgd.py @@ -0,0 +1,27 @@ +from ..optimizer import Optimizer +from norch.tensor import Tensor +import time + +class SGD(Optimizer): + def __init__(self, parameters, lr=1e-1, momentum=0): + super().__init__(parameters) + self.lr = lr + self.momentum = momentum + self._cache = {'velocity': [p.zeros_like() for (_, _, p) in self.parameters]} + + def step(self): + for i, (module, name, _) in enumerate(self.parameters): + parameter = getattr(module, name) + + velocity = self._cache['velocity'][i] + + velocity = self.momentum * velocity - self.lr * parameter.grad + + updated_parameter = parameter + velocity + + setattr(module, name, updated_parameter) + + self._cache['velocity'][i] = velocity + + parameter.detach() + velocity.detach() diff --git a/build/lib/norch/tensor.py b/build/lib/norch/tensor.py new file mode 100644 index 0000000..2e80e72 --- /dev/null +++ b/build/lib/norch/tensor.py @@ -0,0 +1,1033 @@ +import ctypes +import os +from .autograd.functions import * + +class CTensor(ctypes.Structure): + _fields_ = [ + ('data', ctypes.POINTER(ctypes.c_float)), + ('strides', ctypes.POINTER(ctypes.c_int)), + ('shape', ctypes.POINTER(ctypes.c_int)), + ('ndim', ctypes.c_int), + ('size', ctypes.c_int), + ('device', ctypes.c_char_p) + ] + +class Tensor: + module_dir = os.path.dirname(os.path.abspath(__file__)) + _C = ctypes.CDLL(os.path.join(module_dir, "libtensor.so")) + + def __init__(self, data=None, device="cpu", requires_grad=False): + + if data != None: + if isinstance(data, (float, int)): + data = [data] + + data, shape = self.flatten(data) + + self.shape = shape.copy() + + self.data_ctype = (ctypes.c_float * len(data))(*data.copy()) + self.shape_ctype = (ctypes.c_int * len(shape))(*shape.copy()) + self.ndim_ctype = ctypes.c_int(len(shape)) + self.device_ctype = device.encode('utf-8') + + self.ndim = len(shape) + self.device = device + + self.numel = 1 + for s in self.shape: + self.numel *= s + + self.requires_grad = requires_grad + self.grad = None + self.grad_fn = None + + Tensor._C.create_tensor.argtypes = [ctypes.POINTER(ctypes.c_float), ctypes.POINTER(ctypes.c_int), ctypes.c_int, ctypes.c_char_p] + Tensor._C.create_tensor.restype = ctypes.POINTER(CTensor) + + + self.tensor = Tensor._C.create_tensor( + self.data_ctype, + self.shape_ctype, + self.ndim_ctype, + self.device_ctype + ) + + else: + self.tensor = None, + self.shape = None, + self.ndim = None, + self.device = device + self.requires_grad = None + self.grad = None + self.grad_fn = None + + def flatten(self, nested_list): + def flatten_recursively(nested_list): + flat_data = [] + shape = [] + if isinstance(nested_list, list): + for sublist in nested_list: + inner_data, inner_shape = flatten_recursively(sublist) + flat_data.extend(inner_data) + shape.append(len(nested_list)) + shape.extend(inner_shape) + else: + flat_data.append(nested_list) + return flat_data, shape + + flat_data, shape = flatten_recursively(nested_list) + return flat_data, shape + + def ones_like(self): + + Tensor._C.ones_like_tensor.argtypes = [ctypes.POINTER(CTensor)] + Tensor._C.ones_like_tensor.restype = ctypes.POINTER(CTensor) + Tensor._C.ones_like_tensor(self.tensor) + + result_tensor_ptr = Tensor._C.ones_like_tensor(self.tensor) + + result_data = Tensor() + result_data.tensor = result_tensor_ptr + result_data.shape = self.shape.copy() + result_data.ndim = self.ndim + result_data.device = self.device + result_data.numel = self.numel + + return result_data + + def zeros_like(self): + + Tensor._C.zeros_like_tensor.argtypes = [ctypes.POINTER(CTensor)] + Tensor._C.zeros_like_tensor.restype = ctypes.POINTER(CTensor) + Tensor._C.zeros_like_tensor(self.tensor) + + result_tensor_ptr = Tensor._C.zeros_like_tensor(self.tensor) + + result_data = Tensor() + result_data.tensor = result_tensor_ptr + result_data.shape = self.shape.copy() + result_data.ndim = self.ndim + result_data.device = self.device + result_data.numel = self.numel + + return result_data + + def reshape(self, new_shape): + # Calculate the total number of elements in the tensor + total_elements = self.numel + + # Check for the presence of -1 in new_shape + if new_shape.count(-1) > 1: + raise ValueError("Only one dimension can be inferred (set to -1).") + + inferred_dim = None + known_dims_product = 1 + for dim in new_shape: + if dim == -1: + inferred_dim = dim + else: + known_dims_product *= dim + + # Calculate the inferred dimension if -1 is present + if inferred_dim == -1: + inferred_dim_size = total_elements // known_dims_product + new_shape = [inferred_dim_size if dim == -1 else dim for dim in new_shape] + + new_shape_ctype = (ctypes.c_int * len(new_shape))(*new_shape) + new_ndim_ctype = ctypes.c_int(len(new_shape)) + + Tensor._C.reshape_tensor.argtypes = [ctypes.POINTER(CTensor), ctypes.POINTER(ctypes.c_int), ctypes.c_int] + Tensor._C.reshape_tensor.restype = ctypes.POINTER(CTensor) + result_tensor_ptr = Tensor._C.reshape_tensor(self.tensor, new_shape_ctype, new_ndim_ctype) + + result_data = Tensor() + result_data.tensor = result_tensor_ptr + result_data.shape = new_shape.copy() + result_data.ndim = len(new_shape) + result_data.device = self.device + result_data.numel = self.numel + + result_data.requires_grad = self.requires_grad + if result_data.requires_grad: + result_data.grad_fn = ReshapeBackward(self) + + return result_data + + def unsqueeze(self, dim): + if dim < 0: + dim = self.ndim + dim + 1 + + # Ensure the dimension is valid + if dim > self.ndim: + raise ValueError("Dimension out of range (expected to be in range of [0, {0}], but got {1})".format(self.ndim, dim)) + + # Create the new shape with an extra dimension of size 1 + new_shape = self.shape[:dim] + [1] + self.shape[dim:] + + return self.reshape(new_shape) + + def squeeze(self, dim=None): + if dim is not None: + if dim < 0: + dim = self.ndim + dim + + # Ensure the dimension is valid + if dim >= self.ndim or dim < 0: + raise ValueError("Dimension out of range (expected to be in range of [0, {0}), but got {1})".format(self.ndim, dim)) + + # Only squeeze the specified dimension if its size is 1 + if self.shape[dim] != 1: + return self + #raise ValueError("Dimension {0} does not have size 1 and cannot be squeezed".format(dim)) + + # Create the new shape without the specified dimension + new_shape = self.shape[:dim] + self.shape[dim+1:] + else: + # Create the new shape by removing all dimensions of size 1 + new_shape = [s for s in self.shape if s != 1] + + return self.reshape(new_shape) + + def to(self, device): + device = str(device) + + self.device = device + self.device_ctype = self.device.encode('utf-8') + + Tensor._C.to_device.argtypes = [ctypes.POINTER(CTensor), ctypes.c_char_p] + Tensor._C.to_device.restype = None + Tensor._C.to_device(self.tensor, self.device_ctype) + + return self + + def backward(self, gradient=None): + if not self.requires_grad: + return + + if gradient is None: + if self.shape == [1]: + gradient = Tensor([1]).to(self.device) + else: + raise RuntimeError("Gradient argument must be specified for non-scalar tensors.") + + + stack = [(self, gradient)] + visited = set() + + while stack: + tensor, grad = stack.pop() + + if tensor.grad is None: + tensor.grad = grad + else: + tensor.grad += grad + + # Propagate gradients to inputs if not a leaf tensor + if tensor.grad_fn is not None: + grads = tensor.grad_fn.backward(grad) + for tensor, grad in zip(tensor.grad_fn.input, grads): + if isinstance(tensor, Tensor) and tensor not in visited: + stack.append((tensor, grad)) + visited.add(tensor) + + def zero_grad(self): + self.grad = None + + def detach(self): + self.grad = None + self.grad_fn = None + + def __getitem__(self, indices): + if isinstance(indices, int): + indices = [indices] + if len(indices) != self.ndim: + raise ValueError("Number of indices must match the number of dimensions") + + Tensor._C.get_item.argtypes = [ctypes.POINTER(CTensor), ctypes.POINTER(ctypes.c_int)] + Tensor._C.get_item.restype = ctypes.c_float + + indices = (ctypes.c_int * len(indices))(*indices) + value = Tensor._C.get_item(self.tensor, indices) + + return value + + def __str__(self): + def print_recursively(tensor, depth, index): + if depth == tensor.ndim - 1: + result = "" + for i in range(tensor.shape[-1]): + index[-1] = i + result += str(tensor[tuple(index)]) + ", " + return result.strip() + else: + result = "" + if depth > 0: + result += "\n" + " " * ((depth - 1) * 4) + for i in range(tensor.shape[depth]): + index[depth] = i + result += "[" + result += print_recursively(tensor, depth + 1, index) + "]," + if i < tensor.shape[depth] - 1: + result += "\n" + " " * (depth * 4) + return result.strip(",") + + index = [0] * self.ndim + result = "tensor([" + result += print_recursively(self, 0, index) + result += f"""], device="{self.device}", requires_grad={self.requires_grad})""" + return result + + def __repr__(self): + return self.__str__() + + def __add__(self, other): + if isinstance(other, (int, float)): + other = other * self.ones_like() + + broadcasted_shape_add = [] + + # Function to determine if broadcasting is needed and get the broadcasted shape + def broadcast_shape(shape1, shape2): + if shape1 == shape2: + return shape1, False + + max_len = max(len(shape1), len(shape2)) + shape1 = [1] * (max_len - len(shape1)) + shape1 + shape2 = [1] * (max_len - len(shape2)) + shape2 + + + for dim1, dim2 in zip(shape1, shape2): + if dim1 != dim2 and dim1 != 1 and dim2 != 1: + raise ValueError("Shapes are not compatible for broadcasting") + broadcasted_shape_add.append(max(dim1, dim2)) + return broadcasted_shape_add, True + + broadcasted_shape_add, needs_broadcasting = broadcast_shape(self.shape, other.shape) + + if needs_broadcasting: + # Call add_broadcasted_tensor if broadcasting is needed + if other.ndim == self.ndim - 1: + other = other.reshape([1] + other.shape) + + elif self.ndim == other.ndim - 1: + self = self.reshape([1] + self.shape) + + + + Tensor._C.add_broadcasted_tensor.argtypes = [ctypes.POINTER(CTensor), ctypes.POINTER(CTensor)] + Tensor._C.add_broadcasted_tensor.restype = ctypes.POINTER(CTensor) + + result_tensor_ptr = Tensor._C.add_broadcasted_tensor(self.tensor, other.tensor) + + result_data = Tensor() + result_data.tensor = result_tensor_ptr + result_data.shape = broadcasted_shape_add.copy() + result_data.ndim = len(broadcasted_shape_add) + + result_data.device = self.device + result_data.numel = 1 + for s in result_data.shape: + result_data.numel *= s + + result_data.requires_grad = self.requires_grad or other.requires_grad + if result_data.requires_grad: + result_data.grad_fn = AddBroadcastedBackward(self, other) + + else: + # Call add_tensor if shapes are identical + Tensor._C.add_tensor.argtypes = [ctypes.POINTER(CTensor), ctypes.POINTER(CTensor)] + Tensor._C.add_tensor.restype = ctypes.POINTER(CTensor) + + result_tensor_ptr = Tensor._C.add_tensor(self.tensor, other.tensor) + + result_data = Tensor() + result_data.tensor = result_tensor_ptr + result_data.shape = self.shape.copy() + result_data.ndim = self.ndim + + result_data.device = self.device + result_data.numel = self.numel # Update this to calculate the correct number of elements if broadcasting + + result_data.requires_grad = self.requires_grad or other.requires_grad + if result_data.requires_grad: + result_data.grad_fn = AddBackward(self, other) + + return result_data + + + def __radd__(self, other): + if isinstance(other, (int, float)): + other = other * self.ones_like() + + if self.shape != other.shape: + raise ValueError("Tensors must have the same shape for addition") + + Tensor._C.add_tensor.argtypes = [ctypes.POINTER(CTensor), ctypes.POINTER(CTensor)] + Tensor._C.add_tensor.restype = ctypes.POINTER(CTensor) + + result_tensor_ptr = Tensor._C.add_tensor(other.tensor, self.tensor) + + result_data = Tensor() + result_data.tensor = result_tensor_ptr + result_data.shape = self.shape.copy() + result_data.ndim = self.ndim + result_data.device = self.device + result_data.numel = self.numel + + result_data.requires_grad = self.requires_grad or other.requires_grad + if result_data.requires_grad: + result_data.grad_fn = AddBackward(other, self) + + return result_data + + def __sub__(self, other): + if isinstance(other, (int, float)): + other = other * self.ones_like() + + broadcasted_shape_sub = [] + + # Function to determine if broadcasting is needed and get the broadcasted shape + def broadcast_shape(shape1, shape2): + if shape1 == shape2: + return shape1, False + + max_len = max(len(shape1), len(shape2)) + shape1 = [1] * (max_len - len(shape1)) + shape1 + shape2 = [1] * (max_len - len(shape2)) + shape2 + + + for dim1, dim2 in zip(shape1, shape2): + if dim1 != dim2 and dim1 != 1 and dim2 != 1: + raise ValueError("Shapes are not compatible for broadcasting") + broadcasted_shape_sub.append(max(dim1, dim2)) + return broadcasted_shape_sub, True + + broadcasted_shape_sub, needs_broadcasting = broadcast_shape(self.shape, other.shape) + + if needs_broadcasting: + if other.ndim == self.ndim - 1: + other = other.reshape([1] + other.shape) + + elif self.ndim == other.ndim - 1: + self = self.reshape([1] + self.shape) + + # Call add_broadcasted_tensor if broadcasting is needed + Tensor._C.sub_broadcasted_tensor.argtypes = [ctypes.POINTER(CTensor), ctypes.POINTER(CTensor)] + Tensor._C.sub_broadcasted_tensor.restype = ctypes.POINTER(CTensor) + + result_tensor_ptr = Tensor._C.sub_broadcasted_tensor(self.tensor, other.tensor) + + result_data = Tensor() + result_data.tensor = result_tensor_ptr + result_data.shape = broadcasted_shape_sub.copy() + result_data.ndim = len(broadcasted_shape_sub) + + result_data.device = self.device + result_data.numel = self.numel # Update this to calculate the correct number of elements if broadcasting + + result_data.requires_grad = self.requires_grad or other.requires_grad + if result_data.requires_grad: + result_data.grad_fn = SubBroadcastedBackward(self, other) + + else: + # Call add_tensor if shapes are identical + Tensor._C.sub_tensor.argtypes = [ctypes.POINTER(CTensor), ctypes.POINTER(CTensor)] + Tensor._C.sub_tensor.restype = ctypes.POINTER(CTensor) + + result_tensor_ptr = Tensor._C.sub_tensor(self.tensor, other.tensor) + + result_data = Tensor() + result_data.tensor = result_tensor_ptr + result_data.shape = self.shape.copy() + result_data.ndim = self.ndim + + result_data.device = self.device + result_data.numel = self.numel # Update this to calculate the correct number of elements if broadcasting + + result_data.requires_grad = self.requires_grad or other.requires_grad + if result_data.requires_grad: + result_data.grad_fn = SubBackward(self, other) + + return result_data + + def __rsub__(self, other): + if isinstance(other, (int, float)): + other = other * self.ones_like() + + if self.shape != other.shape: + raise ValueError("Tensors must have the same shape for subtraction") + + Tensor._C.sub_tensor.argtypes = [ctypes.POINTER(CTensor), ctypes.POINTER(CTensor)] + Tensor._C.sub_tensor.restype = ctypes.POINTER(CTensor) + + result_tensor_ptr = Tensor._C.sub_tensor(other.tensor, self.tensor) + + result_data = Tensor() + result_data.tensor = result_tensor_ptr + result_data.shape = self.shape.copy() + result_data.ndim = self.ndim + result_data.device = self.device + result_data.numel = self.numel + + result_data.requires_grad = self.requires_grad or other.requires_grad + if result_data.requires_grad: + result_data.grad_fn = SubBackward(other, self) + + return result_data + + def __mul__(self, other): + if isinstance(other, (int, float)): + result_data = Tensor() + result_data.shape = self.shape.copy() + result_data.ndim = self.ndim + result_data.device = self.device + result_data.numel = self.numel + + Tensor._C.scalar_mul_tensor.argtypes = [ctypes.POINTER(CTensor), ctypes.c_float] + Tensor._C.scalar_mul_tensor.restype = ctypes.POINTER(CTensor) + + result_data.tensor = Tensor._C.scalar_mul_tensor(self.tensor, ctypes.c_float(other)) + + result_data.requires_grad = self.requires_grad + if result_data.requires_grad: + result_data.grad_fn = ScalarMulBackward(self, other) + + return result_data + elif isinstance(other, Tensor): + if self.shape != other.shape: + raise ValueError("Tensors must have the same shape for element-wise multiplication") + + Tensor._C.elementwise_mul_tensor.argtypes = [ctypes.POINTER(CTensor), ctypes.POINTER(CTensor)] + Tensor._C.elementwise_mul_tensor.restype = ctypes.POINTER(CTensor) + + result_tensor_ptr = Tensor._C.elementwise_mul_tensor(self.tensor, other.tensor) + + result_data = Tensor() + result_data.tensor = result_tensor_ptr + result_data.shape = self.shape.copy() + result_data.ndim = self.ndim + result_data.device = self.device + result_data.numel = self.numel + + result_data.requires_grad = self.requires_grad or other.requires_grad + if result_data.requires_grad: + result_data.grad_fn = ElementwiseMulBackward(self, other) + + return result_data + else: + raise TypeError("Unsupported operand type(s) for *: '{}' and '{}'".format(type(self), type(other))) + + def __rmul__(self, other): + return self.__mul__(other) + + def __neg__(self): + return self.__mul__(-1) + + def __pos__(self): + return self + + def __matmul__(self, other): + if self.ndim < 3 and other.ndim == 3: + #broadcasted 2D x 3D matmul + + Tensor._C.broadcasted_batched_matmul_tensor.argtypes = [ctypes.POINTER(CTensor), ctypes.POINTER(CTensor)] + Tensor._C.broadcasted_batched_matmul_tensor.restype = ctypes.POINTER(CTensor) + + result_tensor_ptr = Tensor._C.broadcasted_batched_matmul_tensor(self.tensor, other.tensor) + + result_data = Tensor() + result_data.tensor = result_tensor_ptr + result_data.shape = [other.shape[0], self.shape[0], other.shape[2]] + result_data.ndim = 3 + result_data.device = self.device + result_data.numel = 1 + for s in result_data.shape: + result_data.numel *= s + + + elif self.ndim == 3 and other.ndim == 3: + #broadcasted 3D x 3D matmul + + Tensor._C.batched_matmul_tensor.argtypes = [ctypes.POINTER(CTensor), ctypes.POINTER(CTensor)] + Tensor._C.batched_matmul_tensor.restype = ctypes.POINTER(CTensor) + + result_tensor_ptr = Tensor._C.batched_matmul_tensor(self.tensor, other.tensor) + + result_data = Tensor() + result_data.tensor = result_tensor_ptr + result_data.shape = [other.shape[0], self.shape[1], other.shape[2]] + result_data.ndim = 3 + result_data.device = self.device + result_data.numel = 1 + for s in result_data.shape: + result_data.numel *= s + else: + #2D matmul + if self.ndim != 2 or other.ndim != 2: + raise ValueError("Matrix multiplication requires 2D tensors") + + if self.shape[1] != other.shape[0]: + raise ValueError("Incompatible shapes for matrix multiplication") + + Tensor._C.matmul_tensor.argtypes = [ctypes.POINTER(CTensor), ctypes.POINTER(CTensor)] + Tensor._C.matmul_tensor.restype = ctypes.POINTER(CTensor) + + result_tensor_ptr = Tensor._C.matmul_tensor(self.tensor, other.tensor) + + result_data = Tensor() + result_data.tensor = result_tensor_ptr + result_data.shape = [self.shape[0], other.shape[1]] + result_data.ndim = 2 + result_data.device = self.device + result_data.numel = 1 + for s in result_data.shape: + result_data.numel *= s + + result_data.requires_grad = self.requires_grad or other.requires_grad + if result_data.requires_grad: + result_data.grad_fn = MatmulBackward(self, other) + + return result_data + + def __pow__(self, other): + other = float(other) + Tensor._C.tensor_pow_scalar.argtypes = [ctypes.POINTER(CTensor), ctypes.c_float] + Tensor._C.tensor_pow_scalar.restype = ctypes.POINTER(CTensor) + + result_tensor_ptr = Tensor._C.tensor_pow_scalar(self.tensor, ctypes.c_float(other)) + + result_data = Tensor() + result_data.tensor = result_tensor_ptr + result_data.shape = self.shape.copy() + result_data.ndim = self.ndim + result_data.device = self.device + result_data.numel = self.numel + + result_data.requires_grad = self.requires_grad + if result_data.requires_grad: + result_data.grad_fn = PowBackward(self, other) + + return result_data + + def __rpow__(self, other): + other = float(other) + Tensor._C.scalar_pow_tensor.argtypes = [ctypes.c_float, ctypes.POINTER(CTensor)] + Tensor._C.scalar_pow_tensor.restype = ctypes.POINTER(CTensor) + + result_tensor_ptr = Tensor._C.scalar_pow_tensor(ctypes.c_float(other), self.tensor) + + result_data = Tensor() + result_data.tensor = result_tensor_ptr + result_data.shape = self.shape.copy() + result_data.ndim = self.ndim + result_data.device = self.device + result_data.numel = self.numel + + result_data.requires_grad = self.requires_grad + if result_data.requires_grad: + result_data.grad_fn = PowBackward(other, self) + + return result_data + + def __truediv__(self, other): + if isinstance(other, (int, float)): + other = float(other) + Tensor._C.tensor_div_scalar.argtypes = [ctypes.POINTER(CTensor), ctypes.c_float] + Tensor._C.tensor_div_scalar.restype = ctypes.POINTER(CTensor) + + result_tensor_ptr = Tensor._C.tensor_div_scalar(self.tensor, ctypes.c_float(other)) + + result_data = Tensor() + result_data.tensor = result_tensor_ptr + result_data.shape = self.shape.copy() + result_data.ndim = self.ndim + result_data.device = self.device + result_data.numel = self.numel + + result_data.requires_grad = self.requires_grad + if result_data.requires_grad: + result_data.grad_fn = DivisionBackward(self, other) + + elif isinstance(self, Tensor) and isinstance(other, Tensor): + if other.numel == 1: + return self.__truediv__(other.tensor.contents.data[0]) + + Tensor._C.tensor_div_tensor.argtypes = [ctypes.POINTER(CTensor), ctypes.POINTER(CTensor)] + Tensor._C.tensor_div_tensor.restype = ctypes.POINTER(CTensor) + + result_tensor_ptr = Tensor._C.tensor_div_tensor(self.tensor, other.tensor) + + result_data = Tensor() + result_data.tensor = result_tensor_ptr + result_data.shape = self.shape.copy() + result_data.ndim = self.ndim + result_data.device = self.device + result_data.numel = self.numel + + result_data.requires_grad = self.requires_grad or other.requires_grad + if result_data.requires_grad: + result_data.grad_fn = DivisionBackward(self, other) + + return result_data + + + + def __rtruediv__(self, other): + other = float(other) + + Tensor._C.scalar_div_tensor.argtypes = [ctypes.c_float, ctypes.POINTER(CTensor)] + Tensor._C.scalar_div_tensor.restype = ctypes.POINTER(CTensor) + + result_tensor_ptr = Tensor._C.scalar_div_tensor(ctypes.c_float(other), self.tensor) + + result_data = Tensor() + result_data.tensor = result_tensor_ptr + result_data.shape = self.shape.copy() + result_data.ndim = self.ndim + result_data.device = self.device + result_data.numel = self.numel + + result_data.requires_grad = self.requires_grad + if result_data.requires_grad: + result_data.grad_fn = DivisionBackward(other, self) + + return result_data + + + def equal(self, other): + + if isinstance(other, Tensor) and other.numel == 1: + # other is a single value tensor + other = self.zeros_like() + other + return self.equal(other) + + if not isinstance(other, Tensor): + # other is a single value + if isinstance(other, (int, float)): + other = self.zeros_like() + other + + return self.equal(other) + else: + return False + + broadcasted_shape_add = [] + + # Function to determine if broadcasting is needed and get the broadcasted shape + def broadcast_shape(shape1, shape2): + if shape1 == shape2: + return shape1, False + + max_len = max(len(shape1), len(shape2)) + shape1 = [1] * (max_len - len(shape1)) + shape1 + shape2 = [1] * (max_len - len(shape2)) + shape2 + + + for dim1, dim2 in zip(shape1, shape2): + if dim1 != dim2 and dim1 != 1 and dim2 != 1: + raise ValueError("Shapes are not compatible for broadcasting") + broadcasted_shape_add.append(max(dim1, dim2)) + return broadcasted_shape_add, True + + broadcasted_shape_add, needs_broadcasting = broadcast_shape(self.shape, other.shape) + + if needs_broadcasting: + if other.ndim == self.ndim - 1: + other = other.reshape([1] + other.shape) + + elif self.ndim == other.ndim - 1: + self = self.reshape([1] + self.shape) + + # Call equal_broadcasted_tensor if broadcasting is needed + Tensor._C.equal_broadcasted_tensor.argtypes = [ctypes.POINTER(CTensor), ctypes.POINTER(CTensor)] + Tensor._C.equal_broadcasted_tensor.restype = ctypes.POINTER(CTensor) + + result_tensor_ptr = Tensor._C.equal_broadcasted_tensor(self.tensor, other.tensor) + + result_data = Tensor() + result_data.tensor = result_tensor_ptr + result_data.shape = broadcasted_shape_add.copy() + result_data.ndim = len(broadcasted_shape_add) + + result_data.device = self.device + result_data.numel = 1 + for s in result_data.shape: + result_data.numel *= s + + else: + Tensor._C.equal_tensor.argtypes = [ctypes.POINTER(CTensor), ctypes.POINTER(CTensor)] + Tensor._C.equal_tensor.restype = ctypes.POINTER(CTensor) + + result_tensor_ptr = Tensor._C.equal_tensor(self.tensor, other.tensor) + + result_data = Tensor() + result_data.tensor = result_tensor_ptr + result_data.shape = self.shape.copy() + result_data.ndim = self.ndim + result_data.device = self.device + result_data.numel = self.numel + + return result_data + + def log(self): + Tensor._C.log_tensor.argtypes = [ctypes.POINTER(CTensor)] + Tensor._C.log_tensor.restype = ctypes.POINTER(CTensor) + + result_tensor_ptr = Tensor._C.log_tensor(self.tensor) + + result_data = Tensor() + result_data.tensor = result_tensor_ptr + result_data.shape = self.shape.copy() + result_data.ndim = self.ndim + result_data.device = self.device + result_data.numel = self.numel + + result_data.requires_grad = self.requires_grad + if result_data.requires_grad: + result_data.grad_fn = LogBackward(self) + + return result_data + + def sum(self, axis=None, keepdim=False): + if axis is not None and axis < 0: + axis = self.ndim + axis + + if axis == None: + axis = -1 + + if axis > self.ndim - 1: + raise ValueError(f"Error: axis argument {axis} cannot be higher than tensor dimension {self.ndim}") + + Tensor._C.sum_tensor.argtypes = [ctypes.POINTER(CTensor), ctypes.c_int, ctypes.c_bool] + Tensor._C.sum_tensor.restype = ctypes.POINTER(CTensor) + + result_tensor_ptr = Tensor._C.sum_tensor(self.tensor, axis, keepdim) + + result_data = Tensor() + result_data.tensor = result_tensor_ptr + + if axis == -1: + if keepdim: + result_data.ndim = self.ndim + result_data.shape = [1] * self.ndim + + else: + result_data.shape = [1] + result_data.ndim = 1 + else: + if keepdim: + result_data.shape = self.shape[:axis] + [1] + self.shape[axis+1:] + else: + result_data.shape = self.shape[:axis] + self.shape[axis+1:] + result_data.ndim = len(result_data.shape) + + result_data.device = self.device + result_data.numel = 1 + for s in result_data.shape: + result_data.numel *= s + + result_data.requires_grad = self.requires_grad + if result_data.requires_grad: + result_data.grad_fn = SumBackward(self, axis, keepdim=keepdim) + + return result_data + + def max(self, axis=None, keepdim=False): + if axis is not None and axis < 0: + axis = self.ndim + axis + + if axis == None: + axis = -1 + + if axis > self.ndim - 1: + raise ValueError(f"Error: axis argument {axis} cannot be higher than tensor dimension {self.ndim}") + + Tensor._C.max_tensor.argtypes = [ctypes.POINTER(CTensor), ctypes.c_int, ctypes.c_bool] + Tensor._C.max_tensor.restype = ctypes.POINTER(CTensor) + + result_tensor_ptr = Tensor._C.max_tensor(self.tensor, axis, keepdim) + + result_data = Tensor() + result_data.tensor = result_tensor_ptr + + if axis == -1: + if keepdim: + result_data.ndim = self.ndim + result_data.shape = [1] * self.ndim + + else: + result_data.shape = [1] + result_data.ndim = 1 + else: + if keepdim: + result_data.shape = self.shape[:axis] + [1] + self.shape[axis+1:] + else: + result_data.shape = self.shape[:axis] + self.shape[axis+1:] + result_data.ndim = len(result_data.shape) + + result_data.device = self.device + result_data.numel = 1 + for s in result_data.shape: + result_data.numel *= s + + result_data.requires_grad = self.requires_grad + if result_data.requires_grad: + result_data.grad_fn = MaxBackward(self, axis, keepdim=keepdim) + + return result_data + + def min(self, axis=None, keepdim=False): + if axis is not None and axis < 0: + axis = self.ndim + axis + + if axis == None: + axis = -1 + + if axis > self.ndim - 1: + raise ValueError(f"Error: axis argument {axis} must be smaller than tensor dimension {self.ndim}") + + Tensor._C.min_tensor.argtypes = [ctypes.POINTER(CTensor), ctypes.c_int, ctypes.c_bool] + Tensor._C.min_tensor.restype = ctypes.POINTER(CTensor) + + result_tensor_ptr = Tensor._C.min_tensor(self.tensor, axis, keepdim) + + result_data = Tensor() + result_data.tensor = result_tensor_ptr + + if axis == -1: + if keepdim: + result_data.ndim = self.ndim + result_data.shape = [1] * self.ndim + + else: + result_data.shape = [1] + result_data.ndim = 1 + else: + if keepdim: + result_data.shape = self.shape[:axis] + [1] + self.shape[axis+1:] + else: + result_data.shape = self.shape[:axis] + self.shape[axis+1:] + result_data.ndim = len(result_data.shape) + + result_data.device = self.device + result_data.numel = 1 + for s in result_data.shape: + result_data.numel *= s + + result_data.requires_grad = self.requires_grad + if result_data.requires_grad: + result_data.grad_fn = MinBackward(self, axis, keepdim=keepdim) + + return result_data + + + def sin(self): + Tensor._C.sin_tensor.argtypes = [ctypes.POINTER(CTensor)] + Tensor._C.sin_tensor.restype = ctypes.POINTER(CTensor) + + result_tensor_ptr = Tensor._C.sin_tensor(self.tensor) + + result_data = Tensor() + result_data.tensor = result_tensor_ptr + result_data.shape = self.shape.copy() + result_data.ndim = self.ndim + result_data.device = self.device + result_data.numel = self.numel + + result_data.requires_grad = self.requires_grad + if result_data.requires_grad: + result_data.grad_fn = SinBackward(self) + + return result_data + + def cos(self): + Tensor._C.cos_tensor.argtypes = [ctypes.POINTER(CTensor)] + Tensor._C.cos_tensor.restype = ctypes.POINTER(CTensor) + + result_tensor_ptr = Tensor._C.cos_tensor(self.tensor) + + result_data = Tensor() + result_data.tensor = result_tensor_ptr + result_data.shape = self.shape.copy() + result_data.ndim = self.ndim + result_data.device = self.device + result_data.numel = self.numel + + result_data.requires_grad = self.requires_grad + if result_data.requires_grad: + result_data.grad_fn = CosBackward(self) + + return result_data + + def sigmoid(self): + Tensor._C.sigmoid_tensor.argtypes = [ctypes.POINTER(CTensor)] + Tensor._C.sigmoid_tensor.restype = ctypes.POINTER(CTensor) + + result_tensor_ptr = Tensor._C.sigmoid_tensor(self.tensor) + + result_data = Tensor() + result_data.tensor = result_tensor_ptr + result_data.shape = self.shape.copy() + result_data.ndim = self.ndim + result_data.device = self.device + result_data.numel = self.numel + + result_data.requires_grad = self.requires_grad + if result_data.requires_grad: + result_data.grad_fn = SigmoidBackward(self) + + return result_data + + + + def transpose(self, axis1, axis2): + if axis1 < 0: + axis1 = self.ndim + axis1 + if axis2 < 0: + axis2 = self.ndim + axis2 + + Tensor._C.transpose_axes_tensor.argtypes = [ctypes.POINTER(CTensor), ctypes.c_int, ctypes.c_int] + Tensor._C.transpose_axes_tensor.restype = ctypes.POINTER(CTensor) + + result_tensor_ptr = Tensor._C.transpose_axes_tensor(self.tensor, axis1, axis2) + + result_data = Tensor() + result_data.tensor = result_tensor_ptr + result_data.shape = self.shape.copy() + result_data.shape[axis1] = self.shape[axis2] + result_data.shape[axis2] = self.shape[axis1] + result_data.ndim = self.ndim + result_data.device = self.device + result_data.numel = self.numel + + result_data.requires_grad = self.requires_grad + if result_data.requires_grad: + result_data.grad_fn = TransposeBackward(self, axis1, axis2) + + return result_data + + @property + def T(self): + Tensor._C.transpose_tensor.argtypes = [ctypes.POINTER(CTensor)] + Tensor._C.transpose_tensor.restype = ctypes.POINTER(CTensor) + + result_tensor_ptr = Tensor._C.transpose_tensor(self.tensor) + + result_data = Tensor() + result_data.tensor = result_tensor_ptr + result_data.shape = self.shape.copy()[::-1] + result_data.ndim = self.ndim + result_data.device = self.device + result_data.numel = self.numel + + result_data.requires_grad = self.requires_grad + if result_data.requires_grad: + result_data.grad_fn = TBackward(self) + + return result_data + + def detach(self): + self.grad = None + self.grad_fn = None + + return self \ No newline at end of file diff --git a/build/lib/norch/utils/__init__.py b/build/lib/norch/utils/__init__.py new file mode 100644 index 0000000..c6adfd1 --- /dev/null +++ b/build/lib/norch/utils/__init__.py @@ -0,0 +1,2 @@ +from .utils import * +from .data import * \ No newline at end of file diff --git a/build/lib/norch/utils/data/__init__.py b/build/lib/norch/utils/data/__init__.py new file mode 100644 index 0000000..2397554 --- /dev/null +++ b/build/lib/norch/utils/data/__init__.py @@ -0,0 +1,4 @@ +from .dataset import * +from .example import * +from .dataloader import * +from .batch import * \ No newline at end of file diff --git a/build/lib/norch/utils/data/batch.py b/build/lib/norch/utils/data/batch.py new file mode 100644 index 0000000..3c9b92c --- /dev/null +++ b/build/lib/norch/utils/data/batch.py @@ -0,0 +1,13 @@ +class Batch(object): + """ + A ``Batch``depends on a ``Dataset`` and is made of ``batch_size`` ``Example``. + """ + def __init__(self, example, batch_size): + self.example = example + self.batch_size = batch_size + + def __getitem__(self, item): + return self.example[item] + + def __len__(self): + return self.batch_size \ No newline at end of file diff --git a/build/lib/norch/utils/data/dataloader.py b/build/lib/norch/utils/data/dataloader.py new file mode 100644 index 0000000..01bd41e --- /dev/null +++ b/build/lib/norch/utils/data/dataloader.py @@ -0,0 +1,20 @@ +import numpy as np +from .batch import Batch + + +class Dataloader(object): + + def __init__(self, dataset, batch_size=32): + self.dataset = dataset + self.batch_size = batch_size + + def __iter__(self): + starts = np.arange(0, len(self.dataset), self.batch_size) + + for start in starts: + end = start + self.batch_size + batch_size = min(end, len(self.dataset)) - start + yield Batch(self.dataset[start:end], batch_size) + + def __len__(self): + return len(self.dataset) // self.batch_size diff --git a/build/lib/norch/utils/data/dataset.py b/build/lib/norch/utils/data/dataset.py new file mode 100644 index 0000000..d38c1ea --- /dev/null +++ b/build/lib/norch/utils/data/dataset.py @@ -0,0 +1,74 @@ +from abc import ABC, abstractmethod +import os +from norch.utils import extract_to_dir, download_from_url +from .example import Example +import norch + + +class Dataset(ABC): + r""" + Abstract Dataset class. All dataset for machine learning purposes can inherits from this architecture, + for convenience. + """ + urls = [] + name = '' + dirname = '' + + def __init__(self, examples, fields): + self.examples = examples + self.fields = fields + + @classmethod + def splits(cls, train=None, test=None, valid=None, root='.'): + raise NotImplementedError + + @classmethod + def download(cls, root): + r"""Download and unzip a web archive (.zip, .gz, or .tgz). + + Args: + root (str): Folder to download data to. + + Returns: + string: Path to extracted dataset. + """ + path_dirname = os.path.join(root, cls.dirname) + path_name = os.path.join(path_dirname, cls.name) + if not os.path.isdir(path_dirname): + for url in cls.urls: + filename = os.path.basename(url) + zpath = os.path.join(path_dirname, filename) + if not os.path.isfile(zpath): + if not os.path.exists(os.path.dirname(zpath)): + os.makedirs(os.path.dirname(zpath)) + print(f'Download {filename} from {url} to {zpath}') + download_from_url(url, zpath) + extract_to_dir(zpath, path_name) + + return path_name + + def __repr__(self): + name = self.__class__.__name__ + string = f"Dataset {name}(" + tab = " " + for (key, value) in self.__dict__.items(): + if key[0] != "_": + if isinstance(value, Example): + fields = self.fields + for (name, field) in fields: + string += f"\n{tab}({name}): {field.__class__.__name__}" \ + f"(transform={True if field.transform is not None else None}, dtype={field.dtype})" + elif isinstance(value, norch.Tensor): + string += f"\n{tab}({key}): {value.__class__.__name__}(shape={value.shape}, dtype={value.dtype})" + else: + string += f"\n{tab}({key}): {value.__class__.__name__}" + return f'{string}\n)' + + def __getitem__(self, item): + return self.examples[item] + + def __setitem__(self, key, value): + self.examples[key] = value + + def __len__(self): + return len(self.examples) diff --git a/build/lib/norch/utils/data/example.py b/build/lib/norch/utils/data/example.py new file mode 100644 index 0000000..a4f4ebf --- /dev/null +++ b/build/lib/norch/utils/data/example.py @@ -0,0 +1,65 @@ +# File: example.py +# Creation: Wednesday August 19th 2020 +# Author: Arthur Dujardin +# Contact: arthur.dujardin@ensg.eu +# arthurd@ifi.uio.no +# -------- +# Copyright (c) 2020 Arthur Dujardin + + +class Field(object): + r""" + A ``Field`` defines the data to process from a raw dataset. It will convert the data into a tensor. The data can + be a string, integers, float etc. The data is meant to be preprocessed and the attribute ``transform`` handles + the way the user want to process the raw data. + """ + + def __init__(self, transform=None, dtype=None): + self.transform = transform + self.dtype = dtype + + def process(self, value): + """Applies the transformation and changes the data's type if necessary. + + Args: + value (Tensor): + + Returns: + + """ + if self.transform is not None: + value = self.transform(value) + if self.dtype is not None: + value = value.astype(self.dtype) + return value + + +class Example(object): + r""" + Store a single training / testing example, and store it as an attribute. + Highly inspired from PyTorch `Example `__. + + """ + @classmethod + def fromlist(cls, values, fields): + """ + Add an example from a list of data with respect to the fields. + + Args: + values: raw data + fields (tuple(string, Field)): fields to preprocess the data on + + Returns: + None + """ + example = cls() + for (value, field) in zip(values, fields): + assert len(field) == 2, f"expected a field template similar to \ + ('name_field', Field()) but got {format(field)}" + assert isinstance(field[1], Field), f"expected a field template similar to \ + ('name_field', Field()) but got {format(field)}" + name = field[0] + value = field[1].process(value) + setattr(example, name, value) + + return example diff --git a/build/lib/norch/utils/utils.py b/build/lib/norch/utils/utils.py new file mode 100644 index 0000000..f8b9b30 --- /dev/null +++ b/build/lib/norch/utils/utils.py @@ -0,0 +1,120 @@ +import random +import requests +import tarfile +import zipfile +import shutil +import os + +def generate_random_list(shape): + """ + Generate a list with random numbers and shape 'shape' + """ + if len(shape) == 0: + return [] + else: + inner_shape = shape[1:] + if len(inner_shape) == 0: + return [random.uniform(-1, 1) for _ in range(shape[0])] + else: + return [generate_random_list(inner_shape) for _ in range(shape[0])] + + +def download_from_url(url, save_path, chunk_size=128): + """Download a file from an URL. + + Original answer from https://stackoverflow.com/questions/9419162/download-returned-zip-file-from-url + + Args: + url (str): path to the URL. + save_path (str): path to the saving directory. + chunk_size (int): download chunk. + + Returns: + None + """ + response = requests.get(url, stream=True) + total = response.headers.get('content-length') + with open(save_path, 'wb') as f: + if total is None: + f.write(response.content) + else: + downloaded = 0 + total = int(total) + for data in response.iter_content(chunk_size=max(int(total / 1000), 1024 * 1024)): + downloaded += len(data) + f.write(data) + progress_bar(downloaded, total, "Downloading...") + + +def extract_to_dir(filename, dirpath='.'): + # Does not create folder twice with the same name + name, ext = os.path.splitext(filename) + # if os.path.basename(name) == os.path.basename(dirpath): + # dirpath = '.' + # Extract + print(dirpath) + print("Extracting...", end="") + if tarfile.is_tarfile(filename): + tarfile.open(filename, 'r').extractall(dirpath) + elif zipfile.is_zipfile(filename): + zipfile.ZipFile(filename, 'r').extractall(dirpath) + elif ext == '.gz': + if not os.path.exists(dirpath): + os.mkdir(dirpath) + shutil.move(filename, os.path.join(dirpath, os.path.basename(filename))) + print(f" | NOTE: gzip files are not extracted, and moved to {dirpath}", end="") + # Return the path where the file was extracted + print(" | Done !") + return os.path.abspath(dirpath) + +def progress_bar(current_index, max_index, prefix=None, suffix=None, start_time=None): + """Display a progress bar and duration. + + Args: + current_index (int): current state index (or epoch number). + max_index (int): maximal numbers of state. + prefix (str, optional): prefix of the progress bar. The default is None. + suffix (str, optional): suffix of the progress bar. The default is None. + start_time (float, optional): starting time of the progress bar. If not None, it will display the time + spent from the beginning to the current state. The default is None. + + Returns: + None. Display the progress bar in the console. + """ + # Add a prefix to the progress bar + prefix = "" if prefix is None else str(prefix) + " " + + # Get the percentage + percentage = current_index * 100 // max_index + loading = "[" + "=" * (percentage // 2) + " " * (50 - percentage // 2) + "]" + progress_display = "\r{0}{1:3d}% | {2}".format(prefix, percentage, loading) + + # Add a suffix to the progress bar + progress_display += "" if suffix is None else " | " + str(suffix) + + # Add a timer + if start_time is not None: + time_min, time_sec = get_time(start_time, time.time()) + time_display = " | Time: {0}m {1}s".format(time_min, time_sec) + progress_display += time_display + + # Print the progress bar + # TODO: return a string instead + print(progress_display, end="{}".format("" if current_index < max_index else " | Done !\n")) + +def get_time(start_time, end_time): + """Get ellapsed time in minutes and seconds. + + Args: + start_time (float): strarting time + end_time (float): ending time + + Returns: + elapsed_mins (float): elapsed time in minutes + elapsed_secs (float): elapsed time in seconds. + """ + elapsed_time = end_time - start_time + elapsed_mins = int(elapsed_time / 60) + elapsed_secs = int(elapsed_time - (elapsed_mins * 60)) + + return elapsed_mins, elapsed_secs diff --git a/build/lib/norch/utils/utils_unittests.py b/build/lib/norch/utils/utils_unittests.py new file mode 100644 index 0000000..4e0a12e --- /dev/null +++ b/build/lib/norch/utils/utils_unittests.py @@ -0,0 +1,24 @@ +import torch + +def to_torch(custom_tensor): + shape = custom_tensor.shape + pytorch_tensor = torch.zeros(shape) + + def _iterate_indices(shape): + if len(shape) == 0: + yield () + else: + for index in range(shape[0]): + for sub_indices in _iterate_indices(shape[1:]): + yield (index,) + sub_indices + + # Iterate over all elements using the custom tensor's __getitem__ method + for indices in _iterate_indices(shape): + value = custom_tensor[indices] + pytorch_tensor[tuple(indices)] = value + + return pytorch_tensor + +def compare_torch(tensor1, tensor2, epsilon=1e-5): + diff = torch.abs(tensor1 - tensor2) + return torch.all(diff < epsilon) \ No newline at end of file diff --git a/build/lib/tests/__init__.py b/build/lib/tests/__init__.py new file mode 100644 index 0000000..4750076 --- /dev/null +++ b/build/lib/tests/__init__.py @@ -0,0 +1,3 @@ +from .test_operations import * +from .test_autograd import * +from .test_nn import * \ No newline at end of file diff --git a/build/lib/tests/test_autograd.py b/build/lib/tests/test_autograd.py new file mode 100644 index 0000000..3be772c --- /dev/null +++ b/build/lib/tests/test_autograd.py @@ -0,0 +1,993 @@ +import unittest +import norch +from norch.utils import utils_unittests as utils +import torch +import os + +class TestTensorAutograd(unittest.TestCase): + def setUp(self): + self.device = os.environ.get('device') + if self.device is None or self.device != 'cuda': + self.device = 'cpu' + + print(f"Running tests on: {self.device}") + + + def test_addition(self): + """ + Test autograd from addition two tensors: tensor1 + tensor2 + """ + norch_tensor1 = norch.Tensor([[[1, 2.5], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True).to(self.device) + norch_tensor2 = norch.Tensor([[[1, 1.], [1, 1.9]], [[1, 1], [1, 1]]], requires_grad=True).to(self.device) + norch_result = (norch_tensor1 + norch_tensor2).sum() + norch_result.backward() + norch_tensor1_grad = utils.to_torch(norch_tensor1.grad).to(self.device) + norch_tensor2_grad = utils.to_torch(norch_tensor2.grad).to(self.device) + + torch_tensor1 = torch.tensor([[[1, 2.5], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True, device=self.device) + torch_tensor2 = torch.tensor([[[1, 1.], [1, 1.9]], [[1, 1], [1, 1]]], requires_grad=True, device=self.device) + torch_result = (torch_tensor1 + torch_tensor2).sum() + torch_result.backward() + torch_tensor1_grad = torch_tensor1.grad + torch_tensor2_grad = torch_tensor2.grad + + self.assertTrue(utils.compare_torch(norch_tensor1_grad, torch_tensor1_grad)) + self.assertTrue(utils.compare_torch(norch_tensor2_grad, torch_tensor2_grad)) + + def test_sum_axis(self): + """ + Test autograd from sum specifying axis + """ + norch_tensor1 = norch.Tensor([[[1, 2.5], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True).to(self.device) + norch_tensor2 = norch.Tensor([[[1, 1.], [1, 1.9]], [[1, 1], [1, 1]]], requires_grad=True).to(self.device) + norch_result = (norch_tensor1 + norch_tensor2).sum(axis=0).sum(axis=0).sum() + + norch_result.backward() + norch_tensor1_grad = utils.to_torch(norch_tensor1.grad).to(self.device) + norch_tensor2_grad = utils.to_torch(norch_tensor2.grad).to(self.device) + + torch_tensor1 = torch.tensor([[[1, 2.5], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True, device=self.device) + torch_tensor2 = torch.tensor([[[1, 1.], [1, 1.9]], [[1, 1], [1, 1]]], requires_grad=True, device=self.device) + + torch_result = (torch_tensor1 + torch_tensor2).sum(axis=0).sum(axis=0).sum() + torch_result.backward() + torch_tensor1_grad = torch_tensor1.grad + torch_tensor2_grad = torch_tensor2.grad + + self.assertTrue(utils.compare_torch(norch_tensor1_grad, torch_tensor1_grad)) + self.assertTrue(utils.compare_torch(norch_tensor2_grad, torch_tensor2_grad)) + + norch_tensor1 = norch.Tensor([[[1, 2.5], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True).to(self.device) + norch_tensor2 = norch.Tensor([[[1, 1.], [1, 1.9]], [[1, 1], [1, 1]]], requires_grad=True).to(self.device) + norch_result = (norch_tensor1 + norch_tensor2).sum(axis=1).sum() + + norch_result.backward() + norch_tensor1_grad = utils.to_torch(norch_tensor1.grad).to(self.device) + norch_tensor2_grad = utils.to_torch(norch_tensor2.grad).to(self.device) + + torch_tensor1 = torch.tensor([[[1, 2.5], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True, device=self.device) + torch_tensor2 = torch.tensor([[[1, 1.], [1, 1.9]], [[1, 1], [1, 1]]], requires_grad=True, device=self.device) + + torch_result = (torch_tensor1 + torch_tensor2).sum(axis=1).sum() + torch_result.backward() + torch_tensor1_grad = torch_tensor1.grad + torch_tensor2_grad = torch_tensor2.grad + + self.assertTrue(utils.compare_torch(norch_tensor1_grad, torch_tensor1_grad)) + self.assertTrue(utils.compare_torch(norch_tensor2_grad, torch_tensor2_grad)) + + + def test_max(self): + """ + Test autograd from max + """ + norch_tensor = norch.Tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True).to(self.device) + norch_result = norch_tensor.max() + + norch_result.backward() + norch_tensor_grad = utils.to_torch(norch_tensor.grad).to(self.device) + + torch_tensor = torch.tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True, device=self.device) + + torch_result = torch_tensor.max() + torch_result.backward() + torch_tensor_grad = torch_tensor.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad, torch_tensor_grad)) + + def test_max_axis(self): + """ + Test autograd from max specifying axis + """ + norch_tensor = norch.Tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True).to(self.device) + norch_max_axis = norch_tensor.max(axis=1) + norch_result = norch_max_axis.sum() + + norch_result.backward() + norch_tensor_grad = utils.to_torch(norch_tensor.grad).to(self.device) + + torch_tensor = torch.tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True, device=self.device) + + torch_max_axis, _ = torch_tensor.max(axis=1) + torch_result = torch_max_axis.sum() + torch_result.backward() + torch_tensor_grad = torch_tensor.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad, torch_tensor_grad)) + + norch_tensor = norch.Tensor([[[10, 1], [-4, 0]], [[5., 50], [7, 8]]], requires_grad=True).to(self.device) + norch_max_axis = norch_tensor.max(axis=2) + norch_result = norch_max_axis.sum() + + norch_result.backward() + norch_tensor_grad = utils.to_torch(norch_tensor.grad).to(self.device) + + torch_tensor = torch.tensor([[[10, 1], [-4, 0]], [[5., 50], [7, 8]]], requires_grad=True, device=self.device) + + torch_max_axis, _ = torch_tensor.max(axis=2) + torch_result = torch_max_axis.sum() + torch_result.backward() + torch_tensor_grad = torch_tensor.grad + self.assertTrue(utils.compare_torch(norch_tensor_grad, torch_tensor_grad)) + + + ## evaluate case with some repeated values and axis 2 + + #def test_max_axis(self): + # """ + # Test autograd from max specifying axis + # """ + # + # norch_tensor = norch.Tensor([[[10, 10], [-4, 0]], [[5., 50], [7, 8]]], requires_grad=True).to(self.device) + # norch_max_axis = norch_tensor.max(axis=2) + # norch_result = norch_max_axis.sum() + # + # norch_result.backward() + # norch_tensor_grad = utils.to_torch(norch_tensor.grad).to(self.device) + # + # torch_tensor = torch.tensor([[[10, 10], [-4, 0]], [[5., 50], [7, 8]]], requires_grad=True).to(self.device) + # + # torch_max_axis, _ = torch_tensor.max(axis=2) + # torch_result = torch_max_axis.sum() + # torch_result.backward() + # torch_tensor_grad = torch_tensor.grad + # self.assertTrue(utils.compare_torch(norch_tensor_grad, torch_tensor_grad)) + + + #def test_min_axis(self): + # """ + # Test autograd from max specifying axis + # """ + # + # norch_tensor = norch.Tensor([[[10, 10], [-4, 0]], [[5., 50], [7, 8]]], requires_grad=True).to(self.device) + # norch_min_axis = norch_tensor.min(axis=2) + # norch_result = norch_min_axis.sum() + # + # norch_result.backward() + # norch_tensor_grad = utils.to_torch(norch_tensor.grad).to(self.device) + # + # torch_tensor = torch.tensor([[[10, 10], [-4, 0]], [[5., 50], [7, 8]]], requires_grad=True).to(self.device) + # + # torch_min_axis, _ = torch_tensor.min(axis=2) + # torch_result = torch_min_axis.sum() + # torch_result.backward() + # torch_tensor_grad = torch_tensor.grad + # self.assertTrue(utils.compare_torch(norch_tensor_grad, torch_tensor_grad)) + + + + def test_min(self): + """ + Test autograd from min + """ + norch_tensor = norch.Tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True).to(self.device) + norch_result = norch_tensor.min() + + norch_result.backward() + norch_tensor_grad = utils.to_torch(norch_tensor.grad).to(self.device) + + torch_tensor = torch.tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True, device=self.device) + + torch_result = torch_tensor.min() + torch_result.backward() + torch_tensor_grad = torch_tensor.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad, torch_tensor_grad)) + + def test_min_axis(self): + """ + Test autograd from min specifying axis + """ + norch_tensor = norch.Tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True).to(self.device) + norch_min = norch_tensor.min(axis=1) + norch_result = norch_min.sum() + + norch_result.backward() + norch_tensor_grad = utils.to_torch(norch_tensor.grad).to(self.device) + + torch_tensor = torch.tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True, device=self.device) + + torch_min, _ = torch_tensor.min(axis=1) + torch_result = torch_min.sum() + torch_result.backward() + torch_tensor_grad = torch_tensor.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad, torch_tensor_grad)) + + norch_tensor = norch.Tensor([[[10, 1], [-4, 0]], [[5., 50], [7, 8]]], requires_grad=True).to(self.device) + norch_min = norch_tensor.min(axis=2) + norch_result = norch_min.sum() + + norch_result.backward() + norch_tensor_grad = utils.to_torch(norch_tensor.grad).to(self.device) + + torch_tensor = torch.tensor([[[10, 1], [-4, 0]], [[5., 50], [7, 8]]], requires_grad=True, device=self.device) + + torch_min, _ = torch_tensor.min(axis=2) + torch_result = torch_min.sum() + torch_result.backward() + torch_tensor_grad = torch_tensor.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad, torch_tensor_grad)) + + + def test_broadcasted_addition_autograd(self): + """ + Test autograd for broadcasting addition: tensor1 + tensor2 + """ + norch_tensor1 = norch.Tensor([[[1., 2, 3], [4, 5, 6]]], requires_grad=True).to(self.device) # Shape (1, 2, 3) + norch_tensor2 = norch.Tensor([1.5, -1, 0], requires_grad=True).to(self.device) # Shape (3) + norch_result = (norch_tensor1 + norch_tensor2).sum() + norch_result.backward() + norch_tensor1_grad = utils.to_torch(norch_tensor1.grad).to(self.device) + norch_tensor2_grad = utils.to_torch(norch_tensor2.grad).to(self.device) + + torch_tensor1 = torch.tensor([[[1., 2, 3], [4, 5, 6]]], requires_grad=True, device=self.device) # Shape (1, 2, 3) + torch_tensor2 = torch.tensor([1.5, -1, 0], requires_grad=True, device=self.device) # Shape (3) + torch_result = (torch_tensor1 + torch_tensor2).sum() + torch_result.backward() + torch_tensor1_grad = torch_tensor1.grad + torch_tensor2_grad = torch_tensor2.grad + + self.assertTrue(utils.compare_torch(norch_tensor1_grad, torch_tensor1_grad)) + self.assertTrue(utils.compare_torch(norch_tensor2_grad, torch_tensor2_grad)) + + ## reversed order broadcasting + norch_tensor1 = norch.Tensor([[[1., 2, 3], [4, 5, 6]]], requires_grad=True).to(self.device) # Shape (1, 2, 3) + norch_tensor2 = norch.Tensor([1.5, -1, 0], requires_grad=True).to(self.device) # Shape (3) + + norch_result = (norch_tensor2 + norch_tensor1).sum() + norch_result.backward() + norch_tensor1_grad = utils.to_torch(norch_tensor1.grad).to(self.device) + norch_tensor2_grad = utils.to_torch(norch_tensor2.grad).to(self.device) + + torch_tensor1 = torch.tensor([[[1., 2, 3], [4, 5, 6]]], requires_grad=True, device=self.device) # Shape (1, 2, 3) + torch_tensor2 = torch.tensor([1.5, -1, 0], requires_grad=True, device=self.device) # Shape (3) + + torch_result = (torch_tensor2 + torch_tensor1).sum() + torch_result.backward() + torch_tensor1_grad = torch_tensor1.grad + torch_tensor2_grad = torch_tensor2.grad + + self.assertTrue(utils.compare_torch(norch_tensor1_grad, torch_tensor1_grad)) + self.assertTrue(utils.compare_torch(norch_tensor2_grad, torch_tensor2_grad)) + + def test_subtraction(self): + """ + Test autograd from subtraction two tensors: tensor1 - tensor2 + """ + norch_tensor1_sub = norch.Tensor([[[1, 2.5], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True).to(self.device) + norch_tensor2_sub = norch.Tensor([[[1, 1.], [1, 1.9]], [[1, 1], [1, 1]]], requires_grad=True).to(self.device) + norch_result_sub = (norch_tensor1_sub - norch_tensor2_sub).sum() + norch_result_sub.backward() + norch_tensor1_grad_sub = utils.to_torch(norch_tensor1_sub.grad).to(self.device) + norch_tensor2_grad_sub = utils.to_torch(norch_tensor2_sub.grad).to(self.device) + + torch_tensor1_sub = torch.tensor([[[1, 2.5], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True, device=self.device) + torch_tensor2_sub = torch.tensor([[[1, 1.], [1, 1.9]], [[1, 1], [1, 1]]], requires_grad=True, device=self.device) + torch_result_sub = (torch_tensor1_sub - torch_tensor2_sub).sum() + torch_result_sub.backward() + torch_tensor1_grad_sub = torch_tensor1_sub.grad + torch_tensor2_grad_sub = torch_tensor2_sub.grad + + self.assertTrue(utils.compare_torch(norch_tensor1_grad_sub, torch_tensor1_grad_sub)) + self.assertTrue(utils.compare_torch(norch_tensor2_grad_sub, torch_tensor2_grad_sub)) + + def test_broadcasted_subtraction_autograd(self): + """ + Test autograd for broadcasting subtraction: tensor1 - tensor2 + """ + norch_tensor1 = norch.Tensor([[[1., 2, 3], [4, 5, 6]]], requires_grad=True).to(self.device) # Shape (1, 2, 3) + norch_tensor2 = norch.Tensor([1.5, -1, 0], requires_grad=True).to(self.device) # Shape (3) + norch_result = (norch_tensor1 - norch_tensor2).sum() + norch_result.backward() + norch_tensor1_grad = utils.to_torch(norch_tensor1.grad).to(self.device) + norch_tensor2_grad = utils.to_torch(norch_tensor2.grad).to(self.device) + + torch_tensor1 = torch.tensor([[[1., 2, 3], [4, 5, 6]]], requires_grad=True, device=self.device) # Shape (1, 2, 3) + torch_tensor2 = torch.tensor([1.5, -1, 0], requires_grad=True, device=self.device) # Shape (3) + torch_result = (torch_tensor1 - torch_tensor2).sum() + torch_result.backward() + torch_tensor1_grad = torch_tensor1.grad + torch_tensor2_grad = torch_tensor2.grad + + self.assertTrue(utils.compare_torch(norch_tensor1_grad, torch_tensor1_grad)) + self.assertTrue(utils.compare_torch(norch_tensor2_grad, torch_tensor2_grad)) + + # reversed order broadcasting + norch_tensor1 = norch.Tensor([[[1., 2, 3], [4, 5, 6]]], requires_grad=True).to(self.device) # Shape (1, 2, 3) + norch_tensor2 = norch.Tensor([1.5, -1, 0], requires_grad=True).to(self.device) # Shape (3) + + norch_result = (norch_tensor2 - norch_tensor1).sum() + norch_result.backward() + norch_tensor1_grad = utils.to_torch(norch_tensor1.grad).to(self.device) + norch_tensor2_grad = utils.to_torch(norch_tensor2.grad).to(self.device) + + torch_tensor1 = torch.tensor([[[1., 2, 3], [4, 5, 6]]], requires_grad=True, device=self.device) # Shape (1, 2, 3) + torch_tensor2 = torch.tensor([1.5, -1, 0], requires_grad=True, device=self.device) # Shape (3) + + torch_result = (torch_tensor2 - torch_tensor1).sum() + torch_result.backward() + torch_tensor1_grad = torch_tensor1.grad + torch_tensor2_grad = torch_tensor2.grad + + self.assertTrue(utils.compare_torch(norch_tensor1_grad, torch_tensor1_grad)) + self.assertTrue(utils.compare_torch(norch_tensor2_grad, torch_tensor2_grad)) + + + def test_division(self): + """ + Test autograd from dividing two tensors: tensor1 / tensor2 + """ + norch_tensor1_div = norch.Tensor([[[2, 5.1], [6, -8]], [[10, 12], [14, 16]]], requires_grad=True).to(self.device) + norch_tensor2_div = norch.Tensor([[[1, 1], [2, 2.2]], [[3, 3], [4, 4]]], requires_grad=True).to(self.device) + norch_result_div = (norch_tensor1_div / norch_tensor2_div).sum() + norch_result_div.backward() + norch_tensor1_grad_div = utils.to_torch(norch_tensor1_div.grad).to(self.device) + norch_tensor2_grad_div = utils.to_torch(norch_tensor2_div.grad).to(self.device) + + torch_tensor1_div = torch.tensor([[[2, 5.1], [6, -8]], [[10, 12], [14, 16]]], requires_grad=True, device=self.device) + torch_tensor2_div = torch.tensor([[[1, 1], [2, 2.2]], [[3, 3], [4, 4]]], requires_grad=True, device=self.device) + torch_result_div = (torch_tensor1_div / torch_tensor2_div).sum() + torch_result_div.backward() + torch_tensor1_grad_div = torch_tensor1_div.grad + torch_tensor2_grad_div = torch_tensor2_div.grad + + self.assertTrue(utils.compare_torch(norch_tensor1_grad_div, torch_tensor1_grad_div)) + self.assertTrue(utils.compare_torch(norch_tensor2_grad_div, torch_tensor2_grad_div)) + + + def test_tensor_division_scalar(self): + """ + Test autograd from dividing tensor by scalar: tensor / scalar + """ + norch_tensor_div_scalar = norch.Tensor([[[2, 4.7], [6, 8]], [[10, 12], [14, 16]]], requires_grad=True).to(self.device) + scalar = 2 + norch_result_div_scalar = (norch_tensor_div_scalar / scalar).sum() + norch_result_div_scalar.backward() + norch_tensor_grad_div_scalar = utils.to_torch(norch_tensor_div_scalar.grad).to(self.device) + + torch_tensor_div_scalar = torch.tensor([[[2, 4.7], [6, 8]], [[10, 12], [14, 16]]], requires_grad=True, device=self.device) + torch_result_div_scalar = (torch_tensor_div_scalar / scalar).sum() + torch_result_div_scalar.backward() + torch_tensor_grad_div_scalar = torch_tensor_div_scalar.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad_div_scalar, torch_tensor_grad_div_scalar)) + + + def test_scalar_division_tensor(self): + """ + Test autograd from dividing scalar by tensor: scalar / tensor + """ + scalar = 2 + norch_tensor_scalar_div = norch.Tensor([[[1, 2.23], [3, 4]], [[5, 6], [7, 8]]], requires_grad=True).to(self.device) + norch_result_scalar_div = (scalar / norch_tensor_scalar_div).sum() + norch_result_scalar_div.backward() + norch_tensor_grad_scalar_div = utils.to_torch(norch_tensor_scalar_div.grad).to(self.device) + + torch_tensor_scalar_div = torch.tensor([[[1, 2.23], [3, 4]], [[5, 6], [7, 8]]], requires_grad=True, device=self.device) + torch_result_scalar_div = (scalar / torch_tensor_scalar_div).sum() + torch_result_scalar_div.backward() + torch_tensor_grad_scalar_div = torch_tensor_scalar_div.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad_scalar_div, torch_tensor_grad_scalar_div)) + + + def test_power_scalar_tensor(self): + """ + Test autograd from scalar raised to tensor: scalar ** tensor + """ + scalar = 2 + norch_tensor_power_st = norch.Tensor([[[2, 3.21], [4, 2.1]], [[6, 7], [8, 9]]], requires_grad=True).to(self.device) + norch_result_power_st = (scalar ** norch_tensor_power_st).sum() + norch_result_power_st.backward() + norch_tensor_grad_power_st = utils.to_torch(norch_tensor_power_st.grad).to(self.device) + + torch_tensor_power_st = torch.tensor([[[2, 3.21], [4, 2.1]], [[6, 7], [8, 9]]], requires_grad=True, device=self.device) + torch_result_power_st = (scalar ** torch_tensor_power_st).sum() + torch_result_power_st.backward() + torch_tensor_grad_power_st = torch_tensor_power_st.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad_power_st, torch_tensor_grad_power_st)) + + def test_power_tensor_scalar(self): + """ + Test autograd from tensor raised to scalar: tensor ** scalar + """ + scalar = 2 + norch_tensor_power_ts = norch.Tensor([[[2, 3], [4, 2.1]], [[6, 7], [8, 9]]], requires_grad=True).to(self.device) + norch_result_power_ts = (norch_tensor_power_ts ** scalar).sum() + norch_result_power_ts.backward() + norch_tensor_grad_power_ts = utils.to_torch(norch_tensor_power_ts.grad).to(self.device) + + torch_tensor_power_ts = torch.tensor([[[2, 3], [4, 2.1]], [[6, 7], [8, 9]]], requires_grad=True, device=self.device) + torch_result_power_ts = (torch_tensor_power_ts ** scalar).sum() + torch_result_power_ts.backward() + torch_tensor_grad_power_ts = torch_tensor_power_ts.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad_power_ts, torch_tensor_grad_power_ts)) + + def test_matmul(self): + """ + Test autograd from matrix multiplication: matmul(tensor1, tensor2) + """ + norch_tensor1_matmul = norch.Tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True).to(self.device) + norch_tensor2_matmul = norch.Tensor([[[1.1, 3], [4, 5]], [[6, 7], [8, 9]]], requires_grad=True).to(self.device) + norch_result_matmul = (norch_tensor1_matmul @ norch_tensor2_matmul).sum() + norch_result_matmul.backward() + norch_tensor1_grad_matmul = utils.to_torch(norch_tensor1_matmul.grad).to(self.device) + norch_tensor2_grad_matmul = utils.to_torch(norch_tensor2_matmul.grad).to(self.device) + + torch_tensor1_matmul = torch.tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True, device=self.device) + torch_tensor2_matmul = torch.tensor([[[1.1, 3], [4, 5]], [[6, 7], [8, 9]]], requires_grad=True, device=self.device) + torch_result_matmul = (torch_tensor1_matmul @ torch_tensor2_matmul).sum() + torch_result_matmul.backward() + torch_tensor1_grad_matmul = torch_tensor1_matmul.grad + torch_tensor2_grad_matmul = torch_tensor2_matmul.grad + + self.assertTrue(utils.compare_torch(norch_tensor1_grad_matmul, torch_tensor1_grad_matmul)) + self.assertTrue(utils.compare_torch(norch_tensor2_grad_matmul, torch_tensor2_grad_matmul)) + + def test_batched_matmul(self): + """ + Test autograd from batched matrix multiplication: BxMxP = BxNxM @ BxMxP + """ + B = 3 # Batch size + + norch_tensor1_matmul = norch.Tensor([[[1., 2], [3, -4], [5, 6], [7, 8]] for _ in range(B)], requires_grad=True).to(self.device) + norch_tensor2_matmul = norch.Tensor([[[2., 3, 1, 0, 4], [5, -1, 2, 3, 0]] for _ in range(B)], requires_grad=True).to(self.device) + + norch_result_matmul = norch_tensor1_matmul @ norch_tensor2_matmul + norch_result_matmul_sum = norch_result_matmul.sum() # Sum over all elements + norch_result_matmul_sum.backward() + + # Convert gradients to torch tensors + norch_tensor1_grad_matmul = utils.to_torch(norch_tensor1_matmul.grad).to(self.device) + norch_tensor2_grad_matmul = utils.to_torch(norch_tensor2_matmul.grad).to(self.device) + + # Repeat the same process with torch tensors + torch_tensor1_matmul = torch.tensor([[[1., 2], [3, -4], [5, 6], [7, 8]] for _ in range(B)], requires_grad=True, device=self.device) + torch_tensor2_matmul = torch.tensor([[[2., 3, 1, 0, 4], [5, -1, 2, 3, 0]] for _ in range(B)], requires_grad=True, device=self.device) + torch_result_matmul = torch.matmul(torch_tensor1_matmul, torch_tensor2_matmul) + torch_result_matmul_sum = torch_result_matmul.sum() + torch_result_matmul_sum.backward() + + # Extract gradients from torch tensors + torch_tensor1_grad_matmul = torch_tensor1_matmul.grad + torch_tensor2_grad_matmul = torch_tensor2_matmul.grad + + # Assertions to compare the gradients + self.assertTrue(utils.compare_torch(norch_tensor1_grad_matmul, torch_tensor1_grad_matmul)) + self.assertTrue(utils.compare_torch(norch_tensor2_grad_matmul, torch_tensor2_grad_matmul)) + + def test_broadcasted_batched_matmul(self): + """ + Test autograd from broadcasted batched matrix multiplication: BxMxP = NxM @ BxMxP + """ + B = 3 # Batch size + + norch_tensor1_matmul = norch.Tensor([[1., 2], [3, -4], [5, 6], [7, 8]], requires_grad=True).to(self.device) + norch_tensor2_matmul = norch.Tensor([[[2., 3, 1, 0, 4], [5, -1, 2, 3, 0]] for _ in range(B)], requires_grad=True).to(self.device) + + norch_result_matmul = norch_tensor1_matmul @ norch_tensor2_matmul + norch_result_matmul_sum = norch_result_matmul.sum() # Sum over all elements + norch_result_matmul_sum.backward() + + # Convert gradients to torch tensors + norch_tensor1_grad_matmul = utils.to_torch(norch_tensor1_matmul.grad).to(self.device) + norch_tensor2_grad_matmul = utils.to_torch(norch_tensor2_matmul.grad).to(self.device) + + # Repeat the same process with torch tensors + torch_tensor1_matmul = torch.tensor([[1., 2], [3, -4], [5, 6], [7, 8]], requires_grad=True, device=self.device) + torch_tensor2_matmul = torch.tensor([[[2., 3, 1, 0, 4], [5, -1, 2, 3, 0]] for _ in range(B)], requires_grad=True, device=self.device) + torch_result_matmul = torch.matmul(torch_tensor1_matmul, torch_tensor2_matmul) + torch_result_matmul_sum = torch_result_matmul.sum() + torch_result_matmul_sum.backward() + + # Extract gradients from torch tensors + torch_tensor1_grad_matmul = torch_tensor1_matmul.grad + torch_tensor2_grad_matmul = torch_tensor2_matmul.grad + + # Assertions to compare the gradients + self.assertTrue(utils.compare_torch(norch_tensor1_grad_matmul, torch_tensor1_grad_matmul)) + self.assertTrue(utils.compare_torch(norch_tensor2_grad_matmul, torch_tensor2_grad_matmul)) + + + def test_elementwise_mul_scalar(self): + """ + Test autograd from elementwise multiplication with scalar: scalar * tensor + """ + scalar = 2 + norch_tensor_elemwise_mul_scalar = norch.Tensor([[[1.1, 2], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True).to(self.device) + norch_result_elemwise_mul_scalar = (scalar * norch_tensor_elemwise_mul_scalar).sum() + norch_result_elemwise_mul_scalar.backward() + norch_tensor_grad_elemwise_mul_scalar = utils.to_torch(norch_tensor_elemwise_mul_scalar.grad).to(self.device) + + torch_tensor_elemwise_mul_scalar = torch.tensor([[[1.1, 2], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True, device=self.device) + torch_result_elemwise_mul_scalar = (scalar * torch_tensor_elemwise_mul_scalar).sum() + torch_result_elemwise_mul_scalar.backward() + torch_tensor_grad_elemwise_mul_scalar = torch_tensor_elemwise_mul_scalar.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad_elemwise_mul_scalar, torch_tensor_grad_elemwise_mul_scalar)) + + + def test_elementwise_mul_tensor(self): + """ + Test autograd from elementwise multiplication between two tensors: tensor1 * tensor2 + """ + norch_tensor1_elemwise_mul = norch.Tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True).to(self.device) + norch_tensor2_elemwise_mul = norch.Tensor([[[1.1, 3], [4, 5]], [[6, 7], [8, 9]]], requires_grad=True).to(self.device) + norch_result_elemwise_mul = (norch_tensor1_elemwise_mul * norch_tensor2_elemwise_mul).sum() + norch_result_elemwise_mul.backward() + norch_tensor1_grad_elemwise_mul = utils.to_torch(norch_tensor1_elemwise_mul.grad).to(self.device) + norch_tensor2_grad_elemwise_mul = utils.to_torch(norch_tensor2_elemwise_mul.grad).to(self.device) + + torch_tensor1_elemwise_mul = torch.tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True, device=self.device) + torch_tensor2_elemwise_mul = torch.tensor([[[1.1, 3], [4, 5]], [[6, 7], [8, 9]]], requires_grad=True, device=self.device) + torch_result_elemwise_mul = (torch_tensor1_elemwise_mul * torch_tensor2_elemwise_mul).sum() + torch_result_elemwise_mul.backward() + torch_tensor1_grad_elemwise_mul = torch_tensor1_elemwise_mul.grad + torch_tensor2_grad_elemwise_mul = torch_tensor2_elemwise_mul.grad + + self.assertTrue(utils.compare_torch(norch_tensor1_grad_elemwise_mul, torch_tensor1_grad_elemwise_mul)) + self.assertTrue(utils.compare_torch(norch_tensor2_grad_elemwise_mul, torch_tensor2_grad_elemwise_mul)) + + def test_sin_tensor(self): + """ + Test autograd from sin operation: sin(tensor) + """ + norch_sin_tensor = norch.Tensor([[[2, 3.21], [4, 2.1]], [[6, 7], [8, 9]]], requires_grad=True).to(self.device) + norch_result_sin_tensor = (norch_sin_tensor.sin()).sum() + norch_result_sin_tensor.backward() + torch_result_sin_tensor_grad = utils.to_torch(norch_sin_tensor.grad).to(self.device) + + torch_sin_tensor = torch.tensor([[[2, 3.21], [4, 2.1]], [[6, 7], [8, 9]]], requires_grad=True, device=self.device) + torch_expected_sin_tensor = (torch.sin(torch_sin_tensor)).sum() + torch_expected_sin_tensor.backward() + torch_expected_sin_tensor_grad = torch_sin_tensor.grad + + self.assertTrue(utils.compare_torch(torch_result_sin_tensor_grad, torch_expected_sin_tensor_grad)) + + def test_cos_tensor(self): + """ + Test autograd from cosine operation: cos(tensor) + """ + norch_cos_tensor = norch.Tensor([[[2, 3.21], [4, 2.1]], [[6, 7], [8, 9]]], requires_grad=True).to(self.device) + norch_result_cos_tensor = (norch_cos_tensor.sin()).sum() + norch_result_cos_tensor.backward() + torch_result_cos_tensor_grad = utils.to_torch(norch_cos_tensor.grad).to(self.device) + + torch_cos_tensor = torch.tensor([[[2, 3.21], [4, 2.1]], [[6, 7], [8, 9]]], requires_grad=True, device=self.device) + torch_expected_cos_tensor = (torch.sin(torch_cos_tensor)).sum() + torch_expected_cos_tensor.backward() + torch_expected_cos_tensor_grad = torch_cos_tensor.grad + + self.assertTrue(utils.compare_torch(torch_result_cos_tensor_grad, torch_expected_cos_tensor_grad)) + + def test_sigmoid(self): + """ + Test autograd from sigmoid + """ + norch_tensor = norch.Tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True).to(self.device) + norch_sigmoid = norch.sigmoid(norch_tensor) + norch_result = norch_sigmoid.sum() + + norch_result.backward() + norch_tensor_grad = utils.to_torch(norch_tensor.grad).to(self.device) + + torch_tensor = torch.tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True, device=self.device) + torch_sigmoid = torch.sigmoid(torch_tensor) + torch_result = torch_sigmoid.sum() + + torch_result.backward() + torch_tensor_grad = torch_tensor.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad, torch_tensor_grad)) + + def test_mse_loss_autograd(self): + """ + Test the MSELoss with autograd functionality + """ + loss_fn_norch = norch.nn.MSELoss() + loss_fn_torch = torch.nn.MSELoss() + + predictions_norch = norch.Tensor([1.1, 2, 3, 4], requires_grad=True).to(self.device) + labels_norch = norch.Tensor([4, 3, 2.1, 1]).to(self.device) + loss_norch = loss_fn_norch.forward(predictions_norch, labels_norch) + loss_norch.backward() # Backpropagate the loss + grad_norch = predictions_norch.grad + + predictions_torch = torch.tensor([1.1, 2, 3, 4], requires_grad=True, device=self.device) + labels_torch = torch.tensor([4, 3, 2.1, 1], device=self.device) + loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch) + loss_torch_expected.backward() # Backpropagate the loss + grad_torch_expected = predictions_torch.grad + + # Convert norch gradient to torch tensor for comparison + grad_norch_torch = utils.to_torch(grad_norch).to(self.device) + + self.assertTrue(utils.compare_torch(grad_norch_torch, grad_torch_expected)) + + def test_cross_entropy_loss_autograd(self): + """ + Test the CrossEntropyLoss with autograd functionality + """ + loss_fn_norch = norch.nn.CrossEntropyLoss() + loss_fn_torch = torch.nn.CrossEntropyLoss() + + # Test case 1: Single class, single sample + predictions_norch = norch.Tensor([2.0, 1.0, 0.1], requires_grad=True).to(self.device) + labels_norch = norch.Tensor([0]).to(self.device) + loss_norch = loss_fn_norch.forward(predictions_norch, labels_norch) + + loss_norch.backward() # Backpropagate the loss + grad_norch = predictions_norch.grad + + predictions_torch = torch.tensor([2.0, 1.0, 0.1], requires_grad=True, device=self.device) + labels_torch = torch.tensor(0, device=self.device) + loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch) + loss_torch_expected.backward() # Backpropagate the loss + grad_torch_expected = predictions_torch.grad + + # Convert norch gradient to torch tensor for comparison + grad_norch_torch = utils.to_torch(grad_norch).to(self.device) + + self.assertTrue(utils.compare_torch(grad_norch_torch, grad_torch_expected)) + + # Test case 2: Multiple classes, multiple samples + predictions_norch = norch.Tensor([[0.5, 1.5, 2.5], [1.0, 2.0, 3.0]], requires_grad=True).to(self.device) + labels_norch = norch.Tensor([2, 1]).to(self.device) + loss_norch = loss_fn_norch.forward(predictions_norch, labels_norch) + loss_norch.backward() # Backpropagate the loss + grad_norch = predictions_norch.grad + + predictions_torch = torch.tensor([[0.5, 1.5, 2.5], [1.0, 2.0, 3.0]], requires_grad=True, device=self.device) + labels_torch = torch.tensor([2, 1], device=self.device) + loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch) + loss_torch_expected.backward() # Backpropagate the loss + grad_torch_expected = predictions_torch.grad + + # Convert norch gradient to torch tensor for comparison + grad_norch_torch = utils.to_torch(grad_norch).to(self.device) + + self.assertTrue(utils.compare_torch(grad_norch_torch, grad_torch_expected)) + + # Test case 3: Edge case - all predictions are zero + predictions_norch = norch.Tensor([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], requires_grad=True).to(self.device) + labels_norch = norch.Tensor([1, 2]).to(self.device) + loss_norch = loss_fn_norch.forward(predictions_norch, labels_norch) + loss_norch.backward() # Backpropagate the loss + grad_norch = predictions_norch.grad + + predictions_torch = torch.tensor([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], requires_grad=True, device=self.device) + labels_torch = torch.tensor([1, 2], device=self.device) + loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch) + loss_torch_expected.backward() # Backpropagate the loss + grad_torch_expected = predictions_torch.grad + + # Convert norch gradient to torch tensor for comparison + grad_norch_torch = utils.to_torch(grad_norch).to(self.device) + + self.assertTrue(utils.compare_torch(grad_norch_torch, grad_torch_expected)) + + # Test case 4: Batched class probabilities instead of class index + predictions_norch = norch.Tensor([[0.5, 0.2, 0.1], [0.1, 0.5, 0.7]], requires_grad=True).to(self.device) + labels_norch = norch.Tensor([[1., 0, 0], [0, 1, 0]]).to(self.device) + loss_norch = loss_fn_norch.forward(predictions_norch, labels_norch) + loss_norch.backward() # Backpropagate the loss + grad_norch = predictions_norch.grad + + predictions_torch = torch.tensor([[0.5, 0.2, 0.1], [0.1, 0.5, 0.7]], requires_grad=True, device=self.device) + labels_torch = torch.tensor([[1., 0, 0], [0, 1, 0]], device=self.device) + loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch) + loss_torch_expected.backward() # Backpropagate the loss + grad_torch_expected = predictions_torch.grad + + # Convert norch gradient to torch tensor for comparison + grad_norch_torch = utils.to_torch(grad_norch).to(self.device) + + self.assertTrue(utils.compare_torch(grad_norch_torch, grad_torch_expected)) + + + # implement grad pure softmax --> 0 + # def test_softmax(self): + # """ + # Test autograd from softmax + # """ + # norch_tensor = norch.Tensor([[[-5, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True).to(self.device) + # norch_softmax = norch.softmax(norch_tensor, dim=1) + # norch_result = norch_softmax.sum() + + # norch_result.backward() + # norch_tensor_grad = utils.to_torch(norch_tensor.grad).to(self.device) + + # torch_tensor = torch.tensor([[[-5, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True).to(self.device) + # torch_softmax = torch.softmax(torch_tensor, dim=1) + # torch_result = torch_softmax.sum() + + # torch_result.backward() + # torch_tensor_grad = torch_tensor.grad + + # self.assertTrue(utils.compare_torch(norch_tensor_grad, torch_tensor_grad)) + + # norch_tensor = norch.Tensor([[[10, 1], [-4, 0]], [[5., 50], [7, 8]]], requires_grad=True).to(self.device) + # norch_softmax = norch.softmax(norch_tensor, dim=2) + # norch_result = norch_softmax.sum() + + # norch_result.backward() + # norch_tensor_grad = utils.to_torch(norch_tensor.grad).to(self.device) + + # torch_tensor = torch.tensor([[[10, 1], [-4, 0]], [[5., 50], [7, 8]]], requires_grad=True).to(self.device) + + # torch_softmax = torch.softmax(torch_tensor, dim=2) + # torch_result = torch_softmax.sum() + # torch_result.backward() + # torch_tensor_grad = torch_tensor.grad + + # self.assertTrue(utils.compare_torch(norch_tensor_grad, torch_tensor_grad)) + + + def test_reshape(self): + """ + Test autograd from reshaping a tensor: tensor.reshape(shape) + """ + norch_tensor_reshape = norch.Tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True).to(self.device) + new_shape = [2, 4] + norch_result_reshape = norch_tensor_reshape.reshape(new_shape).sum() + norch_result_reshape.backward() + norch_tensor_grad_reshape = utils.to_torch(norch_tensor_reshape.grad).to(self.device) + + torch_tensor_reshape = torch.tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True, device=self.device) + torch_result_reshape = torch_tensor_reshape.reshape(new_shape).sum() + torch_result_reshape.backward() + torch_tensor_grad_reshape = torch_tensor_reshape.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad_reshape, torch_tensor_grad_reshape)) + + + def test_transpose_axes(self): + """ + Test autograd from transposing a tensor with specific axes: tensor.transpose(axis1, axis2) + """ + norch_tensor_transpose = norch.Tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True).to(self.device) + axis1, axis2 = 0, 2 + norch_result_transpose = norch_tensor_transpose.transpose(axis1, axis2).sum() + norch_result_transpose.backward() + norch_tensor_grad_transpose = utils.to_torch(norch_tensor_transpose.grad).to(self.device) + + torch_tensor_transpose = torch.tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True, device=self.device) + torch_result_transpose = torch_tensor_transpose.transpose(axis1, axis2).sum() + torch_result_transpose.backward() + torch_tensor_grad_transpose = torch_tensor_transpose.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad_transpose, torch_tensor_grad_transpose)) + + + def test_T(self): + """ + Test autograd from transposing a tensor using .T attribute + """ + norch_tensor_T = norch.Tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True).to(self.device) + norch_result_T = norch_tensor_T.T.sum() + norch_result_T.backward() + norch_tensor_grad_T = utils.to_torch(norch_tensor_T.grad).to(self.device) + + torch_tensor_T = torch.tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True, device=self.device) + torch_result_T = torch_tensor_T.mT.sum() + torch_result_T.backward() + torch_tensor_grad_T = torch_tensor_T.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad_T, torch_tensor_grad_T)) + + def test_reshape_then_matmul(self): + """ + Test autograd from reshaping a tensor then performing matrix multiplication: matmul(tensor1.reshape(shape), tensor2) + """ + norch_tensor1 = norch.Tensor([[1, 2.1], [3, -4], [5, 6], [7, 8]], requires_grad=True).to(self.device) + norch_tensor2 = norch.Tensor([[1, 5.1], [0.1, -4], [0, 6], [7, 8]], requires_grad=True).to(self.device) + + new_shape = [2, 4] + + norch_result_reshape_matmul = (norch_tensor1.reshape(new_shape) @ norch_tensor2).sum() + norch_result_reshape_matmul.backward() + norch_tensor_grad_reshape_matmul1 = utils.to_torch(norch_tensor1.grad).to(self.device) + norch_tensor_grad_reshape_matmul2 = utils.to_torch(norch_tensor2.grad).to(self.device) + + torch_tensor1 = torch.tensor([[1, 2.1], [3, -4], [5, 6], [7, 8]], dtype=torch.float32, requires_grad=True, device=self.device) + torch_tensor2 = torch.tensor([[1, 5.1], [0.1, -4], [0, 6], [7, 8]], dtype=torch.float32, requires_grad=True, device=self.device) + + torch_result_reshape_matmul = (torch_tensor1.reshape(new_shape) @ torch_tensor2).sum() + torch_result_reshape_matmul.backward() + torch_tensor_grad_reshape_matmul1 = torch_tensor1.grad + torch_tensor_grad_reshape_matmul2 = torch_tensor2.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad_reshape_matmul1, torch_tensor_grad_reshape_matmul1)) + self.assertTrue(utils.compare_torch(norch_tensor_grad_reshape_matmul2, torch_tensor_grad_reshape_matmul2)) + + def test_unsqueeze(self): + """ + Test autograd from unsqueezing a tensor: tensor.unsqueeze(dim) + """ + + # Unsqueeze at dim=0 + norch_tensor_unsqueeze = norch.Tensor([[1., 2], [3, 4]], requires_grad=True).to(self.device) + norch_result_unsqueeze_0 = norch_tensor_unsqueeze.unsqueeze(0).sum() + norch_result_unsqueeze_0.backward() + norch_tensor_grad_unsqueeze_0 = utils.to_torch(norch_tensor_unsqueeze.grad).to(self.device) + + torch_tensor_unsqueeze = torch.tensor([[1., 2], [3, 4]], requires_grad=True, device=self.device) + torch_result_unsqueeze_0 = torch_tensor_unsqueeze.unsqueeze(0).sum() + torch_result_unsqueeze_0.backward() + torch_tensor_grad_unsqueeze_0 = torch_tensor_unsqueeze.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad_unsqueeze_0, torch_tensor_grad_unsqueeze_0)) + + # Unsqueeze at dim=1 + norch_tensor_unsqueeze = norch.Tensor([[1., 2.], [3, 4]], requires_grad=True).to(self.device) + norch_result_unsqueeze_1 = norch_tensor_unsqueeze.unsqueeze(1).sum() + norch_result_unsqueeze_1.backward() + norch_tensor_grad_unsqueeze_1 = utils.to_torch(norch_tensor_unsqueeze.grad).to(self.device) + + torch_tensor_unsqueeze = torch.tensor([[1., 2.], [3, 4]], requires_grad=True, device=self.device) + torch_result_unsqueeze_1 = torch_tensor_unsqueeze.unsqueeze(1).sum() + torch_result_unsqueeze_1.backward() + torch_tensor_grad_unsqueeze_1 = torch_tensor_unsqueeze.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad_unsqueeze_1, torch_tensor_grad_unsqueeze_1)) + + # Unsqueeze at dim=2 + norch_tensor_unsqueeze = norch.Tensor([[1., 2], [3, 4]], requires_grad=True).to(self.device) + norch_result_unsqueeze_2 = norch_tensor_unsqueeze.unsqueeze(2).sum() + norch_result_unsqueeze_2.backward() + norch_tensor_grad_unsqueeze_2 = utils.to_torch(norch_tensor_unsqueeze.grad).to(self.device) + + torch_tensor_unsqueeze = torch.tensor([[1., 2], [3, 4]], requires_grad=True, device=self.device) + torch_result_unsqueeze_2 = torch_tensor_unsqueeze.unsqueeze(2).sum() + torch_result_unsqueeze_2.backward() + torch_tensor_grad_unsqueeze_2 = torch_tensor_unsqueeze.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad_unsqueeze_2, torch_tensor_grad_unsqueeze_2)) + + def test_unsqueeze_then_matmul(self): + """ + Test autograd from unsqueezing a tensor then performing matrix multiplication: matmul(tensor1.unsqueeze(dim), tensor2) + """ + norch_tensor1 = norch.Tensor([[1, 2], [3, 4]], requires_grad=True).to(self.device) + norch_tensor2 = norch.Tensor([[1, 2], [3, 4]], requires_grad=True).to(self.device) + + # Unsqueeze at dim=0 then matmul + norch_result_unsqueeze_matmul = (norch_tensor1 @ norch_tensor2.unsqueeze(0)).sum() + norch_result_unsqueeze_matmul.backward() + norch_tensor_grad_unsqueeze_matmul1 = utils.to_torch(norch_tensor1.grad).to(self.device) + norch_tensor_grad_unsqueeze_matmul2 = utils.to_torch(norch_tensor2.grad).to(self.device) + + torch_tensor1 = torch.tensor([[1, 2], [3, 4]], dtype=torch.float32, requires_grad=True, device=self.device) + torch_tensor2 = torch.tensor([[1, 2], [3, 4]], dtype=torch.float32, requires_grad=True, device=self.device) + + torch_result_unsqueeze_matmul = (torch_tensor1 @ torch_tensor2.unsqueeze(0)).sum() + torch_result_unsqueeze_matmul.backward() + torch_tensor_grad_unsqueeze_matmul1 = torch_tensor1.grad + torch_tensor_grad_unsqueeze_matmul2 = torch_tensor2.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad_unsqueeze_matmul1, torch_tensor_grad_unsqueeze_matmul1)) + self.assertTrue(utils.compare_torch(norch_tensor_grad_unsqueeze_matmul2, torch_tensor_grad_unsqueeze_matmul2)) + + def test_squeeze(self): + """ + Test autograd from squeezing a tensor: tensor.squeeze(dim) + """ + # Squeeze at dim=0 + norch_tensor_squeeze = norch.Tensor([[[1., 2], [3, 4]]], requires_grad=True).to(self.device) + norch_result_squeeze_0 = norch_tensor_squeeze.squeeze(0).sum() + norch_result_squeeze_0.backward() + norch_tensor_grad_squeeze_0 = utils.to_torch(norch_tensor_squeeze.grad).to(self.device) + + torch_tensor_squeeze = torch.tensor([[[1., 2], [3, 4]]], requires_grad=True, device=self.device) + torch_result_squeeze_0 = torch_tensor_squeeze.squeeze(0).sum() + torch_result_squeeze_0.backward() + torch_tensor_grad_squeeze_0 = torch_tensor_squeeze.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad_squeeze_0, torch_tensor_grad_squeeze_0)) + + def test_squeeze_then_matmul(self): + """ + Test autograd from squeezing a tensor then performing matrix multiplication: matmul(tensor1.squeeze(dim), tensor2) + """ + norch_tensor1 = norch.Tensor([[[1., 2], [3, 4]]], requires_grad=True).to(self.device) + norch_tensor2 = norch.Tensor([[[1., 2], [3, 4]]], requires_grad=True).to(self.device) + + # Squeeze at dim=0 then matmul + norch_result_squeeze_matmul = (norch_tensor1.squeeze(0) @ norch_tensor2).sum() + norch_result_squeeze_matmul.backward() + norch_tensor_grad_squeeze_matmul1 = utils.to_torch(norch_tensor1.grad,).to(self.device) + norch_tensor_grad_squeeze_matmul2 = utils.to_torch(norch_tensor2.grad).to(self.device) + + torch_tensor1 = torch.tensor([[[1., 2], [3, 4]]], dtype=torch.float32, requires_grad=True, device=self.device) + torch_tensor2 = torch.tensor([[[1., 2], [3, 4]]], dtype=torch.float32, requires_grad=True, device=self.device) + + torch_result_squeeze_matmul = (torch_tensor1.squeeze(0) @ torch_tensor2).sum() + torch_result_squeeze_matmul.backward() + torch_tensor_grad_squeeze_matmul1 = torch_tensor1.grad + torch_tensor_grad_squeeze_matmul2 = torch_tensor2.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad_squeeze_matmul1, torch_tensor_grad_squeeze_matmul1)) + self.assertTrue(utils.compare_torch(norch_tensor_grad_squeeze_matmul2, torch_tensor_grad_squeeze_matmul2)) + + + def test_T_then_matmul(self): + """ + Test autograd from transposing a tensor then performing matrix multiplication: matmul(tensor.T, tensor) + """ + norch_tensor1 = norch.Tensor([[1, 2.1], [3, -4], [5, 6], [7, 8]], requires_grad=True) + norch_tensor2 = norch.Tensor([[1, 5.1], [0.1, -4], [0, 6], [7, 8]], requires_grad=True) + + norch_result_T_matmul = (norch_tensor1.T @ norch_tensor2).sum() + norch_result_T_matmul.backward() + norch_tensor_grad_T_matmul1 = utils.to_torch(norch_tensor1.grad).to(self.device) + norch_tensor_grad_T_matmult2 = utils.to_torch(norch_tensor2.grad).to(self.device) + + torch_tensor1 = torch.tensor([[1, 2.1], [3, -4], [5, 6], [7, 8]], dtype=torch.float32, requires_grad=True, device=self.device) + torch_tensor2 = torch.tensor([[1, 5.1], [0.1, -4], [0, 6], [7, 8]], dtype=torch.float32, requires_grad=True, device=self.device) + + torch_result_T_matmul = (torch_tensor1.T @ torch_tensor2).sum() + torch_result_T_matmul.backward() + torch_tensor_grad_T_matmul1 = torch_tensor1.grad + torch_tensor_grad_T_matmul2 = torch_tensor2.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad_T_matmul1, torch_tensor_grad_T_matmul1)) + self.assertTrue(utils.compare_torch(norch_tensor_grad_T_matmult2, torch_tensor_grad_T_matmul2)) + + def todo(self): + """ + The code has a problem on the following operation + tensor1.reshape(..) @ tensor1 + print(tensor1.grad) + (also transpsoe and .T) + """ + pass + + def test_transpose_axes_then_matmul(self): + """ + Test autograd from transposing a tensor with specific axes then performing matrix multiplication: matmul(tensor.transpose(axis1, axis2), tensor) + """ + norch_tensor1 = norch.Tensor([[1, 2.1], [3, -4], [5, 6], [7, 8]], requires_grad=True).to(self.device) + norch_tensor2 = norch.Tensor([[1, 5.1], [0.1, -4], [0, 6], [7, 8]], requires_grad=True).to(self.device) + + norch_result_transpose_matmul = (norch_tensor1.transpose(0, 1) @ norch_tensor2).sum() + norch_result_transpose_matmul.backward() + norch_tensor_grad_transpose_matmul1 = utils.to_torch(norch_tensor1.grad).to(self.device) + norch_tensor_grad_transpose_matmult2 = utils.to_torch(norch_tensor2.grad).to(self.device) + + torch_tensor1 = torch.tensor([[1, 2.1], [3, -4], [5, 6], [7, 8]], dtype=torch.float32, requires_grad=True, device=self.device) + torch_tensor2 = torch.tensor([[1, 5.1], [0.1, -4], [0, 6], [7, 8]], dtype=torch.float32, requires_grad=True, device=self.device) + + torch_result_transpose_matmul = (torch_tensor1.T @ torch_tensor2).sum() + torch_result_transpose_matmul.backward() + torch_tensor_grad_transpose_matmul1 = torch_tensor1.grad + torch_tensor_grad_transpose_matmul2 = torch_tensor2.grad + + self.assertTrue(utils.compare_torch(norch_tensor_grad_transpose_matmul1, torch_tensor_grad_transpose_matmul1)) + self.assertTrue(utils.compare_torch(norch_tensor_grad_transpose_matmult2, torch_tensor_grad_transpose_matmul2)) + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/build/lib/tests/test_nn.py b/build/lib/tests/test_nn.py new file mode 100644 index 0000000..532f5b6 --- /dev/null +++ b/build/lib/tests/test_nn.py @@ -0,0 +1,204 @@ +import unittest +import norch +from norch.utils import utils_unittests as utils +import torch +import os + +class TestNNModuleLoss(unittest.TestCase): + + def setUp(self): + self.device = os.environ.get('device') + if self.device is None or self.device != 'cuda': + self.device = 'cpu' + + print(f"Running tests on: {self.device}") + + def test_mse_loss(self): + """ + Test the MSELoss + """ + loss_fn_norch = norch.nn.MSELoss() + loss_fn_torch = torch.nn.MSELoss() + + # Test case 1: Predictions and labels are equal + predictions_norch = norch.Tensor([[1.1, 2, 3, 4], [1.1, 2, 3, 4]]).to(self.device) + labels_norch = norch.Tensor([[1.1, 2, 3, 4], [1.1, 2, 3, 3]]).to(self.device) + loss_norch = loss_fn_norch.forward(predictions_norch, labels_norch) + loss_torch_result = utils.to_torch(loss_norch).to(self.device) + + predictions_torch = torch.tensor([[1.1, 2, 3, 4], [1.1, 2, 3, 4]], device=self.device) + labels_torch = torch.tensor([[1.1, 2, 3, 4], [1.1, 2, 3, 3]], device=self.device) + + loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch) + + self.assertTrue(utils.compare_torch(loss_torch_result, loss_torch_expected)) + + # Test case 2: Predictions and labels are different + predictions_norch = norch.Tensor([1.1, 2, 3, 4]).to(self.device) + labels_norch = norch.Tensor([4, 3, 2.1, 1]).to(self.device) + loss_norch = loss_fn_norch.forward(predictions_norch, labels_norch) + loss_torch_result = utils.to_torch(loss_norch).to(self.device) + + predictions_torch = torch.tensor([1.1, 2, 3, 4], device=self.device) + labels_torch = torch.tensor([4, 3, 2.1, 1], device=self.device) + + loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch) + + self.assertTrue(utils.compare_torch(loss_torch_result, loss_torch_expected)) + + def test_cross_entropy_loss(self): + """ + Test the CrossEntropyLoss + """ + loss_fn_norch = norch.nn.CrossEntropyLoss() + loss_fn_torch = torch.nn.CrossEntropyLoss() + + # Test case 1: Single class, single sample + predictions_norch = norch.Tensor([2.0, 1.0, 0.1]).to(self.device) + labels_norch = norch.Tensor([0]).to(self.device) + loss_norch = loss_fn_norch.forward(predictions_norch, labels_norch) + + loss_torch_result = utils.to_torch(loss_norch).to(self.device) + + predictions_torch = torch.tensor([2.0, 1.0, 0.1], device=self.device) + labels_torch = torch.tensor(0, device=self.device) + + loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch) + + self.assertTrue(utils.compare_torch(loss_torch_result, loss_torch_expected)) + + # Test case 2: Multiple classes, multiple samples + predictions_norch = norch.Tensor([[0.5, 1.5, 2.5], [1.0, 2.0, 3.0]]).to(self.device) + labels_norch = norch.Tensor([2, 1]).to(self.device) + loss_norch = loss_fn_norch.forward(predictions_norch, labels_norch) + loss_torch_result = utils.to_torch(loss_norch).to(self.device) + + + predictions_torch = torch.tensor([[0.5, 1.5, 2.5], [1.0, 2.0, 3.0]], device=self.device) + labels_torch = torch.tensor([2, 1], device=self.device) + + loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch) + + self.assertTrue(utils.compare_torch(loss_torch_result, loss_torch_expected)) + + # Test case 3: Edge case - all predictions are zero + predictions_norch = norch.Tensor([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]).to(self.device) + labels_norch = norch.Tensor([1, 2]).to(self.device) + loss_norch = loss_fn_norch.forward(predictions_norch, labels_norch) + loss_torch_result = utils.to_torch(loss_norch).to(self.device) + + predictions_torch = torch.tensor([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], device=self.device) + labels_torch = torch.tensor([1, 2], device=self.device) + + loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch) + + self.assertTrue(utils.compare_torch(loss_torch_result, loss_torch_expected)) + + # Test case 4: Class probabilities instead of class index + predictions_norch = norch.Tensor([0.5, 0.2, 0.1]).to(self.device) + labels_norch = norch.Tensor([1., 0, 0]).to(self.device) + loss_norch = loss_fn_norch.forward(predictions_norch, labels_norch) + loss_torch_result = utils.to_torch(loss_norch).to(self.device) + + predictions_torch = torch.tensor([0.5, 0.2, 0.1]) + labels_torch = torch.tensor([1., 0, 0]) + + predictions_torch.to(self.device) + labels_torch.to(self.device) + + loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch) + + self.assertTrue(utils.compare_torch(loss_torch_result, loss_torch_expected)) + + # Test case 4: Batched class probabilities instead of class index + predictions_norch = norch.Tensor([[0.5, 0.2, 0.1], [0.1, 0.5, 0.7]]).to(self.device) + labels_norch = norch.Tensor([[1., 0, 0], [0, 1, 0]]).to(self.device) + loss_norch = loss_fn_norch.forward(predictions_norch, labels_norch) + loss_torch_result = utils.to_torch(loss_norch).to(self.device) + + predictions_torch = torch.tensor([[0.5, 0.2, 0.1], [0.1, 0.5, 0.7]], device=self.device) + labels_torch = torch.tensor([[1., 0, 0], [0, 1, 0]], device=self.device) + + loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch) + + self.assertTrue(utils.compare_torch(loss_torch_result, loss_torch_expected)) + + +class TestNNModuleActivationFn(unittest.TestCase): + + def setUp(self): + self.device = os.environ.get('device') + if self.device is None or self.device != 'cuda': + self.device = 'cpu' + + def test_sigmoid_activation(self): + """ + Test Sigmoid activation function + """ + sigmoid_fn_norch = norch.nn.Sigmoid() + sigmoid_fn_torch = torch.nn.Sigmoid() + + # Test case 1: Positive input + x = norch.Tensor([[1, 2, 3]]).to(self.device) + sigmoid_norch = sigmoid_fn_norch.forward(x) + sigmoid_torch_result = utils.to_torch(sigmoid_norch).to(self.device) + + x = torch.tensor([[1, 2, 3]], device=self.device) + + sigmoid_torch_expected = sigmoid_fn_torch.forward(x) + + self.assertTrue(utils.compare_torch(sigmoid_torch_result, sigmoid_torch_expected)) + + # Test case 1: Negative input + x = norch.Tensor([-1, 2, -3]).to(self.device) + sigmoid_norch = sigmoid_fn_norch.forward(x) + sigmoid_torch_result = utils.to_torch(sigmoid_norch).to(self.device) + + x = torch.tensor([-1, 2, -3], device=self.device) + + sigmoid_torch_expected = sigmoid_fn_torch.forward(x) + + self.assertTrue(utils.compare_torch(sigmoid_torch_result, sigmoid_torch_expected)) + + # Test case 1: Zero input + x = norch.Tensor([0, 0, 0]).to(self.device) + sigmoid_norch = sigmoid_fn_norch.forward(x) + sigmoid_torch_result = utils.to_torch(sigmoid_norch).to(self.device) + + x = torch.tensor([0, 0, 0], device=self.device) + + sigmoid_torch_expected = sigmoid_fn_torch.forward(x) + + self.assertTrue(utils.compare_torch(sigmoid_torch_result, sigmoid_torch_expected)) + + def test_softmax_activation(self): + """ + Test Softmax activation function + """ + + # Test different axes + axes = [0, 1, 2, -1] + + # Define the input tensors for different test cases + test_cases = [ + (norch.Tensor([[[1., 2, 3], [4, 5, 6]]]).to(self.device), torch.tensor([[[1., 2, 3], [4, 5, 6]]], device=self.device)), + (norch.Tensor([[[1., -1, 0], [2, -2, 0]]]).to(self.device), torch.tensor([[[1., -1, 0], [2, -2, 0]]], device=self.device)), + (norch.Tensor([[[0., 0, 0], [0, 0, 0]]]).to(self.device), torch.tensor([[[0., 0, 0], [0, 0, 0]]], device=self.device)) + ] + + for dim in axes: + softmax_fn_norch = norch.nn.Softmax(dim=dim) + softmax_fn_torch = torch.nn.Softmax(dim=dim) + + for norch_input, torch_input in test_cases: + + # Forward pass using norch + softmax_norch = softmax_fn_norch.forward(norch_input) + softmax_torch_result = utils.to_torch(softmax_norch).to(self.device) + + # Forward pass using torch + softmax_torch_expected = softmax_fn_torch.forward(torch_input) + + # Compare the results + self.assertTrue(utils.compare_torch(softmax_torch_result, softmax_torch_expected)) + diff --git a/build/lib/tests/test_operations.py b/build/lib/tests/test_operations.py new file mode 100644 index 0000000..262cee4 --- /dev/null +++ b/build/lib/tests/test_operations.py @@ -0,0 +1,753 @@ +import unittest +import norch +from norch.utils import utils_unittests as utils +import torch +import sys +import os + +class TestTensorOperations(unittest.TestCase): + + def setUp(self): + self.device = os.environ.get('device') + if self.device is None or self.device != 'cuda': + self.device = 'cpu' + + print(f"Running tests on: {self.device}") + + def test_creation_and_conversion(self): + """ + Test creation and convertion of norch tensor to pytorch + """ + norch_tensor = norch.Tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + torch_tensor = utils.to_torch(norch_tensor) + self.assertTrue(torch.is_tensor(torch_tensor)) + + def test_addition(self): + """ + Test addition two tensors: tensor1 + tensor2 + """ + norch_tensor1 = norch.Tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + norch_tensor2 = norch.Tensor([[[1, 1], [1, 1]], [[1, 1], [1, 1]]]).to(self.device) + norch_result = norch_tensor1 + norch_tensor2 + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor1 = torch.tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + torch_tensor2 = torch.tensor([[[1, 1], [1, 1]], [[1, 1], [1, 1]]]).to(self.device) + torch_expected = torch_tensor1 + torch_tensor2 + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + def test_addition_broadcasted(self): + """ + Test addition of two tensors with broadcasting: tensor1 + tensor2 + """ + norch_tensor1 = norch.Tensor([[[1, 2, 3], [4, 5, 6]]]).to(self.device) # Shape (1, 2, 3) + norch_tensor2 = norch.Tensor([1, 1, 1]).to(self.device) # Shape (3) + norch_result = norch_tensor1 + norch_tensor2 + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor1 = torch.tensor([[[1, 2, 3], [4, 5, 6]]]).to(self.device) # Shape (1, 2, 3) + torch_tensor2 = torch.tensor([1, 1, 1]).to(self.device) # Shape (3) + torch_expected = torch_tensor1 + torch_tensor2 + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + norch_tensor1 = norch.Tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]]).to(self.device) # Shape (1, 2, 3) + norch_tensor2 = norch.Tensor([[10, 10], [5, 6]]).to(self.device) # Shape (3) + norch_result = norch_tensor1 + norch_tensor2 + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor1 = torch.tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]]).to(self.device) # Shape (1, 2, 3) + torch_tensor2 = torch.tensor([[[10, 10], [5, 6]]]).to(self.device) # Shape (3) + torch_expected = torch_tensor1 + torch_tensor2 + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + # reversed order broadcasting + norch_tensor1 = norch.Tensor([[0, 2]]).to(self.device) + norch_tensor2 = norch.Tensor([[3, 4], [5, -1]]).to(self.device) + norch_result = norch_tensor1 + norch_tensor2 + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor1 = torch.tensor([[0, 2]]).to(self.device) + torch_tensor2 = torch.tensor([[3, 4], [5, -1]]).to(self.device) + torch_expected = torch_tensor1 + torch_tensor2 + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + norch_result = norch_tensor2 + norch_tensor1 + torch_expected = torch_tensor2 + torch_tensor1 + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + + + def test_subtraction(self): + """ + Test subtraction of two tensors: tensor1 - tensor2 + """ + norch_tensor1 = norch.Tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + norch_tensor2 = norch.Tensor([[[1, 1], [1, 1]], [[1, 1], [1, 1]]]).to(self.device) + norch_result = norch_tensor1 - norch_tensor2 + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor1 = torch.tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + torch_tensor2 = torch.tensor([[[1, 1], [1, 1]], [[1, 1], [1, 1]]]).to(self.device) + torch_expected = torch_tensor1 - torch_tensor2 + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + def test_broadcasting_subtraction(self): + """ + Test subtraction of two tensors with broadcasting: tensor1 - tensor2 + """ + norch_tensor1 = norch.Tensor([[[1, 2, 3], [4, 5, 6]]]).to(self.device) # Shape (1, 2, 3) + norch_tensor2 = norch.Tensor([1, 1, 1]).to(self.device) # Shape (3) + norch_result = norch_tensor1 - norch_tensor2 + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor1 = torch.tensor([[[1, 2, 3], [4, 5, 6]]]).to(self.device) # Shape (1, 2, 3) + torch_tensor2 = torch.tensor([1, 1, 1]).to(self.device) # Shape (3) + torch_expected = torch_tensor1 - torch_tensor2 + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + # reversed order broadcasting + norch_result = norch_tensor2 - norch_tensor1 + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_expected = torch_tensor2 - torch_tensor1 + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + + def test_division_by_scalar(self): + """ + Test division of a tensor by a scalar: tensor / scalar + """ + norch_tensor = norch.Tensor([[[2, 4], [6, -8]], [[10, 12], [14, 16]]]).to(self.device) + scalar = 2 + norch_result = norch_tensor / scalar + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor = torch.tensor([[[2, 4], [6, -8]], [[10, 12], [14, 16]]]).to(self.device) + torch_expected = torch_tensor / scalar + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + def test_scalar_division_by_tensor(self): + """ + Test scalar division by a tensor: scalar / tensor + """ + scalar = 10 + norch_tensor = norch.Tensor([[[2, 4], [6, -8]], [[10, 12], [14, 16]]]).to(self.device) + norch_result = scalar / norch_tensor + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor = torch.tensor([[[2, 4], [6, -8]], [[10, 12], [14, 16]]]).to(self.device) + torch_expected = scalar / torch_tensor + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + def test_matrix_multiplication(self): + """ + Test matrix multiplication: tensor1 @ tensor2 + """ + norch_tensor1 = norch.Tensor([[[1., 2], [3, 4]], [[5, 6], [7, 8]]]).to(self.device) + norch_tensor2 = norch.Tensor([[[1., 0], [0, 1]], [[-1, 0], [0, -1]]]).to(self.device) + norch_result = norch_tensor1 @ norch_tensor2 + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor1 = torch.tensor([[[1., 2], [3, 4]], [[5, 6], [7, 8]]]).to(self.device) + torch_tensor2 = torch.tensor([[[1., 0], [0, 1]], [[-1, 0], [0, -1]]]).to(self.device) + torch_expected = torch_tensor1 @ torch_tensor2 + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + def test_elementwise_multiplication_by_scalar(self): + """ + Test elementwise multiplication of a tensor by a scalar: tensor * scalar + """ + norch_tensor = norch.Tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + scalar = 2 + norch_result = norch_tensor * scalar + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor = torch.tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + torch_expected = torch_tensor * scalar + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + def test_elementwise_multiplication_by_tensor(self): + """ + Test elementwise multiplication of two tensors: tensor1 * tensor2 + """ + norch_tensor1 = norch.Tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + norch_tensor2 = norch.Tensor([[[2, 2], [2, 2]], [[2, 2], [2, 2]]]).to(self.device) + norch_result = norch_tensor1 * norch_tensor2 + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor1 = torch.tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + torch_tensor2 = torch.tensor([[[2, 2], [2, 2]], [[2, 2], [2, 2]]]).to(self.device) + torch_expected = torch_tensor1 * torch_tensor2 + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + def test_reshape(self): + """ + Test reshaping of a tensor: tensor.reshape(shape) + """ + norch_tensor = norch.Tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + new_shape = [2, 4] + norch_result = norch_tensor.reshape(new_shape) + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor = torch.tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + torch_expected = torch_tensor.reshape(new_shape) + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + def test_unsqueeze(self): + """ + Test unsqueeze operation on a tensor + """ + norch_tensor = norch.Tensor([[1, 2], [3, 4]]).to(self.device) + + # Unsqueeze at dim=0 + norch_unsqueeze_0 = norch_tensor.unsqueeze(0) + torch_unsqueeze_0 = utils.to_torch(norch_unsqueeze_0).to(self.device) + torch_tensor = torch.tensor([[1, 2], [3, 4]]).to(self.device) + torch_expected_0 = torch_tensor.unsqueeze(0) + self.assertTrue(utils.compare_torch(torch_unsqueeze_0, torch_expected_0)) + + # Unsqueeze at dim=1 + norch_unsqueeze_1 = norch_tensor.unsqueeze(1) + torch_unsqueeze_1 = utils.to_torch(norch_unsqueeze_1).to(self.device) + torch_expected_1 = torch_tensor.unsqueeze(1) + self.assertTrue(utils.compare_torch(torch_unsqueeze_1, torch_expected_1)) + + # Unsqueeze at dim=2 + norch_unsqueeze_2 = norch_tensor.unsqueeze(2) + torch_unsqueeze_2 = utils.to_torch(norch_unsqueeze_2).to(self.device) + torch_expected_2 = torch_tensor.unsqueeze(2) + self.assertTrue(utils.compare_torch(torch_unsqueeze_2, torch_expected_2)) + + # Unsqueeze at dim=-1 + norch_unsqueeze_neg_1 = norch_tensor.unsqueeze(-1) + torch_unsqueeze_neg_1 = utils.to_torch(norch_unsqueeze_neg_1).to(self.device) + torch_expected_neg_1 = torch_tensor.unsqueeze(-1) + self.assertTrue(utils.compare_torch(torch_unsqueeze_neg_1, torch_expected_neg_1)) + + # Unsqueeze at dim=-2 + norch_unsqueeze_neg_2 = norch_tensor.unsqueeze(-2) + torch_unsqueeze_neg_2 = utils.to_torch(norch_unsqueeze_neg_2).to(self.device) + torch_expected_neg_2 = torch_tensor.unsqueeze(-2) + self.assertTrue(utils.compare_torch(torch_unsqueeze_neg_2, torch_expected_neg_2)) + + def test_squeeze(self): + """ + Test squeeze operation on a tensor + """ + # Create a tensor with some dimensions of size 1 + norch_tensor = norch.Tensor([[[1, 2], [3, 4]]]).to(self.device) # shape [1, 2, 2] + + # Squeeze at dim=0 + norch_squeeze_0 = norch_tensor.squeeze(0) + torch_squeeze_0 = utils.to_torch(norch_squeeze_0).to(self.device) + torch_tensor = torch.tensor([[[1, 2], [3, 4]]]).to(self.device) + torch_expected_0 = torch_tensor.squeeze(0) + self.assertTrue(utils.compare_torch(torch_squeeze_0, torch_expected_0)) + + # Create a tensor with a dimension of size 1 in the middle + norch_tensor_middle_1 = norch.Tensor([[[1, 2]], [[3, 4]]]).to(self.device) # shape [2, 1, 2] + + # Squeeze at dim=1 + norch_squeeze_1 = norch_tensor_middle_1.squeeze(1) + torch_squeeze_1 = utils.to_torch(norch_squeeze_1).to(self.device) + torch_tensor_middle_1 = torch.tensor([[[1, 2]], [[3, 4]]]).to(self.device) + torch_expected_1 = torch_tensor_middle_1.squeeze(1) + self.assertTrue(utils.compare_torch(torch_squeeze_1, torch_expected_1)) + + # Squeeze at dim=-2 (same as dim=1 in this case) + norch_squeeze_neg_2 = norch_tensor_middle_1.squeeze(-2) + torch_squeeze_neg_2 = utils.to_torch(norch_squeeze_neg_2).to(self.device) + torch_expected_neg_2 = torch_tensor_middle_1.squeeze(-2) + self.assertTrue(utils.compare_torch(torch_squeeze_neg_2, torch_expected_neg_2)) + + # Squeeze all dimensions of size 1 (None) + norch_tensor_all_1 = norch.Tensor([[[[1, 2], [3, 4]]]]).to(self.device) # shape [1, 1, 2, 2] + norch_squeeze_all = norch_tensor_all_1.squeeze() + torch_squeeze_all = utils.to_torch(norch_squeeze_all).to(self.device) + torch_tensor_all_1 = torch.tensor([[[[1, 2], [3, 4]]]]).to(self.device) + torch_expected_all = torch_tensor_all_1.squeeze() + self.assertTrue(utils.compare_torch(torch_squeeze_all, torch_expected_all)) + + # Squeeze no dimensions (no dimensions of size 1) + norch_tensor_no_1 = norch.Tensor([[1, 2], [3, 4]]).to(self.device) # shape [2, 2] + norch_squeeze_none = norch_tensor_no_1.squeeze() + torch_squeeze_none = utils.to_torch(norch_squeeze_none).to(self.device) + torch_tensor_no_1 = torch.tensor([[1, 2], [3, 4]]).to(self.device) + torch_expected_none = torch_tensor_no_1.squeeze() + self.assertTrue(utils.compare_torch(torch_squeeze_none, torch_expected_none)) + + + def test_transpose(self): + """ + Test transposition of a tensor: tensor.transpose(dim1, dim2) + """ + norch_tensor = norch.Tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + dim1, dim2 = 0, 2 + norch_result = norch_tensor.transpose(dim1, dim2) + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor = torch.tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + torch_expected = torch_tensor.transpose(dim1, dim2) + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + def test_logarithm(self): + """ + Test elementwise logarithm of a tensor: tensor.log() + """ + norch_tensor = norch.Tensor([[[1, 2], [3, 4]], [[5, 6], [7, 8]]]).to(self.device) + norch_result = norch_tensor.log() + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor = torch.tensor([[[1, 2], [3, 4]], [[5, 6], [7, 8]]]).to(self.device) + torch_expected = torch.log(torch_tensor) + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + def test_sum(self): + """ + Test summation of a tensor: tensor.sum() + """ + norch_tensor = norch.Tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + norch_result = norch_tensor.sum() + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor = torch.tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + torch_expected = torch.sum(torch_tensor) + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + def test_sum_axis(self): + """ + Test summation of a tensor along a specific axis without keeping the dimensions + """ + norch_tensor = norch.Tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + norch_result = norch_tensor.sum(axis=1) + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor = torch.tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + torch_expected = torch.sum(torch_tensor, dim=1) + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + # negative axis + + norch_tensor = norch.Tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + norch_result = norch_tensor.sum(axis=-2) + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor = torch.tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + torch_expected = torch.sum(torch_tensor, dim=-2) + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + + def test_sum_axis_keepdim(self): + """ + Test summation of a tensor along a specific axis with keepdim=True + """ + norch_tensor = norch.Tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + norch_result = norch_tensor.sum(axis=1, keepdim=True) + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor = torch.tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + torch_expected = torch.sum(torch_tensor, dim=1, keepdim=True) + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + def test_max(self): + """ + Test max of a tensor: tensor.max() + """ + norch_tensor = norch.Tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + norch_result = norch_tensor.max() + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor = torch.tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + torch_expected = torch.max(torch_tensor) + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + def test_max_axis(self): + """ + Test max of a tensor along a specific axis without keeping the dimensions + """ + norch_tensor = norch.Tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + norch_result = norch_tensor.max(axis=1) + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor = torch.tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + torch_expected, _ = torch.max(torch_tensor, dim=1) + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + # negative axis + + norch_tensor = norch.Tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + norch_result = norch_tensor.max(axis=-1) + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor = torch.tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + torch_expected, _ = torch.max(torch_tensor, dim=-1) + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + + def test_max_axis_keepdim(self): + """ + Test max of a tensor along a specific axis with keepdim=True + """ + norch_tensor = norch.Tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + norch_result = norch_tensor.max(axis=1, keepdim=True) + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor = torch.tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + torch_expected, _ = torch.max(torch_tensor, dim=1, keepdim=True) + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + def test_min(self): + """ + Test min of a tensor: tensor.min() + """ + norch_tensor = norch.Tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + norch_result = norch_tensor.min() + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor = torch.tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + torch_expected = torch.min(torch_tensor) + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + def test_min_axis(self): + """ + Test min of a tensor along a specific axis without keeping the dimensions + """ + norch_tensor = norch.Tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + norch_result = norch_tensor.min(axis=1) + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor = torch.tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + torch_expected, _ = torch.min(torch_tensor, dim=1) + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + # negative axis + + norch_tensor = norch.Tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + norch_result = norch_tensor.min(axis=-1) + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor = torch.tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + torch_expected, _ = torch.min(torch_tensor, dim=-1) + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + + def test_min_axis_keepdim(self): + """ + Test min of a tensor along a specific axis with keepdim=True + """ + norch_tensor = norch.Tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + norch_result = norch_tensor.min(axis=1, keepdim=True) + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor = torch.tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + torch_expected, _ = torch.min(torch_tensor, dim=1, keepdim=True) + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + def test_1D_T(self): + """ + Test transposition of a 1D tensor. + """ + norch_tensor = norch.Tensor([1, 2, 3, 4]).to(self.device) + norch_result = norch_tensor.T + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor = torch.tensor([1, 2, 3, 4]).to(self.device) + torch_expected = torch_tensor.T + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + def test_2D_T(self): + """ + Test transposition of a 2D tensor. + """ + norch_tensor = norch.Tensor([[1, 2, 3], [4, 5, 6]]).to(self.device) + norch_result = norch_tensor.T + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor = torch.tensor([[1, 2, 3], [4, 5, 6]]).to(self.device) + torch_expected = torch_tensor.T + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + def test_3D_T(self): + """ + Test transposition of a tensor: tensor.T + """ + norch_tensor = norch.Tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + norch_result = norch_tensor.T + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor = torch.tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + torch_expected = torch_tensor.T + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + def test_matmul(self): + """ + Test matrix multiplication: MxP = NxM @ MxP + """ + # Creating batched tensors for Norch + norch_tensor1 = norch.Tensor([[1., 2], [3, -4], [5, 6], [7, 8]]).to(self.device) + norch_tensor2 = norch.Tensor([[2., 3, 1, 0, 4], [5, -1, 2, 3, 0]]).to(self.device) + + norch_result = norch_tensor1 @ norch_tensor2 + torch_result = utils.to_torch(norch_result).to(self.device) + + # Converting to PyTorch tensors for comparison + torch_tensor1 = torch.tensor([[1., 2], [3, -4], [5, 6], [7, 8]]).to(self.device) + torch_tensor2 = torch.tensor([[2., 3, 1, 0, 4], [5, -1, 2, 3, 0]]).to(self.device) + + torch_expected = torch.matmul(torch_tensor1, torch_tensor2) + + # Comparing results + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + def test_reshape_then_matmul(self): + """ + Test reshaping a tensor followed by matrix multiplication: (tensor.reshape(shape) @ other_tensor) + """ + norch_tensor = norch.Tensor([[1., 2], [3, -4], [5, 6], [7, 8]]).to(self.device) + new_shape = [2, 4] + norch_reshaped = norch_tensor.reshape(new_shape) + + norch_result = norch_reshaped @ norch_tensor + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor = torch.tensor([[1., 2], [3, -4], [5, 6], [7, 8]]).to(self.device) + torch_expected = torch_tensor.reshape(new_shape) @ torch_tensor + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + def test_batched_matmul(self): + """ + Test batched matrix multiplication: BxMxP = BxNxM @ BxMxP + """ + B = 3 # Batch size + + # Creating batched tensors for Norch + norch_tensor1 = norch.Tensor([[[1., 2], [3, -4], [5, 6], [7, 8]] for _ in range(B)]).to(self.device) + norch_tensor2 = norch.Tensor([[[2., 3, 1, 0, 4], [5, -1, 2, 3, 0]] for _ in range(B)]).to(self.device) + + norch_result = norch_tensor1 @ norch_tensor2 + torch_result = utils.to_torch(norch_result).to(self.device) + + # Converting to PyTorch tensors for comparison + torch_tensor1 = torch.tensor([[[1., 2], [3, -4], [5, 6], [7, 8]] for _ in range(B)]).to(self.device) + torch_tensor2 = torch.tensor([[[2., 3, 1, 0, 4], [5, -1, 2, 3, 0]] for _ in range(B)]).to(self.device) + + torch_expected = torch.matmul(torch_tensor1, torch_tensor2) + + # Comparing results + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + + def test_broadcasted_batched_matmul(self): + """ + Test broadcasted batched matrix multiplication: BxMxP = NxM @ BxMxP + """ + B = 3 # Batch size + + # Creating batched tensors for Norch + norch_tensor1 = norch.Tensor([[1., 2], [3, -4], [5, 6], [7, 8]]).to(self.device) + norch_tensor2 = norch.Tensor([[[2., 3, 1, 0, 4], [5, -1, 2, 3, 0]] for _ in range(B)]).to(self.device) + + norch_result = norch_tensor1 @ norch_tensor2 + torch_result = utils.to_torch(norch_result).to(self.device) + + # Converting to PyTorch tensors for comparison + torch_tensor1 = torch.tensor([[1., 2], [3, -4], [5, 6], [7, 8]]).to(self.device) + torch_tensor2 = torch.tensor([[[2., 3, 1, 0, 4], [5, -1, 2, 3, 0]] for _ in range(B)]).to(self.device) + + torch_expected = torch.matmul(torch_tensor1, torch_tensor2) + + # Comparing results + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + + + def test_transpose_then_matmul(self): + """ + Test transposing a tensor followed by matrix multiplication: (tensor.transpose(dim1, dim2) @ other_tensor) + """ + norch_tensor = norch.Tensor([[[1., 2], [3, 4]], [[5, 6], [7, 8]]]).to(self.device) + dim1, dim2 = 0, 2 + norch_result = norch_tensor.transpose(dim1, dim2) @ norch_tensor + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor = torch.tensor([[[1., 2], [3, 4]], [[5, 6], [7, 8]]]).to(self.device) + torch_expected = torch_tensor.transpose(dim1, dim2) @ torch_tensor + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + def test_add_div_matmul_then_reshape(self): + """ + Test a combination of operations: (tensor.sum() + other_tensor) / scalar @ another_tensor followed by reshape + """ + norch_tensor1 = norch.Tensor([[[1., 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + norch_tensor2 = norch.Tensor([[[1., 1], [1, 1]], [[1, 1], [1, 1]]]).to(self.device) + scalar = 2 + new_shape = [2, 4] + norch_result = ((norch_tensor1 + norch_tensor2) / scalar) @ norch_tensor1 + norch_result = norch_result.reshape(new_shape) + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor1 = torch.tensor([[[1., 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + torch_tensor2 = torch.tensor([[[1., 1], [1, 1]], [[1, 1], [1, 1]]]).to(self.device) + torch_expected = ((torch_tensor1 + torch_tensor2) / scalar) @ torch_tensor1 + torch_expected = torch_expected.reshape(new_shape) + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + def test_scalar_power_tensor(self): + """ + Test scalar power of a tensor: scalar ** tensor + """ + scalar = 3 + norch_tensor = norch.Tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + norch_result = scalar ** norch_tensor + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor = torch.tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + torch_expected = scalar ** torch_tensor + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + def test_tensor_power_scalar(self): + """ + Test tensor power of a scalar: tensor ** scalar + """ + scalar = 3 + norch_tensor = norch.Tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + norch_result = norch_tensor ** scalar + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor = torch.tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + torch_expected = torch_tensor ** scalar + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + def test_tensor_sin(self): + """ + Test sine function on tensor + """ + norch_tensor = norch.Tensor([[[0, 30], [45, 60]], [[90, 120], [135, 180]]]).to(self.device) + norch_result = norch_tensor.sin() + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor = torch.tensor([[[0, 30], [45, 60]], [[90, 120], [135, 180]]]).to(self.device) + torch_expected = torch.sin(torch_tensor) + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + def test_tensor_cos(self): + """ + Test cosine function on tensor + """ + norch_tensor = norch.Tensor([[[0, 30], [45, 60]], [[90, 120], [135, 180]]]).to(self.device) + norch_result = norch_tensor.cos() + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor = torch.tensor([[[0, 30], [45, 60]], [[90, 120], [135, 180]]]).to(self.device) + torch_expected = torch.cos(torch_tensor) + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + def test_equal(self): + """ + Test equal two tensors: tensor1.equal(tensor2) + """ + norch_tensor1 = norch.Tensor([[[1, 2], [3, -4]], [[5, 1], [7, 8]]]).to(self.device) + norch_tensor2 = norch.Tensor([[[1, 1], [1, 1]], [[1, 1], [1, 1]]]).to(self.device) + norch_result = norch_tensor1.equal(norch_tensor2) + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor1 = torch.tensor([[[1, 2], [3, -4]], [[5, 1], [7, 8]]]).to(self.device) + torch_tensor2 = torch.tensor([[[1, 1], [1, 1]], [[1, 1], [1, 1]]]).to(self.device) + torch_expected = (torch_tensor1 == torch_tensor2).float() + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + def test_broadcasted_equal(self): + """ + Test broadcasted equal two tensors: tensor1.equal(tensor2) + """ + norch_tensor1 = norch.Tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]]).to(self.device) + norch_tensor2 = norch.Tensor([[[10, 10]], [[5, 6]]]).to(self.device) + norch_result = norch_tensor1.equal(norch_tensor2) + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor1 = torch.tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]]).to(self.device) + torch_tensor2 = torch.tensor([[[10, 10]], [[5, 6]]]).to(self.device) + torch_expected = (torch_tensor1 == torch_tensor2).float() + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + norch_tensor1 = norch.Tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]]).to(self.device) + norch_tensor2 = norch.Tensor([[[10.0,], [-4.0,]],[[6.0,], [8.0,]]]).to(self.device) + norch_result = norch_tensor1.equal(norch_tensor2) + torch_result = utils.to_torch(norch_result).to(self.device) + + torch_tensor1 = torch.tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]]).to(self.device) + torch_tensor2 = torch.tensor([[[10.0,], [-4.0,]],[[6.0,], [8.0,]]]).to(self.device) + torch_expected = (torch_tensor1 == torch_tensor2).float() + + self.assertTrue(utils.compare_torch(torch_result, torch_expected)) + + + def test_zeros_like(self): + """ + Test creating a tensor of zeros with the same shape as another tensor. + """ + norch_tensor = norch.Tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + norch_zeros = norch_tensor.zeros_like() + torch_zeros_result = utils.to_torch(norch_zeros).to(self.device) + + torch_tensor_expected = torch.tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + torch_zeros_expected = torch.zeros_like(torch_tensor_expected) + + self.assertTrue(utils.compare_torch(torch_zeros_result, torch_zeros_expected)) + + def test_ones_like(self): + """ + Test creating a tensor of ones with the same shape as another tensor. + """ + norch_tensor = norch.Tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + norch_ones = norch_tensor.ones_like() + torch_ones_result = utils.to_torch(norch_ones).to(self.device) + + torch_tensor_expected = torch.tensor([[[1, 2], [3, -4]], [[5, 6], [7, 8]]]).to(self.device) + torch_ones_expected = torch.ones_like(torch_tensor_expected) + + self.assertTrue(utils.compare_torch(torch_ones_result, torch_ones_expected)) + + +if __name__ == '__main__': + unittest.main() diff --git a/setup.py b/setup.py index 8774eb8..d94cfc4 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ apt_get_dependencies = [ apt_nccl = [ 'libnccl2=2.21.5-1+cuda12.2', - 'libnccl-dev=2.21.5-1+cuda12.2 ' + 'libnccl-dev=2.21.5-1+cuda12.2' ] subprocess.check_call(['sudo', 'apt-get', 'update']) @@ -27,14 +27,16 @@ for package in apt_dependencies: subprocess.check_call(['sudo', 'apt', 'install', package]) for package in apt_get_dependencies: - subprocess.check_call(['sudo', 'apt-get', 'install', 'y', package]) + subprocess.check_call(['sudo', 'apt-get', 'install', '-y', package]) -subprocess.check_call(['wget', 'https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-keyring_1.0-1_all.deb ']) +subprocess.check_call(['wget', 'https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-keyring_1.0-1_all.deb']) subprocess.check_call(['sudo', 'dpkg', '-i', 'cuda-keyring_1.0-1_all.deb']) for package in apt_nccl: subprocess.check_call(['sudo', 'apt', 'install', package]) +subprocess.check_call(['rm', 'cuda-keyring_1.0-1_all.deb']) + class CustomInstall(install): def run(self): subprocess.call(['make', '-C', 'build'])