diff --git a/README.md b/README.md index 5266f7a..fad9f4a 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,7 @@ import random random.seed(1) BATCH_SIZE = 32 -device = "cpu" +device = "cuda" #cpu epochs = 10 transform = transforms.Sequential( diff --git a/examples/train.ipynb b/examples/train.ipynb index f1e51a5..c53ea86 100644 --- a/examples/train.ipynb +++ b/examples/train.ipynb @@ -99,7 +99,7 @@ "outputs": [], "source": [ "BATCH_SIZE = 32\n", - "device = \"cpu\"\n", + "device = \"cuda\" #cpu\n", "epochs = 10\n", "\n", "transform = transforms.Sequential(\n", diff --git a/norch/__init__.py b/norch/__init__.py index 25fe68e..7bc1d53 100644 --- a/norch/__init__.py +++ b/norch/__init__.py @@ -4,6 +4,6 @@ from .optim import * from .utils import * from .norchvision import * -__version__ = "0.0.3" +__version__ = "0.0.4" __author__ = 'Lucas de Lima Nogueira' __credits__ = 'Lucas de Lima Nogueira' \ No newline at end of file diff --git a/norch/__pycache__/tensor.cpython-38.pyc b/norch/__pycache__/tensor.cpython-38.pyc index bbd6f48..882aabd 100644 Binary files a/norch/__pycache__/tensor.cpython-38.pyc and b/norch/__pycache__/tensor.cpython-38.pyc differ diff --git a/norch/autograd/__pycache__/functions.cpython-38.pyc b/norch/autograd/__pycache__/functions.cpython-38.pyc index 154463f..11732bb 100644 Binary files a/norch/autograd/__pycache__/functions.cpython-38.pyc and b/norch/autograd/__pycache__/functions.cpython-38.pyc differ diff --git a/norch/autograd/functions.py b/norch/autograd/functions.py index 917bd94..477837d 100644 --- a/norch/autograd/functions.py +++ b/norch/autograd/functions.py @@ -136,7 +136,7 @@ class SumBackward: input_shape = self.input[0].shape.copy() if self.axis == -1: # If axis is None, sum reduces the tensor to a scalar. - grad_output = float(gradient.tensor.contents.data[0]) * self.input[0].ones_like() + grad_output = float(gradient[[0] * len(gradient.shape)]) * self.input[0].ones_like() else: if self.keepdim: @@ -214,9 +214,9 @@ class MaxBackward: max_value = self.input[0].max() mask = self.input[0].equal(max_value) - grad_output = float(gradient.tensor.contents.data[0]) * self.input[0].ones_like() + grad_output = float(gradient[[0] * len(gradient.shape)]) * self.input[0].ones_like() - grad_output = (grad_output * mask) / mask.sum().tensor.contents.data[0] + grad_output = (grad_output * mask) / mask.sum()[0] else: @@ -250,9 +250,9 @@ class MinBackward: min_value = self.input[0].min() mask = self.input[0].equal(min_value) - grad_output = float(gradient.tensor.contents.data[0]) * self.input[0].ones_like() + grad_output = float(gradient[[0] * len(gradient.shape)]) * self.input[0].ones_like() - grad_output = (grad_output * mask) / mask.sum().tensor.contents.data[0] + grad_output = (grad_output * mask) / mask.sum()[0] else: if self.keepdim: diff --git a/norch/nn/__pycache__/loss.cpython-38.pyc b/norch/nn/__pycache__/loss.cpython-38.pyc index 52d18ca..26a0e8b 100644 Binary files a/norch/nn/__pycache__/loss.cpython-38.pyc and b/norch/nn/__pycache__/loss.cpython-38.pyc differ diff --git a/norch/nn/__pycache__/module.cpython-38.pyc b/norch/nn/__pycache__/module.cpython-38.pyc index 7089e21..f03aec3 100644 Binary files a/norch/nn/__pycache__/module.cpython-38.pyc and b/norch/nn/__pycache__/module.cpython-38.pyc differ diff --git a/norch/nn/functional.py b/norch/nn/functional.py index 4c4d757..29fb68d 100644 --- a/norch/nn/functional.py +++ b/norch/nn/functional.py @@ -23,10 +23,10 @@ def softmax(x, dim=None): def one_hot_encode(x, num_classes): one_hot = [[0] * num_classes for _ in range(x.numel)] - + # Set the appropriate elements to 1 for i in range(x.numel): - target_idx = int(x.tensor.contents.data[i]) + target_idx = int(x[i]) one_hot[i][target_idx] = 1 return norch.Tensor(one_hot) \ No newline at end of file diff --git a/norch/nn/loss.py b/norch/nn/loss.py index 124da6f..6c0abea 100644 --- a/norch/nn/loss.py +++ b/norch/nn/loss.py @@ -44,7 +44,7 @@ class CrossEntropyLoss(Loss): if input.ndim == 1: if target.numel == 1: num_classes = input.shape[0] - target = norch.one_hot_encode(target, num_classes) + target = norch.one_hot_encode(target, num_classes).to(target.device) logits = norch.softmax(input, dim=0) target = target.reshape(logits.shape) @@ -67,7 +67,7 @@ class CrossEntropyLoss(Loss): # target -> Ground truth class indices: num_classes = input.shape[1] - target = norch.one_hot_encode(target, num_classes) + target = norch.one_hot_encode(target, num_classes).to(target.device) batch_size = input.shape[0] logits = norch.softmax(input, dim=1) diff --git a/norch/nn/module.py b/norch/nn/module.py index eaabd9a..0135bb6 100644 --- a/norch/nn/module.py +++ b/norch/nn/module.py @@ -51,8 +51,10 @@ class Module(ABC): parameter.zero_grad() def to(self, device): - for _, _, parameter in self.parameters(): - parameter.to(device) + for module, name, _ in self.parameters(): + parameter = getattr(module, name) + parameter = parameter.to(device) + setattr(module, name, parameter) return self diff --git a/norch/tensor.py b/norch/tensor.py index 8063cba..6861f14 100644 --- a/norch/tensor.py +++ b/norch/tensor.py @@ -205,7 +205,7 @@ class Tensor: if gradient is None: if self.shape == [1]: - gradient = Tensor([1]) + gradient = Tensor([1]).to(self.device) else: raise RuntimeError("Gradient argument must be specified for non-scalar tensors.") diff --git a/setup.py b/setup.py index 06c8016..36bff91 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ class CustomInstall(install): setuptools.setup( name = "norch", - version = "0.0.3", + version = "0.0.4", author = "Lucas de Lima", author_email = "nogueiralucasdelima@gmail.com", description = "A deep learning framework", diff --git a/test.py b/test.py deleted file mode 100644 index 94f5195..0000000 --- a/test.py +++ /dev/null @@ -1,7 +0,0 @@ -import norch -from norch.utils import utils_unittests as utils - -device = "cpu" - -norch_tensor = norch.Tensor([[[[1, 2], [3, -4]], [[5, 6], [7, 8]]], [[[1, 2], [3, -4]], [[5, 6], [7, 8]]]]).to(device) -norch_result = norch_tensor.sum(axis=0) diff --git a/tests/test_autograd.py b/tests/test_autograd.py index 82e6b61..3be772c 100644 --- a/tests/test_autograd.py +++ b/tests/test_autograd.py @@ -24,12 +24,8 @@ class TestTensorAutograd(unittest.TestCase): norch_tensor1_grad = utils.to_torch(norch_tensor1.grad).to(self.device) norch_tensor2_grad = utils.to_torch(norch_tensor2.grad).to(self.device) - torch_tensor1 = torch.tensor([[[1, 2.5], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True) - torch_tensor2 = torch.tensor([[[1, 1.], [1, 1.9]], [[1, 1], [1, 1]]], requires_grad=True) - - torch_tensor1.to(self.device) - torch_tensor2.to(self.device) - + torch_tensor1 = torch.tensor([[[1, 2.5], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True, device=self.device) + torch_tensor2 = torch.tensor([[[1, 1.], [1, 1.9]], [[1, 1], [1, 1]]], requires_grad=True, device=self.device) torch_result = (torch_tensor1 + torch_tensor2).sum() torch_result.backward() torch_tensor1_grad = torch_tensor1.grad @@ -50,11 +46,8 @@ class TestTensorAutograd(unittest.TestCase): norch_tensor1_grad = utils.to_torch(norch_tensor1.grad).to(self.device) norch_tensor2_grad = utils.to_torch(norch_tensor2.grad).to(self.device) - torch_tensor1 = torch.tensor([[[1, 2.5], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True) - torch_tensor2 = torch.tensor([[[1, 1.], [1, 1.9]], [[1, 1], [1, 1]]], requires_grad=True) - - torch_tensor1.to(self.device) - torch_tensor2.to(self.device) + torch_tensor1 = torch.tensor([[[1, 2.5], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True, device=self.device) + torch_tensor2 = torch.tensor([[[1, 1.], [1, 1.9]], [[1, 1], [1, 1]]], requires_grad=True, device=self.device) torch_result = (torch_tensor1 + torch_tensor2).sum(axis=0).sum(axis=0).sum() torch_result.backward() @@ -72,11 +65,8 @@ class TestTensorAutograd(unittest.TestCase): norch_tensor1_grad = utils.to_torch(norch_tensor1.grad).to(self.device) norch_tensor2_grad = utils.to_torch(norch_tensor2.grad).to(self.device) - torch_tensor1 = torch.tensor([[[1, 2.5], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True) - torch_tensor2 = torch.tensor([[[1, 1.], [1, 1.9]], [[1, 1], [1, 1]]], requires_grad=True) - - torch_tensor1.to(self.device) - torch_tensor2.to(self.device) + torch_tensor1 = torch.tensor([[[1, 2.5], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True, device=self.device) + torch_tensor2 = torch.tensor([[[1, 1.], [1, 1.9]], [[1, 1], [1, 1]]], requires_grad=True, device=self.device) torch_result = (torch_tensor1 + torch_tensor2).sum(axis=1).sum() torch_result.backward() @@ -97,9 +87,7 @@ class TestTensorAutograd(unittest.TestCase): norch_result.backward() norch_tensor_grad = utils.to_torch(norch_tensor.grad).to(self.device) - torch_tensor = torch.tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True) - - torch_tensor.to(self.device) + torch_tensor = torch.tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True, device=self.device) torch_result = torch_tensor.max() torch_result.backward() @@ -118,9 +106,7 @@ class TestTensorAutograd(unittest.TestCase): norch_result.backward() norch_tensor_grad = utils.to_torch(norch_tensor.grad).to(self.device) - torch_tensor = torch.tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True) - - torch_tensor.to(self.device) + torch_tensor = torch.tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True, device=self.device) torch_max_axis, _ = torch_tensor.max(axis=1) torch_result = torch_max_axis.sum() @@ -136,9 +122,7 @@ class TestTensorAutograd(unittest.TestCase): norch_result.backward() norch_tensor_grad = utils.to_torch(norch_tensor.grad).to(self.device) - torch_tensor = torch.tensor([[[10, 1], [-4, 0]], [[5., 50], [7, 8]]], requires_grad=True) - - torch_tensor.to(self.device) + torch_tensor = torch.tensor([[[10, 1], [-4, 0]], [[5., 50], [7, 8]]], requires_grad=True, device=self.device) torch_max_axis, _ = torch_tensor.max(axis=2) torch_result = torch_max_axis.sum() @@ -202,9 +186,7 @@ class TestTensorAutograd(unittest.TestCase): norch_result.backward() norch_tensor_grad = utils.to_torch(norch_tensor.grad).to(self.device) - torch_tensor = torch.tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True) - - torch_tensor.to(self.device) + torch_tensor = torch.tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True, device=self.device) torch_result = torch_tensor.min() torch_result.backward() @@ -223,9 +205,7 @@ class TestTensorAutograd(unittest.TestCase): norch_result.backward() norch_tensor_grad = utils.to_torch(norch_tensor.grad).to(self.device) - torch_tensor = torch.tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True) - - torch_tensor.to(self.device) + torch_tensor = torch.tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True, device=self.device) torch_min, _ = torch_tensor.min(axis=1) torch_result = torch_min.sum() @@ -241,9 +221,7 @@ class TestTensorAutograd(unittest.TestCase): norch_result.backward() norch_tensor_grad = utils.to_torch(norch_tensor.grad).to(self.device) - torch_tensor = torch.tensor([[[10, 1], [-4, 0]], [[5., 50], [7, 8]]], requires_grad=True) - - torch_tensor.to(self.device) + torch_tensor = torch.tensor([[[10, 1], [-4, 0]], [[5., 50], [7, 8]]], requires_grad=True, device=self.device) torch_min, _ = torch_tensor.min(axis=2) torch_result = torch_min.sum() @@ -264,12 +242,8 @@ class TestTensorAutograd(unittest.TestCase): norch_tensor1_grad = utils.to_torch(norch_tensor1.grad).to(self.device) norch_tensor2_grad = utils.to_torch(norch_tensor2.grad).to(self.device) - torch_tensor1 = torch.tensor([[[1., 2, 3], [4, 5, 6]]], requires_grad=True) # Shape (1, 2, 3) - torch_tensor2 = torch.tensor([1.5, -1, 0], requires_grad=True) # Shape (3) - - torch_tensor1.to(self.device) - torch_tensor2.to(self.device) - + torch_tensor1 = torch.tensor([[[1., 2, 3], [4, 5, 6]]], requires_grad=True, device=self.device) # Shape (1, 2, 3) + torch_tensor2 = torch.tensor([1.5, -1, 0], requires_grad=True, device=self.device) # Shape (3) torch_result = (torch_tensor1 + torch_tensor2).sum() torch_result.backward() torch_tensor1_grad = torch_tensor1.grad @@ -287,11 +261,8 @@ class TestTensorAutograd(unittest.TestCase): norch_tensor1_grad = utils.to_torch(norch_tensor1.grad).to(self.device) norch_tensor2_grad = utils.to_torch(norch_tensor2.grad).to(self.device) - torch_tensor1 = torch.tensor([[[1., 2, 3], [4, 5, 6]]], requires_grad=True) # Shape (1, 2, 3) - torch_tensor2 = torch.tensor([1.5, -1, 0], requires_grad=True) # Shape (3) - - torch_tensor1.to(self.device) - torch_tensor2.to(self.device) + torch_tensor1 = torch.tensor([[[1., 2, 3], [4, 5, 6]]], requires_grad=True, device=self.device) # Shape (1, 2, 3) + torch_tensor2 = torch.tensor([1.5, -1, 0], requires_grad=True, device=self.device) # Shape (3) torch_result = (torch_tensor2 + torch_tensor1).sum() torch_result.backward() @@ -312,12 +283,8 @@ class TestTensorAutograd(unittest.TestCase): norch_tensor1_grad_sub = utils.to_torch(norch_tensor1_sub.grad).to(self.device) norch_tensor2_grad_sub = utils.to_torch(norch_tensor2_sub.grad).to(self.device) - torch_tensor1_sub = torch.tensor([[[1, 2.5], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True) - torch_tensor2_sub = torch.tensor([[[1, 1.], [1, 1.9]], [[1, 1], [1, 1]]], requires_grad=True) - - torch_tensor1_sub.to(self.device) - torch_tensor2_sub.to(self.device) - + torch_tensor1_sub = torch.tensor([[[1, 2.5], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True, device=self.device) + torch_tensor2_sub = torch.tensor([[[1, 1.], [1, 1.9]], [[1, 1], [1, 1]]], requires_grad=True, device=self.device) torch_result_sub = (torch_tensor1_sub - torch_tensor2_sub).sum() torch_result_sub.backward() torch_tensor1_grad_sub = torch_tensor1_sub.grad @@ -337,12 +304,8 @@ class TestTensorAutograd(unittest.TestCase): norch_tensor1_grad = utils.to_torch(norch_tensor1.grad).to(self.device) norch_tensor2_grad = utils.to_torch(norch_tensor2.grad).to(self.device) - torch_tensor1 = torch.tensor([[[1., 2, 3], [4, 5, 6]]], requires_grad=True) # Shape (1, 2, 3) - torch_tensor2 = torch.tensor([1.5, -1, 0], requires_grad=True) # Shape (3) - - torch_tensor1.to(self.device) - torch_tensor2.to(self.device) - + torch_tensor1 = torch.tensor([[[1., 2, 3], [4, 5, 6]]], requires_grad=True, device=self.device) # Shape (1, 2, 3) + torch_tensor2 = torch.tensor([1.5, -1, 0], requires_grad=True, device=self.device) # Shape (3) torch_result = (torch_tensor1 - torch_tensor2).sum() torch_result.backward() torch_tensor1_grad = torch_tensor1.grad @@ -360,11 +323,8 @@ class TestTensorAutograd(unittest.TestCase): norch_tensor1_grad = utils.to_torch(norch_tensor1.grad).to(self.device) norch_tensor2_grad = utils.to_torch(norch_tensor2.grad).to(self.device) - torch_tensor1 = torch.tensor([[[1., 2, 3], [4, 5, 6]]], requires_grad=True) # Shape (1, 2, 3) - torch_tensor2 = torch.tensor([1.5, -1, 0], requires_grad=True) # Shape (3) - - torch_tensor1.to(self.device) - torch_tensor2.to(self.device) + torch_tensor1 = torch.tensor([[[1., 2, 3], [4, 5, 6]]], requires_grad=True, device=self.device) # Shape (1, 2, 3) + torch_tensor2 = torch.tensor([1.5, -1, 0], requires_grad=True, device=self.device) # Shape (3) torch_result = (torch_tensor2 - torch_tensor1).sum() torch_result.backward() @@ -386,12 +346,8 @@ class TestTensorAutograd(unittest.TestCase): norch_tensor1_grad_div = utils.to_torch(norch_tensor1_div.grad).to(self.device) norch_tensor2_grad_div = utils.to_torch(norch_tensor2_div.grad).to(self.device) - torch_tensor1_div = torch.tensor([[[2, 5.1], [6, -8]], [[10, 12], [14, 16]]], requires_grad=True) - torch_tensor2_div = torch.tensor([[[1, 1], [2, 2.2]], [[3, 3], [4, 4]]], requires_grad=True) - - torch_tensor1_div.to(self.device) - torch_tensor2_div.to(self.device) - + torch_tensor1_div = torch.tensor([[[2, 5.1], [6, -8]], [[10, 12], [14, 16]]], requires_grad=True, device=self.device) + torch_tensor2_div = torch.tensor([[[1, 1], [2, 2.2]], [[3, 3], [4, 4]]], requires_grad=True, device=self.device) torch_result_div = (torch_tensor1_div / torch_tensor2_div).sum() torch_result_div.backward() torch_tensor1_grad_div = torch_tensor1_div.grad @@ -411,10 +367,7 @@ class TestTensorAutograd(unittest.TestCase): norch_result_div_scalar.backward() norch_tensor_grad_div_scalar = utils.to_torch(norch_tensor_div_scalar.grad).to(self.device) - torch_tensor_div_scalar = torch.tensor([[[2, 4.7], [6, 8]], [[10, 12], [14, 16]]], requires_grad=True) - - torch_tensor_div_scalar.to(self.device) - + torch_tensor_div_scalar = torch.tensor([[[2, 4.7], [6, 8]], [[10, 12], [14, 16]]], requires_grad=True, device=self.device) torch_result_div_scalar = (torch_tensor_div_scalar / scalar).sum() torch_result_div_scalar.backward() torch_tensor_grad_div_scalar = torch_tensor_div_scalar.grad @@ -432,10 +385,7 @@ class TestTensorAutograd(unittest.TestCase): norch_result_scalar_div.backward() norch_tensor_grad_scalar_div = utils.to_torch(norch_tensor_scalar_div.grad).to(self.device) - torch_tensor_scalar_div = torch.tensor([[[1, 2.23], [3, 4]], [[5, 6], [7, 8]]], requires_grad=True) - - torch_tensor_scalar_div.to(self.device) - + torch_tensor_scalar_div = torch.tensor([[[1, 2.23], [3, 4]], [[5, 6], [7, 8]]], requires_grad=True, device=self.device) torch_result_scalar_div = (scalar / torch_tensor_scalar_div).sum() torch_result_scalar_div.backward() torch_tensor_grad_scalar_div = torch_tensor_scalar_div.grad @@ -453,10 +403,7 @@ class TestTensorAutograd(unittest.TestCase): norch_result_power_st.backward() norch_tensor_grad_power_st = utils.to_torch(norch_tensor_power_st.grad).to(self.device) - torch_tensor_power_st = torch.tensor([[[2, 3.21], [4, 2.1]], [[6, 7], [8, 9]]], requires_grad=True) - - torch_tensor_power_st.to(self.device) - + torch_tensor_power_st = torch.tensor([[[2, 3.21], [4, 2.1]], [[6, 7], [8, 9]]], requires_grad=True, device=self.device) torch_result_power_st = (scalar ** torch_tensor_power_st).sum() torch_result_power_st.backward() torch_tensor_grad_power_st = torch_tensor_power_st.grad @@ -473,10 +420,7 @@ class TestTensorAutograd(unittest.TestCase): norch_result_power_ts.backward() norch_tensor_grad_power_ts = utils.to_torch(norch_tensor_power_ts.grad).to(self.device) - torch_tensor_power_ts = torch.tensor([[[2, 3], [4, 2.1]], [[6, 7], [8, 9]]], requires_grad=True) - - torch_tensor_power_ts.to(self.device) - + torch_tensor_power_ts = torch.tensor([[[2, 3], [4, 2.1]], [[6, 7], [8, 9]]], requires_grad=True, device=self.device) torch_result_power_ts = (torch_tensor_power_ts ** scalar).sum() torch_result_power_ts.backward() torch_tensor_grad_power_ts = torch_tensor_power_ts.grad @@ -494,12 +438,8 @@ class TestTensorAutograd(unittest.TestCase): norch_tensor1_grad_matmul = utils.to_torch(norch_tensor1_matmul.grad).to(self.device) norch_tensor2_grad_matmul = utils.to_torch(norch_tensor2_matmul.grad).to(self.device) - torch_tensor1_matmul = torch.tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True) - torch_tensor2_matmul = torch.tensor([[[1.1, 3], [4, 5]], [[6, 7], [8, 9]]], requires_grad=True) - - torch_tensor1_matmul.to(self.device) - torch_tensor2_matmul.to(self.device) - + torch_tensor1_matmul = torch.tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True, device=self.device) + torch_tensor2_matmul = torch.tensor([[[1.1, 3], [4, 5]], [[6, 7], [8, 9]]], requires_grad=True, device=self.device) torch_result_matmul = (torch_tensor1_matmul @ torch_tensor2_matmul).sum() torch_result_matmul.backward() torch_tensor1_grad_matmul = torch_tensor1_matmul.grad @@ -526,12 +466,8 @@ class TestTensorAutograd(unittest.TestCase): norch_tensor2_grad_matmul = utils.to_torch(norch_tensor2_matmul.grad).to(self.device) # Repeat the same process with torch tensors - torch_tensor1_matmul = torch.tensor([[[1., 2], [3, -4], [5, 6], [7, 8]] for _ in range(B)], requires_grad=True) - torch_tensor2_matmul = torch.tensor([[[2., 3, 1, 0, 4], [5, -1, 2, 3, 0]] for _ in range(B)], requires_grad=True) - - torch_tensor1_matmul.to(self.device) - torch_tensor2_matmul.to(self.device) - + torch_tensor1_matmul = torch.tensor([[[1., 2], [3, -4], [5, 6], [7, 8]] for _ in range(B)], requires_grad=True, device=self.device) + torch_tensor2_matmul = torch.tensor([[[2., 3, 1, 0, 4], [5, -1, 2, 3, 0]] for _ in range(B)], requires_grad=True, device=self.device) torch_result_matmul = torch.matmul(torch_tensor1_matmul, torch_tensor2_matmul) torch_result_matmul_sum = torch_result_matmul.sum() torch_result_matmul_sum.backward() @@ -562,12 +498,8 @@ class TestTensorAutograd(unittest.TestCase): norch_tensor2_grad_matmul = utils.to_torch(norch_tensor2_matmul.grad).to(self.device) # Repeat the same process with torch tensors - torch_tensor1_matmul = torch.tensor([[1., 2], [3, -4], [5, 6], [7, 8]], requires_grad=True) - torch_tensor2_matmul = torch.tensor([[[2., 3, 1, 0, 4], [5, -1, 2, 3, 0]] for _ in range(B)], requires_grad=True) - - torch_tensor1_matmul.to(self.device) - torch_tensor2_matmul.to(self.device) - + torch_tensor1_matmul = torch.tensor([[1., 2], [3, -4], [5, 6], [7, 8]], requires_grad=True, device=self.device) + torch_tensor2_matmul = torch.tensor([[[2., 3, 1, 0, 4], [5, -1, 2, 3, 0]] for _ in range(B)], requires_grad=True, device=self.device) torch_result_matmul = torch.matmul(torch_tensor1_matmul, torch_tensor2_matmul) torch_result_matmul_sum = torch_result_matmul.sum() torch_result_matmul_sum.backward() @@ -591,10 +523,7 @@ class TestTensorAutograd(unittest.TestCase): norch_result_elemwise_mul_scalar.backward() norch_tensor_grad_elemwise_mul_scalar = utils.to_torch(norch_tensor_elemwise_mul_scalar.grad).to(self.device) - torch_tensor_elemwise_mul_scalar = torch.tensor([[[1.1, 2], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True) - - torch_tensor_elemwise_mul_scalar.to(self.device) - + torch_tensor_elemwise_mul_scalar = torch.tensor([[[1.1, 2], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True, device=self.device) torch_result_elemwise_mul_scalar = (scalar * torch_tensor_elemwise_mul_scalar).sum() torch_result_elemwise_mul_scalar.backward() torch_tensor_grad_elemwise_mul_scalar = torch_tensor_elemwise_mul_scalar.grad @@ -613,12 +542,8 @@ class TestTensorAutograd(unittest.TestCase): norch_tensor1_grad_elemwise_mul = utils.to_torch(norch_tensor1_elemwise_mul.grad).to(self.device) norch_tensor2_grad_elemwise_mul = utils.to_torch(norch_tensor2_elemwise_mul.grad).to(self.device) - torch_tensor1_elemwise_mul = torch.tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True).to(self.device) - torch_tensor2_elemwise_mul = torch.tensor([[[1.1, 3], [4, 5]], [[6, 7], [8, 9]]], requires_grad=True).to(self.device) - - torch_tensor1_elemwise_mul.to(self.device) - torch_tensor2_elemwise_mul.to(self.device) - + torch_tensor1_elemwise_mul = torch.tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True, device=self.device) + torch_tensor2_elemwise_mul = torch.tensor([[[1.1, 3], [4, 5]], [[6, 7], [8, 9]]], requires_grad=True, device=self.device) torch_result_elemwise_mul = (torch_tensor1_elemwise_mul * torch_tensor2_elemwise_mul).sum() torch_result_elemwise_mul.backward() torch_tensor1_grad_elemwise_mul = torch_tensor1_elemwise_mul.grad @@ -636,10 +561,7 @@ class TestTensorAutograd(unittest.TestCase): norch_result_sin_tensor.backward() torch_result_sin_tensor_grad = utils.to_torch(norch_sin_tensor.grad).to(self.device) - torch_sin_tensor = torch.tensor([[[2, 3.21], [4, 2.1]], [[6, 7], [8, 9]]], requires_grad=True) - - torch_sin_tensor.to(self.device) - + torch_sin_tensor = torch.tensor([[[2, 3.21], [4, 2.1]], [[6, 7], [8, 9]]], requires_grad=True, device=self.device) torch_expected_sin_tensor = (torch.sin(torch_sin_tensor)).sum() torch_expected_sin_tensor.backward() torch_expected_sin_tensor_grad = torch_sin_tensor.grad @@ -655,10 +577,7 @@ class TestTensorAutograd(unittest.TestCase): norch_result_cos_tensor.backward() torch_result_cos_tensor_grad = utils.to_torch(norch_cos_tensor.grad).to(self.device) - torch_cos_tensor = torch.tensor([[[2, 3.21], [4, 2.1]], [[6, 7], [8, 9]]], requires_grad=True) - - torch_cos_tensor.to(self.device) - + torch_cos_tensor = torch.tensor([[[2, 3.21], [4, 2.1]], [[6, 7], [8, 9]]], requires_grad=True, device=self.device) torch_expected_cos_tensor = (torch.sin(torch_cos_tensor)).sum() torch_expected_cos_tensor.backward() torch_expected_cos_tensor_grad = torch_cos_tensor.grad @@ -676,10 +595,7 @@ class TestTensorAutograd(unittest.TestCase): norch_result.backward() norch_tensor_grad = utils.to_torch(norch_tensor.grad).to(self.device) - torch_tensor = torch.tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True) - - torch_tensor.to(self.device) - + torch_tensor = torch.tensor([[[10, 10], [-4, -4]], [[5., 6], [7, 8]]], requires_grad=True, device=self.device) torch_sigmoid = torch.sigmoid(torch_tensor) torch_result = torch_sigmoid.sum() @@ -701,12 +617,8 @@ class TestTensorAutograd(unittest.TestCase): loss_norch.backward() # Backpropagate the loss grad_norch = predictions_norch.grad - predictions_torch = torch.tensor([1.1, 2, 3, 4], requires_grad=True) - labels_torch = torch.tensor([4, 3, 2.1, 1]) - - predictions_torch.to(self.device) - labels_torch.to(self.device) - + predictions_torch = torch.tensor([1.1, 2, 3, 4], requires_grad=True, device=self.device) + labels_torch = torch.tensor([4, 3, 2.1, 1], device=self.device) loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch) loss_torch_expected.backward() # Backpropagate the loss grad_torch_expected = predictions_torch.grad @@ -731,12 +643,8 @@ class TestTensorAutograd(unittest.TestCase): loss_norch.backward() # Backpropagate the loss grad_norch = predictions_norch.grad - predictions_torch = torch.tensor([2.0, 1.0, 0.1], requires_grad=True) - labels_torch = torch.tensor(0) - - predictions_torch.to(self.device) - labels_torch.to(self.device) - + predictions_torch = torch.tensor([2.0, 1.0, 0.1], requires_grad=True, device=self.device) + labels_torch = torch.tensor(0, device=self.device) loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch) loss_torch_expected.backward() # Backpropagate the loss grad_torch_expected = predictions_torch.grad @@ -753,12 +661,8 @@ class TestTensorAutograd(unittest.TestCase): loss_norch.backward() # Backpropagate the loss grad_norch = predictions_norch.grad - predictions_torch = torch.tensor([[0.5, 1.5, 2.5], [1.0, 2.0, 3.0]], requires_grad=True) - labels_torch = torch.tensor([2, 1]) - - predictions_torch.to(self.device) - labels_torch.to(self.device) - + predictions_torch = torch.tensor([[0.5, 1.5, 2.5], [1.0, 2.0, 3.0]], requires_grad=True, device=self.device) + labels_torch = torch.tensor([2, 1], device=self.device) loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch) loss_torch_expected.backward() # Backpropagate the loss grad_torch_expected = predictions_torch.grad @@ -775,12 +679,8 @@ class TestTensorAutograd(unittest.TestCase): loss_norch.backward() # Backpropagate the loss grad_norch = predictions_norch.grad - predictions_torch = torch.tensor([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], requires_grad=True) - labels_torch = torch.tensor([1, 2]) - - predictions_torch.to(self.device) - labels_torch.to(self.device) - + predictions_torch = torch.tensor([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], requires_grad=True, device=self.device) + labels_torch = torch.tensor([1, 2], device=self.device) loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch) loss_torch_expected.backward() # Backpropagate the loss grad_torch_expected = predictions_torch.grad @@ -797,12 +697,8 @@ class TestTensorAutograd(unittest.TestCase): loss_norch.backward() # Backpropagate the loss grad_norch = predictions_norch.grad - predictions_torch = torch.tensor([[0.5, 0.2, 0.1], [0.1, 0.5, 0.7]], requires_grad=True) - labels_torch = torch.tensor([[1., 0, 0], [0, 1, 0]]) - - predictions_torch.to(self.device) - labels_torch.to(self.device) - + predictions_torch = torch.tensor([[0.5, 0.2, 0.1], [0.1, 0.5, 0.7]], requires_grad=True, device=self.device) + labels_torch = torch.tensor([[1., 0, 0], [0, 1, 0]], device=self.device) loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch) loss_torch_expected.backward() # Backpropagate the loss grad_torch_expected = predictions_torch.grad @@ -861,10 +757,7 @@ class TestTensorAutograd(unittest.TestCase): norch_result_reshape.backward() norch_tensor_grad_reshape = utils.to_torch(norch_tensor_reshape.grad).to(self.device) - torch_tensor_reshape = torch.tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True) - - torch_tensor_reshape.to(self.device) - + torch_tensor_reshape = torch.tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True, device=self.device) torch_result_reshape = torch_tensor_reshape.reshape(new_shape).sum() torch_result_reshape.backward() torch_tensor_grad_reshape = torch_tensor_reshape.grad @@ -882,10 +775,7 @@ class TestTensorAutograd(unittest.TestCase): norch_result_transpose.backward() norch_tensor_grad_transpose = utils.to_torch(norch_tensor_transpose.grad).to(self.device) - torch_tensor_transpose = torch.tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True) - - torch_tensor_transpose.to(self.device) - + torch_tensor_transpose = torch.tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True, device=self.device) torch_result_transpose = torch_tensor_transpose.transpose(axis1, axis2).sum() torch_result_transpose.backward() torch_tensor_grad_transpose = torch_tensor_transpose.grad @@ -902,10 +792,7 @@ class TestTensorAutograd(unittest.TestCase): norch_result_T.backward() norch_tensor_grad_T = utils.to_torch(norch_tensor_T.grad).to(self.device) - torch_tensor_T = torch.tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True) - - torch_tensor_T.to(self.device) - + torch_tensor_T = torch.tensor([[[1, 2.1], [3, -4]], [[5, 6], [7, 8]]], requires_grad=True, device=self.device) torch_result_T = torch_tensor_T.mT.sum() torch_result_T.backward() torch_tensor_grad_T = torch_tensor_T.grad @@ -926,11 +813,8 @@ class TestTensorAutograd(unittest.TestCase): norch_tensor_grad_reshape_matmul1 = utils.to_torch(norch_tensor1.grad).to(self.device) norch_tensor_grad_reshape_matmul2 = utils.to_torch(norch_tensor2.grad).to(self.device) - torch_tensor1 = torch.tensor([[1, 2.1], [3, -4], [5, 6], [7, 8]], dtype=torch.float32, requires_grad=True) - torch_tensor2 = torch.tensor([[1, 5.1], [0.1, -4], [0, 6], [7, 8]], dtype=torch.float32, requires_grad=True) - - torch_tensor1.to(self.device) - torch_tensor2.to(self.device) + torch_tensor1 = torch.tensor([[1, 2.1], [3, -4], [5, 6], [7, 8]], dtype=torch.float32, requires_grad=True, device=self.device) + torch_tensor2 = torch.tensor([[1, 5.1], [0.1, -4], [0, 6], [7, 8]], dtype=torch.float32, requires_grad=True, device=self.device) torch_result_reshape_matmul = (torch_tensor1.reshape(new_shape) @ torch_tensor2).sum() torch_result_reshape_matmul.backward() @@ -951,10 +835,7 @@ class TestTensorAutograd(unittest.TestCase): norch_result_unsqueeze_0.backward() norch_tensor_grad_unsqueeze_0 = utils.to_torch(norch_tensor_unsqueeze.grad).to(self.device) - torch_tensor_unsqueeze = torch.tensor([[1., 2], [3, 4]], requires_grad=True) - - torch_tensor_unsqueeze.to(self.device) - + torch_tensor_unsqueeze = torch.tensor([[1., 2], [3, 4]], requires_grad=True, device=self.device) torch_result_unsqueeze_0 = torch_tensor_unsqueeze.unsqueeze(0).sum() torch_result_unsqueeze_0.backward() torch_tensor_grad_unsqueeze_0 = torch_tensor_unsqueeze.grad @@ -967,10 +848,7 @@ class TestTensorAutograd(unittest.TestCase): norch_result_unsqueeze_1.backward() norch_tensor_grad_unsqueeze_1 = utils.to_torch(norch_tensor_unsqueeze.grad).to(self.device) - torch_tensor_unsqueeze = torch.tensor([[1., 2.], [3, 4]], requires_grad=True) - - torch_tensor_unsqueeze.to(self.device) - + torch_tensor_unsqueeze = torch.tensor([[1., 2.], [3, 4]], requires_grad=True, device=self.device) torch_result_unsqueeze_1 = torch_tensor_unsqueeze.unsqueeze(1).sum() torch_result_unsqueeze_1.backward() torch_tensor_grad_unsqueeze_1 = torch_tensor_unsqueeze.grad @@ -983,10 +861,7 @@ class TestTensorAutograd(unittest.TestCase): norch_result_unsqueeze_2.backward() norch_tensor_grad_unsqueeze_2 = utils.to_torch(norch_tensor_unsqueeze.grad).to(self.device) - torch_tensor_unsqueeze = torch.tensor([[1., 2], [3, 4]], requires_grad=True) - - torch_tensor_unsqueeze.to(self.device) - + torch_tensor_unsqueeze = torch.tensor([[1., 2], [3, 4]], requires_grad=True, device=self.device) torch_result_unsqueeze_2 = torch_tensor_unsqueeze.unsqueeze(2).sum() torch_result_unsqueeze_2.backward() torch_tensor_grad_unsqueeze_2 = torch_tensor_unsqueeze.grad @@ -1006,11 +881,8 @@ class TestTensorAutograd(unittest.TestCase): norch_tensor_grad_unsqueeze_matmul1 = utils.to_torch(norch_tensor1.grad).to(self.device) norch_tensor_grad_unsqueeze_matmul2 = utils.to_torch(norch_tensor2.grad).to(self.device) - torch_tensor1 = torch.tensor([[1, 2], [3, 4]], dtype=torch.float32, requires_grad=True) - torch_tensor2 = torch.tensor([[1, 2], [3, 4]], dtype=torch.float32, requires_grad=True) - - torch_tensor1.to(self.device) - torch_tensor2.to(self.device) + torch_tensor1 = torch.tensor([[1, 2], [3, 4]], dtype=torch.float32, requires_grad=True, device=self.device) + torch_tensor2 = torch.tensor([[1, 2], [3, 4]], dtype=torch.float32, requires_grad=True, device=self.device) torch_result_unsqueeze_matmul = (torch_tensor1 @ torch_tensor2.unsqueeze(0)).sum() torch_result_unsqueeze_matmul.backward() @@ -1030,10 +902,7 @@ class TestTensorAutograd(unittest.TestCase): norch_result_squeeze_0.backward() norch_tensor_grad_squeeze_0 = utils.to_torch(norch_tensor_squeeze.grad).to(self.device) - torch_tensor_squeeze = torch.tensor([[[1., 2], [3, 4]]], requires_grad=True) - - torch_tensor_squeeze.to(self.device) - + torch_tensor_squeeze = torch.tensor([[[1., 2], [3, 4]]], requires_grad=True, device=self.device) torch_result_squeeze_0 = torch_tensor_squeeze.squeeze(0).sum() torch_result_squeeze_0.backward() torch_tensor_grad_squeeze_0 = torch_tensor_squeeze.grad @@ -1050,14 +919,11 @@ class TestTensorAutograd(unittest.TestCase): # Squeeze at dim=0 then matmul norch_result_squeeze_matmul = (norch_tensor1.squeeze(0) @ norch_tensor2).sum() norch_result_squeeze_matmul.backward() - norch_tensor_grad_squeeze_matmul1 = utils.to_torch(norch_tensor1.grad).to(self.device) + norch_tensor_grad_squeeze_matmul1 = utils.to_torch(norch_tensor1.grad,).to(self.device) norch_tensor_grad_squeeze_matmul2 = utils.to_torch(norch_tensor2.grad).to(self.device) - torch_tensor1 = torch.tensor([[[1., 2], [3, 4]]], dtype=torch.float32, requires_grad=True) - torch_tensor2 = torch.tensor([[[1., 2], [3, 4]]], dtype=torch.float32, requires_grad=True) - - torch_tensor1.to(self.device) - torch_tensor2.to(self.device) + torch_tensor1 = torch.tensor([[[1., 2], [3, 4]]], dtype=torch.float32, requires_grad=True, device=self.device) + torch_tensor2 = torch.tensor([[[1., 2], [3, 4]]], dtype=torch.float32, requires_grad=True, device=self.device) torch_result_squeeze_matmul = (torch_tensor1.squeeze(0) @ torch_tensor2).sum() torch_result_squeeze_matmul.backward() @@ -1080,11 +946,8 @@ class TestTensorAutograd(unittest.TestCase): norch_tensor_grad_T_matmul1 = utils.to_torch(norch_tensor1.grad).to(self.device) norch_tensor_grad_T_matmult2 = utils.to_torch(norch_tensor2.grad).to(self.device) - torch_tensor1 = torch.tensor([[1, 2.1], [3, -4], [5, 6], [7, 8]], dtype=torch.float32, requires_grad=True) - torch_tensor2 = torch.tensor([[1, 5.1], [0.1, -4], [0, 6], [7, 8]], dtype=torch.float32, requires_grad=True) - - torch_tensor1.to(self.device) - torch_tensor2.to(self.device) + torch_tensor1 = torch.tensor([[1, 2.1], [3, -4], [5, 6], [7, 8]], dtype=torch.float32, requires_grad=True, device=self.device) + torch_tensor2 = torch.tensor([[1, 5.1], [0.1, -4], [0, 6], [7, 8]], dtype=torch.float32, requires_grad=True, device=self.device) torch_result_T_matmul = (torch_tensor1.T @ torch_tensor2).sum() torch_result_T_matmul.backward() @@ -1099,7 +962,7 @@ class TestTensorAutograd(unittest.TestCase): The code has a problem on the following operation tensor1.reshape(..) @ tensor1 print(tensor1.grad) - (also transpose and .T) + (also transpsoe and .T) """ pass @@ -1115,12 +978,9 @@ class TestTensorAutograd(unittest.TestCase): norch_tensor_grad_transpose_matmul1 = utils.to_torch(norch_tensor1.grad).to(self.device) norch_tensor_grad_transpose_matmult2 = utils.to_torch(norch_tensor2.grad).to(self.device) - torch_tensor1 = torch.tensor([[1, 2.1], [3, -4], [5, 6], [7, 8]], dtype=torch.float32, requires_grad=True) - torch_tensor2 = torch.tensor([[1, 5.1], [0.1, -4], [0, 6], [7, 8]], dtype=torch.float32, requires_grad=True) - - torch_tensor1.to(self.device) - torch_tensor2.to(self.device) - + torch_tensor1 = torch.tensor([[1, 2.1], [3, -4], [5, 6], [7, 8]], dtype=torch.float32, requires_grad=True, device=self.device) + torch_tensor2 = torch.tensor([[1, 5.1], [0.1, -4], [0, 6], [7, 8]], dtype=torch.float32, requires_grad=True, device=self.device) + torch_result_transpose_matmul = (torch_tensor1.T @ torch_tensor2).sum() torch_result_transpose_matmul.backward() torch_tensor_grad_transpose_matmul1 = torch_tensor1.grad diff --git a/tests/test_nn.py b/tests/test_nn.py index 4896506..532f5b6 100644 --- a/tests/test_nn.py +++ b/tests/test_nn.py @@ -26,11 +26,8 @@ class TestNNModuleLoss(unittest.TestCase): loss_norch = loss_fn_norch.forward(predictions_norch, labels_norch) loss_torch_result = utils.to_torch(loss_norch).to(self.device) - predictions_torch = torch.tensor([[1.1, 2, 3, 4], [1.1, 2, 3, 4]]) - labels_torch = torch.tensor([[1.1, 2, 3, 4], [1.1, 2, 3, 3]]) - - predictions_torch.to(self.device) - labels_torch.to(self.device) + predictions_torch = torch.tensor([[1.1, 2, 3, 4], [1.1, 2, 3, 4]], device=self.device) + labels_torch = torch.tensor([[1.1, 2, 3, 4], [1.1, 2, 3, 3]], device=self.device) loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch) @@ -42,11 +39,8 @@ class TestNNModuleLoss(unittest.TestCase): loss_norch = loss_fn_norch.forward(predictions_norch, labels_norch) loss_torch_result = utils.to_torch(loss_norch).to(self.device) - predictions_torch = torch.tensor([1.1, 2, 3, 4]) - labels_torch = torch.tensor([4, 3, 2.1, 1]) - - predictions_torch.to(self.device) - labels_torch.to(self.device) + predictions_torch = torch.tensor([1.1, 2, 3, 4], device=self.device) + labels_torch = torch.tensor([4, 3, 2.1, 1], device=self.device) loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch) @@ -66,11 +60,8 @@ class TestNNModuleLoss(unittest.TestCase): loss_torch_result = utils.to_torch(loss_norch).to(self.device) - predictions_torch = torch.tensor([2.0, 1.0, 0.1]) - labels_torch = torch.tensor(0) - - predictions_torch.to(self.device) - labels_torch.to(self.device) + predictions_torch = torch.tensor([2.0, 1.0, 0.1], device=self.device) + labels_torch = torch.tensor(0, device=self.device) loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch) @@ -83,11 +74,8 @@ class TestNNModuleLoss(unittest.TestCase): loss_torch_result = utils.to_torch(loss_norch).to(self.device) - predictions_torch = torch.tensor([[0.5, 1.5, 2.5], [1.0, 2.0, 3.0]]) - labels_torch = torch.tensor([2, 1]) - - predictions_torch.to(self.device) - labels_torch.to(self.device) + predictions_torch = torch.tensor([[0.5, 1.5, 2.5], [1.0, 2.0, 3.0]], device=self.device) + labels_torch = torch.tensor([2, 1], device=self.device) loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch) @@ -99,11 +87,8 @@ class TestNNModuleLoss(unittest.TestCase): loss_norch = loss_fn_norch.forward(predictions_norch, labels_norch) loss_torch_result = utils.to_torch(loss_norch).to(self.device) - predictions_torch = torch.tensor([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]) - labels_torch = torch.tensor([1, 2]) - - predictions_torch.to(self.device) - labels_torch.to(self.device) + predictions_torch = torch.tensor([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], device=self.device) + labels_torch = torch.tensor([1, 2], device=self.device) loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch) @@ -131,11 +116,8 @@ class TestNNModuleLoss(unittest.TestCase): loss_norch = loss_fn_norch.forward(predictions_norch, labels_norch) loss_torch_result = utils.to_torch(loss_norch).to(self.device) - predictions_torch = torch.tensor([[0.5, 0.2, 0.1], [0.1, 0.5, 0.7]]) - labels_torch = torch.tensor([[1., 0, 0], [0, 1, 0]]) - - predictions_torch.to(self.device) - labels_torch.to(self.device) + predictions_torch = torch.tensor([[0.5, 0.2, 0.1], [0.1, 0.5, 0.7]], device=self.device) + labels_torch = torch.tensor([[1., 0, 0], [0, 1, 0]], device=self.device) loss_torch_expected = loss_fn_torch(predictions_torch, labels_torch) @@ -161,9 +143,7 @@ class TestNNModuleActivationFn(unittest.TestCase): sigmoid_norch = sigmoid_fn_norch.forward(x) sigmoid_torch_result = utils.to_torch(sigmoid_norch).to(self.device) - x = torch.tensor([[1, 2, 3]]) - - x.to(self.device) + x = torch.tensor([[1, 2, 3]], device=self.device) sigmoid_torch_expected = sigmoid_fn_torch.forward(x) @@ -174,9 +154,7 @@ class TestNNModuleActivationFn(unittest.TestCase): sigmoid_norch = sigmoid_fn_norch.forward(x) sigmoid_torch_result = utils.to_torch(sigmoid_norch).to(self.device) - x = torch.tensor([-1, 2, -3]) - - x.to(self.device) + x = torch.tensor([-1, 2, -3], device=self.device) sigmoid_torch_expected = sigmoid_fn_torch.forward(x) @@ -187,9 +165,7 @@ class TestNNModuleActivationFn(unittest.TestCase): sigmoid_norch = sigmoid_fn_norch.forward(x) sigmoid_torch_result = utils.to_torch(sigmoid_norch).to(self.device) - x = torch.tensor([0, 0, 0]) - - x.to(self.device) + x = torch.tensor([0, 0, 0], device=self.device) sigmoid_torch_expected = sigmoid_fn_torch.forward(x) @@ -205,9 +181,9 @@ class TestNNModuleActivationFn(unittest.TestCase): # Define the input tensors for different test cases test_cases = [ - (norch.Tensor([[[1., 2, 3], [4, 5, 6]]]), torch.tensor([[[1., 2, 3], [4, 5, 6]]])), - (norch.Tensor([[[1., -1, 0], [2, -2, 0]]]), torch.tensor([[[1., -1, 0], [2, -2, 0]]])), - (norch.Tensor([[[0., 0, 0], [0, 0, 0]]]), torch.tensor([[[0., 0, 0], [0, 0, 0]]])) + (norch.Tensor([[[1., 2, 3], [4, 5, 6]]]).to(self.device), torch.tensor([[[1., 2, 3], [4, 5, 6]]], device=self.device)), + (norch.Tensor([[[1., -1, 0], [2, -2, 0]]]).to(self.device), torch.tensor([[[1., -1, 0], [2, -2, 0]]], device=self.device)), + (norch.Tensor([[[0., 0, 0], [0, 0, 0]]]).to(self.device), torch.tensor([[[0., 0, 0], [0, 0, 0]]], device=self.device)) ] for dim in axes: @@ -215,12 +191,7 @@ class TestNNModuleActivationFn(unittest.TestCase): softmax_fn_torch = torch.nn.Softmax(dim=dim) for norch_input, torch_input in test_cases: - # Move tensors to the correct device - norch_input = norch_input.to(self.device) - torch_input = torch_input - torch_input.to(self.device) - # Forward pass using norch softmax_norch = softmax_fn_norch.forward(norch_input) softmax_torch_result = utils.to_torch(softmax_norch).to(self.device)