74 lines
2.5 KiB
Python
74 lines
2.5 KiB
Python
from abc import ABC, abstractmethod
|
|
import os
|
|
from norch.utils.functions import extract_to_dir, download_from_url
|
|
from .example import Example
|
|
import norch
|
|
|
|
|
|
class Dataset(ABC):
|
|
r"""
|
|
Abstract Dataset class. All dataset for machine learning purposes can inherits from this architecture,
|
|
for convenience.
|
|
"""
|
|
urls = []
|
|
name = ''
|
|
dirname = ''
|
|
|
|
def __init__(self, examples, fields):
|
|
self.examples = examples
|
|
self.fields = fields
|
|
|
|
@classmethod
|
|
def splits(cls, train=None, test=None, valid=None, root='.'):
|
|
raise NotImplementedError
|
|
|
|
@classmethod
|
|
def download(cls, root):
|
|
r"""Download and unzip a web archive (.zip, .gz, or .tgz).
|
|
|
|
Args:
|
|
root (str): Folder to download data to.
|
|
|
|
Returns:
|
|
string: Path to extracted dataset.
|
|
"""
|
|
path_dirname = os.path.join(root, cls.dirname)
|
|
path_name = os.path.join(path_dirname, cls.name)
|
|
if not os.path.isdir(path_dirname):
|
|
for url in cls.urls:
|
|
filename = os.path.basename(url)
|
|
zpath = os.path.join(path_dirname, filename)
|
|
if not os.path.isfile(zpath):
|
|
if not os.path.exists(os.path.dirname(zpath)):
|
|
os.makedirs(os.path.dirname(zpath))
|
|
print(f'Download {filename} from {url} to {zpath}')
|
|
download_from_url(url, zpath)
|
|
extract_to_dir(zpath, path_name)
|
|
|
|
return path_name
|
|
|
|
def __repr__(self):
|
|
name = self.__class__.__name__
|
|
string = f"Dataset {name}("
|
|
tab = " "
|
|
for (key, value) in self.__dict__.items():
|
|
if key[0] != "_":
|
|
if isinstance(value, Example):
|
|
fields = self.fields
|
|
for (name, field) in fields:
|
|
string += f"\n{tab}({name}): {field.__class__.__name__}" \
|
|
f"(transform={True if field.transform is not None else None}, dtype={field.dtype})"
|
|
elif isinstance(value, norch.Tensor):
|
|
string += f"\n{tab}({key}): {value.__class__.__name__}(shape={value.shape}, dtype={value.dtype})"
|
|
else:
|
|
string += f"\n{tab}({key}): {value.__class__.__name__}"
|
|
return f'{string}\n)'
|
|
|
|
def __getitem__(self, item):
|
|
return self.examples[item]
|
|
|
|
def __setitem__(self, key, value):
|
|
self.examples[key] = value
|
|
|
|
def __len__(self):
|
|
return len(self.examples)
|