120 lines
4.1 KiB
Python
120 lines
4.1 KiB
Python
import random
|
|
import requests
|
|
import tarfile
|
|
import zipfile
|
|
import shutil
|
|
import os
|
|
|
|
def generate_random_list(shape):
|
|
"""
|
|
Generate a list with random numbers and shape 'shape'
|
|
"""
|
|
if len(shape) == 0:
|
|
return []
|
|
else:
|
|
inner_shape = shape[1:]
|
|
if len(inner_shape) == 0:
|
|
return [random.uniform(-1, 1) for _ in range(shape[0])]
|
|
else:
|
|
return [generate_random_list(inner_shape) for _ in range(shape[0])]
|
|
|
|
|
|
def download_from_url(url, save_path, chunk_size=128):
|
|
"""Download a file from an URL.
|
|
|
|
Original answer from https://stackoverflow.com/questions/9419162/download-returned-zip-file-from-url
|
|
|
|
Args:
|
|
url (str): path to the URL.
|
|
save_path (str): path to the saving directory.
|
|
chunk_size (int): download chunk.
|
|
|
|
Returns:
|
|
None
|
|
"""
|
|
response = requests.get(url, stream=True)
|
|
total = response.headers.get('content-length')
|
|
with open(save_path, 'wb') as f:
|
|
if total is None:
|
|
f.write(response.content)
|
|
else:
|
|
downloaded = 0
|
|
total = int(total)
|
|
for data in response.iter_content(chunk_size=max(int(total / 1000), 1024 * 1024)):
|
|
downloaded += len(data)
|
|
f.write(data)
|
|
progress_bar(downloaded, total, "Downloading...")
|
|
|
|
|
|
def extract_to_dir(filename, dirpath='.'):
|
|
# Does not create folder twice with the same name
|
|
name, ext = os.path.splitext(filename)
|
|
# if os.path.basename(name) == os.path.basename(dirpath):
|
|
# dirpath = '.'
|
|
# Extract
|
|
print(dirpath)
|
|
print("Extracting...", end="")
|
|
if tarfile.is_tarfile(filename):
|
|
tarfile.open(filename, 'r').extractall(dirpath)
|
|
elif zipfile.is_zipfile(filename):
|
|
zipfile.ZipFile(filename, 'r').extractall(dirpath)
|
|
elif ext == '.gz':
|
|
if not os.path.exists(dirpath):
|
|
os.mkdir(dirpath)
|
|
shutil.move(filename, os.path.join(dirpath, os.path.basename(filename)))
|
|
print(f" | NOTE: gzip files are not extracted, and moved to {dirpath}", end="")
|
|
# Return the path where the file was extracted
|
|
print(" | Done !")
|
|
return os.path.abspath(dirpath)
|
|
|
|
def progress_bar(current_index, max_index, prefix=None, suffix=None, start_time=None):
|
|
"""Display a progress bar and duration.
|
|
|
|
Args:
|
|
current_index (int): current state index (or epoch number).
|
|
max_index (int): maximal numbers of state.
|
|
prefix (str, optional): prefix of the progress bar. The default is None.
|
|
suffix (str, optional): suffix of the progress bar. The default is None.
|
|
start_time (float, optional): starting time of the progress bar. If not None, it will display the time
|
|
spent from the beginning to the current state. The default is None.
|
|
|
|
Returns:
|
|
None. Display the progress bar in the console.
|
|
"""
|
|
# Add a prefix to the progress bar
|
|
prefix = "" if prefix is None else str(prefix) + " "
|
|
|
|
# Get the percentage
|
|
percentage = current_index * 100 // max_index
|
|
loading = "[" + "=" * (percentage // 2) + " " * (50 - percentage // 2) + "]"
|
|
progress_display = "\r{0}{1:3d}% | {2}".format(prefix, percentage, loading)
|
|
|
|
# Add a suffix to the progress bar
|
|
progress_display += "" if suffix is None else " | " + str(suffix)
|
|
|
|
# Add a timer
|
|
if start_time is not None:
|
|
time_min, time_sec = get_time(start_time, time.time())
|
|
time_display = " | Time: {0}m {1}s".format(time_min, time_sec)
|
|
progress_display += time_display
|
|
|
|
# Print the progress bar
|
|
# TODO: return a string instead
|
|
print(progress_display, end="{}".format("" if current_index < max_index else " | Done !\n"))
|
|
|
|
def get_time(start_time, end_time):
|
|
"""Get ellapsed time in minutes and seconds.
|
|
|
|
Args:
|
|
start_time (float): strarting time
|
|
end_time (float): ending time
|
|
|
|
Returns:
|
|
elapsed_mins (float): elapsed time in minutes
|
|
elapsed_secs (float): elapsed time in seconds.
|
|
"""
|
|
elapsed_time = end_time - start_time
|
|
elapsed_mins = int(elapsed_time / 60)
|
|
elapsed_secs = int(elapsed_time - (elapsed_mins * 60))
|
|
|
|
return elapsed_mins, elapsed_secs
|