Remove _utils.py (download function moved to data repository)

This commit is contained in:
Paul Romano 2020-03-19 15:10:13 -05:00
parent 2160e6b156
commit 07cfb0cfcc
2 changed files with 0 additions and 96 deletions

View file

@ -1,70 +0,0 @@
import hashlib
import os.path
from pathlib import Path
from urllib.parse import urlparse
from urllib.request import urlopen, Request
_BLOCK_SIZE = 16384
def download(url, checksum=None, as_browser=False, **kwargs):
"""Download file from a URL
Parameters
----------
url : str
URL from which to download
checksum : str or None
MD5 checksum to check against
as_browser : bool
Change User-Agent header to appear as a browser
kwargs : dict
Keyword arguments passed to :func:urllib.request.urlopen
Returns
-------
basename : str
Name of file written locally
"""
if as_browser:
page = Request(url, headers={'User-Agent': 'Mozilla/5.0'})
else:
page = url
with urlopen(page, **kwargs) as response:
# Get file size from header
file_size = response.length
# Check if file already downloaded
basename = Path(urlparse(url).path).name
if os.path.exists(basename):
if os.path.getsize(basename) == file_size:
print('Skipping {}, already downloaded'.format(basename))
return basename
# Copy file to disk in chunks
print('Downloading {}... '.format(basename), end='')
downloaded = 0
with open(basename, 'wb') as fh:
while True:
chunk = response.read(_BLOCK_SIZE)
if not chunk:
break
fh.write(chunk)
downloaded += len(chunk)
status = '{:10} [{:3.2f}%]'.format(
downloaded, downloaded * 100. / file_size)
print(status + '\b'*len(status), end='', flush=True)
print('')
if checksum is not None:
downloadsum = hashlib.md5(open(basename, 'rb').read()).hexdigest()
if downloadsum != checksum:
raise IOError("MD5 checksum for {} does not match. If this is "
"your first time receiving this message, please "
"re-run the script. Otherwise, please contact "
"OpenMC developers by emailing "
"openmc-users@googlegroups.com.".format(basename))
return basename

View file

@ -1,26 +0,0 @@
import os
import filecmp
from openmc import _utils
import pytest
@pytest.fixture()
def download_photos(run_in_tmpdir):
"""use _utils download() function to download the same picture three times,
twice to get unique names, & a third time to use the already downloaded
block of code"""
_utils.download("https://i.ibb.co/HhKFc8x/small.jpg")
_utils.download("https://tinyurl.com/y4t38ugb")
_utils.download("https://tinyurl.com/y4t38ugb", as_browser=True)
def test_checksum_error(run_in_tmpdir):
"""use download() in such a way that will test the checksum error line"""
phrase = "MD5 checksum for y4t38ugb"
with pytest.raises(OSError, match=phrase):
_utils.download("https://tinyurl.com/y4t38ugb", as_browser=True,
checksum="not none")
def test_photos(download_photos):
assert filecmp.cmp("small.jpg", "y4t38ugb")