Browse Source

Only download changed files

main
fluffy 7 months ago
parent
commit
8e6c9a37e0
1 changed files with 50 additions and 20 deletions
  1. +50
    -20
      zenius.py

+ 50
- 20
zenius.py View File

@@ -7,6 +7,7 @@ import os.path
import zipfile
import os
import argparse
import pickle

def parse_args(*args):
parser = argparse.ArgumentParser(description="Mirror simfiles from ZIV")
@@ -25,6 +26,42 @@ def parse_args(*args):

return parser.parse_args(*args)

def retrieve(url, filename, save_headers=None, **kwargs):
print(f'Downloading {url} -> {filename}')
try:
os.makedirs('zips', exist_ok=True)

req = requests.get(url, **kwargs, stream=True)
if req.status_code == 200:
with open(filename, 'wb') as output:
for chunk in req.iter_content(1024):
output.write(chunk)

with zipfile.ZipFile(filename, 'r') as zip:
songdir = f'songs/{groupname}'
print(f'Extracting into {songdir}')
os.makedirs(songdir, exist_ok=True)
zip.extractall(songdir)

if save_headers:
pickle.dump(req.headers, open(save_headers, 'wb'))
elif req.status_code == 304:
print("Not modified")
else:
print(f"Error: {req.status_code} {req.text}")
except requests.BaseHTTPError as e:
print(f'Error downloading: {e.msg}')
except zipfile.BadZipFile:
print(f'Not a zip file: {filename}')
except KeyboardInterrupt as e:
print(f'Download aborting...')
if os.path.isfile(filename):
print(f'Removing partial file {filename}')
os.unlink(filename)
raise e

return req.headers

def mirror(cat_url, args):
request = requests.get(cat_url)
page = BeautifulSoup(request.text, features="html.parser")
@@ -60,26 +97,19 @@ def mirror(cat_url, args):
continue

filename = f'zips/{sim_id}.zip'
if not os.path.isfile(filename):
print(f'Downloading {url} -> {filename}')
try:
os.makedirs('zips', exist_ok=True)
urlretrieve(url, filename)
except KeyboardInterrupt as e:
print(f'Download aborting...')
if os.path.isfile(filename):
print(f'Removing partial file {filename}')
os.unlink(filename)
raise e

try:
with zipfile.ZipFile(filename, 'r') as zip:
songdir = f'songs/{groupname}'
print(f'Extracting into {songdir}')
os.makedirs(songdir, exist_ok=True)
zip.extractall(songdir)
except zipfile.BadZipFile:
print(f'Not a zip file: {filename}')
headers = f'zips/{sim_id}.headers'
if os.path.isfile(headers):
prev_headers = pickle.load(open(headers, 'rb'))
req_headers = {}
if os.path.isfile(filename):
if 'etag' in prev_headers:
req_headers['If-None-Match'] = prev_headers['etag']
if 'last-modified' in prev_headers:
req_headers['If-Modified-Since'] = prev_headers['last-modified']
else:
prev_headers = {}

retrieve(url, filename, headers=req_headers, save_headers=headers)

if __name__ == "__main__":
args = parse_args()


Loading…
Cancel
Save