Browse Source

Initial checkin

main
fluffy 8 months ago
commit
9b76e075c0
7 changed files with 244 additions and 0 deletions
  1. BIN
      .DS_Store
  2. +2
    -0
      .gitignore
  3. +19
    -0
      README.md
  4. +86
    -0
      leech.py
  5. +113
    -0
      poetry.lock
  6. +16
    -0
      pyproject.toml
  7. +8
    -0
      update.sh

BIN
.DS_Store View File


+ 2
- 0
.gitignore View File

@@ -0,0 +1,2 @@
songs/
zips/

+ 19
- 0
README.md View File

@@ -0,0 +1,19 @@
# Zenius-I-Vanisher Downloader

The best source for Stepmania charts is [Zenius-I-Vanisher](https://zenius-i-vanisher.com). Unfortunately, there's no easy way to incrementally update collections.

## Usage

To run this script you'll need Python 3.6 or later, and you'll need to install [Beautiful Soup](https://www.crummy.com/software/BeautifulSoup/) with e.g. `pip install beautifulsoup4`. Put it into your Stepmania folder and run it with e.g.

python3 zenius.py

By default it will download the [latest 20 official SIMfiles](https://zenius-i-vanisher.com/v5.2/simfiles.php?category=latest20official) but you can give it one or more alternate URLs to pull from, for example:

python3 zenius.py https://zenius-i-vanisher.com/v5.2/simfiles.php?category=top_official https://zenius-i-vanisher.com/v5.2/viewsimfilecategory.php?categoryid=41

Currently it doesn't detect if a SIMfile has updated; it will only download one which appears for the first time. If a SIMfile has changed, manually delete its zip file from the `zips` directory.

## Notes

ZIV is an amazing resource. Don't abuse this tool, and consider donating to them to keep it running!

+ 86
- 0
leech.py View File

@@ -0,0 +1,86 @@
import urllib.parse
from urllib.request import urlretrieve
import requests
from bs4 import BeautifulSoup
import urllib
import os.path
import zipfile
import os
import argparse

def parse_args(*args):
parser = argparse.ArgumentParser(description="Mirror simfiles from ZIV")

parser.add_argument('categories', type=str, nargs='*', help='ZIV category pages to mirror',
default=['https://zenius-i-vanisher.com/v5.2/simfiles.php?category=latest20official'])

feature = parser.add_mutually_exclusive_group(required=False)
feature.add_argument('--dry-run', '-n',
help="Only perform a dry run; don't send any pings",
dest='dry_run', action='store_true')
feature.add_argument('--no-dry-run',
help="Send pings normally",
dest='dry_run', action='store_false')
feature.set_defaults(dry_run=False)

return parser.parse_args(*args)

def mirror(cat_url, args):
request = requests.get(cat_url)
page = BeautifulSoup(request.text, features="html.parser")

if 'viewsimfilecategory.php' in cat_url:
simgroup = page.find('div', {'class': 'headertop'}).h1
else:
simgroup = None

for row in page.find_all('tr'):
simfile = row.find("a", href=lambda href: href and "viewsimfile.php" in href)
simgroup = simgroup or row.find("a", href=lambda href: href and "viewsimfilecategory.php" in href)

if not (simfile and simgroup):
continue

songname = ' '.join(simfile.get_text().replace('/', '-').split())
groupname = ' '.join(simgroup.get_text().replace('/', '-').split())

print(f"collection: '{groupname}' simfile: '{songname}'")

simlink = simfile['href']
try:
sim_id = urllib.parse.parse_qs(urllib.parse.urlparse(simfile['href']).query)['simfileid'][0]
except KeyError:
print(f"WARNING: no simfileid found on URL {simlink}")
continue

url = f'https://zenius-i-vanisher.com/v5.2/download.php?type=ddrsimfile&simfileid={sim_id}'

if args.dry_run:
print(f"Dry run requested, not downloading {url}")
continue

filename = f'zips/{sim_id}.zip'
if not os.path.isfile(filename):
print(f'Downloading {url} -> {filename}')
try:
urlretrieve(url, filename)
except KeyboardInterrupt as e:
print(f'Download aborting...')
if os.path.isfile(filename):
print(f'Removing partial file {filename}')
os.unlink(filename)
raise e

try:
with zipfile.ZipFile(filename, 'r') as zip:
songdir = f'songs/{groupname}'
print(f'Extracting into {songdir}')
os.makedirs(songdir, exist_ok=True)
zip.extractall(songdir)
except zipfile.BadZipFile:
print(f'Not a zip file: {filename}')

if __name__ == "__main__":
args = parse_args()
for url in args.categories:
mirror(url, args)

+ 113
- 0
poetry.lock View File

@@ -0,0 +1,113 @@
[[package]]
name = "beautifulsoup4"
version = "4.9.3"
description = "Screen-scraping library"
category = "main"
optional = false
python-versions = "*"

[package.dependencies]
soupsieve = {version = ">1.2", markers = "python_version >= \"3.0\""}

[package.extras]
html5lib = ["html5lib"]
lxml = ["lxml"]

[[package]]
name = "certifi"
version = "2020.12.5"
description = "Python package for providing Mozilla's CA Bundle."
category = "main"
optional = false
python-versions = "*"

[[package]]
name = "chardet"
version = "4.0.0"
description = "Universal encoding detector for Python 2 and 3"
category = "main"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"

[[package]]
name = "idna"
version = "2.10"
description = "Internationalized Domain Names in Applications (IDNA)"
category = "main"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"

[[package]]
name = "requests"
version = "2.25.1"
description = "Python HTTP for Humans."
category = "main"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"

[package.dependencies]
certifi = ">=2017.4.17"
chardet = ">=3.0.2,<5"
idna = ">=2.5,<3"
urllib3 = ">=1.21.1,<1.27"

[package.extras]
security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)"]
socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"]

[[package]]
name = "soupsieve"
version = "2.1"
description = "A modern CSS selector implementation for Beautiful Soup."
category = "main"
optional = false
python-versions = ">=3.5"

[[package]]
name = "urllib3"
version = "1.26.2"
description = "HTTP library with thread-safe connection pooling, file post, and more."
category = "main"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4"

[package.extras]
brotli = ["brotlipy (>=0.6.0)"]
secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"]
socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]

[metadata]
lock-version = "1.1"
python-versions = "^3.8"
content-hash = "c39469c7911fdacba7ee26ed25cb96c250190f44ebd1c14b904cb824dbc7d35d"

[metadata.files]
beautifulsoup4 = [
{file = "beautifulsoup4-4.9.3-py2-none-any.whl", hash = "sha256:4c98143716ef1cb40bf7f39a8e3eec8f8b009509e74904ba3a7b315431577e35"},
{file = "beautifulsoup4-4.9.3-py3-none-any.whl", hash = "sha256:fff47e031e34ec82bf17e00da8f592fe7de69aeea38be00523c04623c04fb666"},
{file = "beautifulsoup4-4.9.3.tar.gz", hash = "sha256:84729e322ad1d5b4d25f805bfa05b902dd96450f43842c4e99067d5e1369eb25"},
]
certifi = [
{file = "certifi-2020.12.5-py2.py3-none-any.whl", hash = "sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830"},
{file = "certifi-2020.12.5.tar.gz", hash = "sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c"},
]
chardet = [
{file = "chardet-4.0.0-py2.py3-none-any.whl", hash = "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5"},
{file = "chardet-4.0.0.tar.gz", hash = "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa"},
]
idna = [
{file = "idna-2.10-py2.py3-none-any.whl", hash = "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"},
{file = "idna-2.10.tar.gz", hash = "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6"},
]
requests = [
{file = "requests-2.25.1-py2.py3-none-any.whl", hash = "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"},
{file = "requests-2.25.1.tar.gz", hash = "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804"},
]
soupsieve = [
{file = "soupsieve-2.1-py3-none-any.whl", hash = "sha256:4bb21a6ee4707bf43b61230e80740e71bfe56e55d1f1f50924b087bb2975c851"},
{file = "soupsieve-2.1.tar.gz", hash = "sha256:6dc52924dc0bc710a5d16794e6b3480b2c7c08b07729505feab2b2c16661ff6e"},
]
urllib3 = [
{file = "urllib3-1.26.2-py2.py3-none-any.whl", hash = "sha256:d8ff90d979214d7b4f8ce956e80f4028fc6860e4431f731ea4a8c08f23f99473"},
{file = "urllib3-1.26.2.tar.gz", hash = "sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08"},
]

+ 16
- 0
pyproject.toml View File

@@ -0,0 +1,16 @@
[tool.poetry]
name = "zivleech"
version = "0.1.0"
description = ""
authors = ["fluffy <fluffy@beesbuzz.biz>"]

[tool.poetry.dependencies]
python = "^3.8"
beautifulsoup4 = "^4.9.3"
requests = "^2.25.1"

[tool.poetry.dev-dependencies]

[build-system]
requires = ["poetry-core>=1.0.0"]
build-backend = "poetry.core.masonry.api"

+ 8
- 0
update.sh View File

@@ -0,0 +1,8 @@
#!/bin/sh

cd "$(dirname $0)"
poetry install
poetry run python3 leech.py "$@"

cd songs
rsync --progress -ave ssh . hegemony.local:/Applications/Stepmania/Songs/

Loading…
Cancel
Save