Verstack Local
841a95c56a
Save the tested release workflows, RAM media processing, program discovery, FLIRT library, and shared-object maintenance. Add Git exclusions, file attributes, and instructions for a later push using a forwarded SSH agent.
161 lines
8.2 KiB
Python
161 lines
8.2 KiB
Python
#!/usr/bin/env python3
|
|
"""HTTP(S) source inspection and bounded RAM downloads; never runs downloaded code."""
|
|
import email.message
|
|
import hashlib
|
|
import http.client
|
|
import ipaddress
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import socket
|
|
import ssl
|
|
import sys
|
|
import time
|
|
import urllib.parse
|
|
|
|
|
|
def progress(path, **value):
|
|
if not path:
|
|
return
|
|
target = pathlib.Path(path)
|
|
temporary = target.with_suffix('.next')
|
|
temporary.write_text(json.dumps(value))
|
|
temporary.replace(target)
|
|
|
|
|
|
def connect(url, method='GET', allow_private=False):
|
|
"""Validate every redirect and pin the validated address through TLS connection."""
|
|
for _ in range(6):
|
|
parsed = urllib.parse.urlsplit(url)
|
|
if parsed.scheme not in ('https', 'http') or not parsed.hostname or parsed.username or parsed.password:
|
|
raise ValueError('Use an HTTP or HTTPS URL without embedded credentials')
|
|
port = parsed.port or (443 if parsed.scheme == 'https' else 80)
|
|
addresses = socket.getaddrinfo(parsed.hostname, port, type=socket.SOCK_STREAM)
|
|
if not addresses:
|
|
raise ValueError('Source hostname could not be resolved')
|
|
ips = [address[4][0] for address in addresses]
|
|
if not allow_private and any(not ipaddress.ip_address(ip).is_global for ip in ips):
|
|
raise ValueError('URL imports require a public internet address')
|
|
cls = http.client.HTTPSConnection if parsed.scheme == 'https' else http.client.HTTPConnection
|
|
connection = cls(parsed.hostname, port, timeout=30)
|
|
def pinned_connection(address, timeout=30, source_address=None):
|
|
last=None
|
|
for ip in ips:
|
|
try:return socket.create_connection((ip,port),timeout,source_address)
|
|
except OSError as error:last=error
|
|
raise last
|
|
connection._create_connection = pinned_connection
|
|
connection.request(method, urllib.parse.urlunsplit(('', '', parsed.path or '/', parsed.query, '')),
|
|
headers={'User-Agent':'Verstack/0.1 (user-requested archive import)', 'Accept-Encoding':'identity'})
|
|
response = connection.getresponse()
|
|
if response.status in (301, 302, 303, 307, 308):
|
|
location = response.getheader('Location')
|
|
connection.close()
|
|
if not location:
|
|
raise ValueError('Redirect is missing a destination')
|
|
url = urllib.parse.urljoin(url, location)
|
|
continue
|
|
if not 200 <= response.status < 300:
|
|
connection.close()
|
|
raise ValueError(f'Source returned HTTP {response.status} {response.reason}')
|
|
return connection, response, url
|
|
raise ValueError('Too many source redirects')
|
|
|
|
|
|
def filename(url, response):
|
|
header = email.message.Message()
|
|
header['Content-Disposition'] = response.getheader('Content-Disposition', '')
|
|
name = header.get_filename() or urllib.parse.unquote(urllib.parse.urlsplit(url).path.rsplit('/',1)[-1]) or 'download.bin'
|
|
if name in ('.', '..') or any(c in name for c in '/\\\x00\r\n') or len(name.encode()) > 240:
|
|
raise ValueError('Source filename is invalid; use a direct file URL')
|
|
return name
|
|
|
|
|
|
def inspect(url, allow_private=False):
|
|
parsed = urllib.parse.urlsplit(url)
|
|
parts = [urllib.parse.unquote(p) for p in parsed.path.split('/') if p]
|
|
if parsed.hostname in ('archive.org', 'www.archive.org') and len(parts) == 2 and parts[0] in ('details','download'):
|
|
identifier = parts[1]
|
|
endpoint = 'https://archive.org/metadata/' + urllib.parse.quote(identifier, safe='')
|
|
connection, response, _ = connect(endpoint, allow_private=allow_private)
|
|
try:
|
|
raw = response.read(8*1024*1024+1)
|
|
if len(raw) > 8*1024*1024:
|
|
raise ValueError('Archive file list exceeds the supported size; use a direct file URL')
|
|
metadata = json.loads(raw)
|
|
finally:
|
|
connection.close()
|
|
files = []
|
|
for entry in metadata.get('files', []):
|
|
name = entry.get('name', '')
|
|
if entry.get('source') != 'original' or name.endswith(('_files.xml','_meta.xml','_meta.sqlite','.torrent')):
|
|
continue
|
|
size = int(entry['size']) if entry.get('size') else None
|
|
files.append({'name':name,'size':size,'url':'https://archive.org/download/'+urllib.parse.quote(identifier,safe='')+'/'+urllib.parse.quote(name,safe='/')})
|
|
if not files:
|
|
raise ValueError('No downloadable original files were found in this archive item')
|
|
return {'kind':'collection','title':metadata.get('metadata',{}).get('title',identifier),'files':files}
|
|
try:
|
|
connection, response, resolved = connect(url, method='HEAD', allow_private=allow_private)
|
|
except ValueError as error:
|
|
if not str(error).startswith(('Source returned HTTP 405 ','Source returned HTTP 501 ')):raise
|
|
# Some file hosts reject HEAD. Read GET headers only, then close the body.
|
|
connection, response, resolved = connect(url, allow_private=allow_private)
|
|
try:
|
|
if 'text/html' in response.getheader('Content-Type','').lower():
|
|
raise ValueError('This URL is a web page. Use a direct file URL or an Internet Archive item link')
|
|
return {'kind':'file','files':[{'name':filename(resolved,response),'url':url,'size':int(response.getheader('Content-Length')) if response.getheader('Content-Length') else None}]}
|
|
finally:
|
|
connection.close()
|
|
|
|
|
|
def download(request):
|
|
connection, response, resolved = connect(request['url'], allow_private=request.get('allow_private',False))
|
|
try:
|
|
if 'text/html' in response.getheader('Content-Type','').lower():
|
|
raise ValueError('The download returned a web page, not a file')
|
|
total = int(response.getheader('Content-Length')) if response.getheader('Content-Length') else None
|
|
limit = int(request['limit'])
|
|
logical_limit=int(request.get('logical_limit',limit))
|
|
if total is not None and total > logical_limit:
|
|
raise ValueError('Download exceeds the supported logical file size')
|
|
name = filename(resolved, response)
|
|
destination = pathlib.Path(request['directory']) / name
|
|
received = allocated = 0
|
|
progress(request.get('progress_file'),stage='Downloading',completed=0,total=total,unit='bytes',detail=name)
|
|
digest = hashlib.sha256()
|
|
started = last = time.monotonic()
|
|
with destination.open('xb') as output:
|
|
while data := response.read(1024*1024):
|
|
received += len(data)
|
|
if received > logical_limit:
|
|
raise ValueError('Download exceeded the supported logical file size')
|
|
if data.count(0)==len(data):output.seek(len(data),1)
|
|
else:
|
|
allocated+=len(data)
|
|
if allocated>limit:raise ValueError('Downloaded data exceeded the RAM import allowance; choose a smaller or compressed source')
|
|
output.write(data)
|
|
digest.update(data)
|
|
now = time.monotonic()
|
|
if now-last >= .25:
|
|
progress(request.get('progress_file'),stage='Downloading',completed=received,total=total,unit='bytes',bytes_per_second=received/max(.001,now-started),detail=name)
|
|
last=now
|
|
output.truncate(received)
|
|
if total is not None and received != total:
|
|
raise ValueError('Download ended before the advertised file size was received')
|
|
if not received:
|
|
raise ValueError('Source returned an empty file')
|
|
progress(request.get('progress_file'),stage='Download complete',completed=received,total=received,unit='bytes',detail=name)
|
|
return {'path':str(destination),'name':name,'bytes':received,'sha256':digest.hexdigest(),'url':request['url']}
|
|
finally:
|
|
connection.close()
|
|
|
|
if __name__ == '__main__':
|
|
request=json.loads(pathlib.Path(sys.argv[1]).read_text())
|
|
try:
|
|
result=inspect(request['url'], request.get('allow_private',False)) if request['mode']=='inspect' else download(request)
|
|
pathlib.Path(request['result_file']).write_text(json.dumps(result))
|
|
except Exception as error:
|
|
pathlib.Path(request['result_file']).write_text(json.dumps({'error':str(error)}))
|
|
sys.exit(1)
|