Verstack Local
f73167de85
Package Apple Silicon VM exports and cached runtime switching with local cabinet controls and conagent provisioning. Preserve experimental SPIKE 2 emulation. Improve preparation concurrency, Ghidra checkpoints, signature importing, sound indexing and client-side spectrograms. Include regression tests and validation notes.
145 lines
7.2 KiB
Python
145 lines
7.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Materialize any extracted SPIKE runtime using verified archive files."""
|
|
import concurrent.futures
|
|
import json
|
|
import os
|
|
from pathlib import Path, PurePosixPath
|
|
import urllib.parse
|
|
import urllib.request
|
|
import subprocess
|
|
import uuid
|
|
import threading
|
|
import runtimes
|
|
|
|
|
|
def manifest(api, snapshot):
|
|
with urllib.request.urlopen(f'{api}/api/snapshots/{snapshot}', timeout=120) as response:
|
|
return json.load(response)
|
|
|
|
|
|
def file_url(api, snapshot, path):
|
|
return f'{api}/api/file/{snapshot}?' + urllib.parse.urlencode({'path': path})
|
|
|
|
|
|
def header(api, snapshot, path):
|
|
request = urllib.request.Request(file_url(api, snapshot, path), headers={'Range':'bytes=0-63'})
|
|
with urllib.request.urlopen(request, timeout=30) as response:
|
|
return response.read(64)
|
|
|
|
|
|
def discover(api, source, game_path=None):
|
|
files = {e['path']: e for e in source['entries'] if e['kind'] == 'file'}
|
|
candidates = [game_path] if game_path else [p for p in files if (p == 'game' or p.endswith('/game')) and '/spike_menu/' not in p and '/etc/' not in p]
|
|
games = []
|
|
for path in candidates:
|
|
if path not in files: raise ValueError('Selected game is absent from snapshot')
|
|
try: arch = runtimes.architecture(header(api, source['id'], path))
|
|
except ValueError: continue
|
|
games.append((path, arch))
|
|
if len(games) != 1: raise ValueError('Choose --game-path: expected exactly one ARM game executable')
|
|
game, arch = games[0]
|
|
return game, arch, discover_system(api, source, arch)
|
|
|
|
|
|
def discover_system(api, source, arch):
|
|
files = {e['path']: e for e in source['entries'] if e['kind'] == 'file'}
|
|
loaders = []
|
|
for path in files:
|
|
name = PurePosixPath(path).name
|
|
if '/lib/' in path and name.startswith('ld-') and '.so' in name:
|
|
try:
|
|
if runtimes.architecture(header(api, source['id'], path)) == arch:
|
|
loaders.append(str(PurePosixPath(path).parent.parent))
|
|
except ValueError: pass
|
|
roots = set(loaders)
|
|
system = next(iter(roots)) if len(roots) == 1 else None
|
|
return system
|
|
|
|
|
|
def materialize(api, destination, snapshot, system_snapshot=None, game_path=None):
|
|
uuid.UUID(snapshot)
|
|
source = manifest(api, snapshot)
|
|
game, arch, system = discover(api, source, game_path)
|
|
system_source = source
|
|
if system_snapshot:
|
|
system_source = manifest(api, system_snapshot)
|
|
system = discover_system(api, system_source, arch)
|
|
if not system: raise ValueError('Full SD system libraries are missing; supply --system-snapshot from a compatible SD extraction')
|
|
root = Path(destination) / snapshot
|
|
root.mkdir(parents=True, exist_ok=True)
|
|
game_root = str(PurePosixPath(game).parent)
|
|
entries = {}
|
|
for selected, prefix in ((system_source, system), (source, game_root)):
|
|
for entry in selected['entries']:
|
|
path = runtimes.safe_path(entry['path'])
|
|
if prefix != '.' and not str(path).startswith(prefix + '/'): continue
|
|
if entry['kind'] == 'directory': continue
|
|
if entry['kind'] != 'file': raise ValueError('Runtime symlinks must be reconstructed from verified files')
|
|
entries[entry['path']] = (selected['id'], entry)
|
|
# SPIKE 2 system menu/boot files can reside outside the game's directory.
|
|
targets = {'game':'game'}
|
|
files = {e['path'] for e in source['entries'] if e['kind']=='file'}
|
|
for name, suffix in (('spike-menu','spike3/spike_menu/game'),('boot-display','spike3/bin/boot_display')):
|
|
if game_root + '/' + suffix in files: targets[name] = suffix
|
|
system_files = {e['path'] for e in system_source['entries'] if e['kind']=='file'}
|
|
if arch == 'armhf':
|
|
for name, suffix in (('spike-menu','usr/local/spike/spike_menu/game'),('boot-display','usr/local/bin/boot_display')):
|
|
if system + '/' + suffix in system_files: targets[name] = 'system/' + suffix
|
|
# Shared system files and unchanged game assets need no new archive read.
|
|
# Verify the existing local file before hard-linking it into another runtime.
|
|
reusable = {}
|
|
for runtime in runtimes.inventory(destination):
|
|
for sid in {runtime.get('snapshot'), runtime.get('system_snapshot', runtime.get('snapshot'))}:
|
|
if not sid: continue
|
|
prior = manifest(api, sid)
|
|
for entry in prior['entries']:
|
|
if entry['kind'] == 'file':
|
|
local = Path(runtime['directory']) / entry['path']
|
|
if local.is_file(): reusable.setdefault(entry['artifact'], local)
|
|
link_lock = threading.Lock()
|
|
verified = set()
|
|
def copy(item):
|
|
sid, entry = item
|
|
target = root / entry['path']; target.parent.mkdir(parents=True, exist_ok=True)
|
|
expected = entry['artifact'].removeprefix('blake3:')
|
|
def digest(path): return subprocess.check_output(['b3sum', str(path)], text=True).split()[0]
|
|
if target.is_file() and target.stat().st_size == entry['size'] and digest(target) == expected: return
|
|
prior = reusable.get(entry['artifact'])
|
|
if prior and prior != target:
|
|
with link_lock:
|
|
if entry['artifact'] not in verified:
|
|
if prior.stat().st_size != entry['size'] or digest(prior) != expected:
|
|
raise ValueError('Existing runtime file failed artifact verification')
|
|
verified.add(entry['artifact'])
|
|
target.unlink(missing_ok=True)
|
|
os.link(prior, target)
|
|
return
|
|
temporary = target.with_name(target.name + '.download')
|
|
try:
|
|
with urllib.request.urlopen(file_url(api,sid,entry['path']),timeout=180) as response, temporary.open('wb') as output:
|
|
for block in iter(lambda: response.read(1024*1024), b''): output.write(block)
|
|
if temporary.stat().st_size != entry['size'] or digest(temporary) != expected:
|
|
raise ValueError('Artifact verification failed: ' + entry['path'])
|
|
temporary.chmod(entry['mode'] & 0o777);os.replace(temporary,target)
|
|
finally: temporary.unlink(missing_ok=True)
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=6) as pool:
|
|
for number,_ in enumerate(pool.map(copy,entries.values()),1):
|
|
if number % 100 == 0: print(f'Verified {number}/{len(entries)} files for {runtimes.label(source["release"])}',flush=True)
|
|
record = dict(schema=2,snapshot=snapshot,release=source['release'],architecture=arch,
|
|
game_root=game_root,system_root=system,system_snapshot=system_source['id'],targets=targets,files=len(entries))
|
|
temporary=root/'runtime.json.next';temporary.write_text(json.dumps(record));temporary.replace(root/'runtime.json')
|
|
print('Runtime materialization complete: '+runtimes.label(source['release']),flush=True)
|
|
return record
|
|
|
|
|
|
if __name__ == '__main__':
|
|
import argparse
|
|
parser=argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--api',default='http://127.0.0.1:8080')
|
|
parser.add_argument('--destination',default='/runtime')
|
|
parser.add_argument('--snapshot',required=True)
|
|
parser.add_argument('--system-snapshot')
|
|
parser.add_argument('--game-path')
|
|
args=parser.parse_args()
|
|
materialize(args.api,args.destination,args.snapshot,args.system_snapshot,args.game_path)
|