Verstack Local
f73167de85
Package Apple Silicon VM exports and cached runtime switching with local cabinet controls and conagent provisioning. Preserve experimental SPIKE 2 emulation. Improve preparation concurrency, Ghidra checkpoints, signature importing, sound indexing and client-side spectrograms. Include regression tests and validation notes.
299 lines
18 KiB
Python
299 lines
18 KiB
Python
#!/usr/bin/env python3
|
|
"""Bounded static media extraction. Profiles adapted from the local triage wiki.
|
|
|
|
Original containers stay in the parent snapshot; exact entry bytes and ranges accompany
|
|
all derivatives. Unsupported layouts are recorded, never interpreted as empty media.
|
|
"""
|
|
import hashlib, json, mmap, pathlib, shutil, struct, sys, wave, zlib
|
|
from collections import deque
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from parallelism import worker_count
|
|
|
|
REVISION = 4
|
|
KNOWN_PCM_RATES = {
|
|
'2f1a958c1f859c339c716e53cc9ee752412373c486ea2a1bfd116eefce4617a3': 44100,
|
|
'a7df00dbbfb75838fd19d57ada30f4247d2ef1bc4b357ca0a64ab60f3a679af0': 44100,
|
|
}
|
|
|
|
def bounds(data, start, size):
|
|
if start < 0 or size < 0 or start > len(data) or size > len(data)-start:
|
|
raise ValueError(f'truncated/out-of-bounds range {start}+{size} of {len(data)}')
|
|
|
|
def safe_name(name):
|
|
if name.startswith('res://'): name=name[6:]
|
|
p=pathlib.PurePosixPath(name)
|
|
if not name or '\x00' in name or '\\' in name or p.is_absolute() or any(x in ('','.','..') for x in name.split('/')):
|
|
raise ValueError('unsafe container entry name')
|
|
return p.as_posix()
|
|
|
|
class Output:
|
|
def __init__(self, root, budget): self.root=root; self.left=budget; self.records=[]
|
|
def write(self, name, data):
|
|
path=self.root/safe_name(name)
|
|
if path.exists(): raise ValueError('duplicate output path')
|
|
if len(data)>self.left: raise ValueError('output budget exceeded')
|
|
self.left-=len(data);path.parent.mkdir(parents=True,exist_ok=True);path.write_bytes(data)
|
|
return name
|
|
def record(self, **row): self.records.append({'decoder_revision':REVISION,'name_confidence':'structural',**row})
|
|
|
|
def sniff(head):
|
|
if head.startswith(b'\x89PNG\r\n\x1a\n'): return '.png'
|
|
if head.startswith(b'\xff\xd8\xff'): return '.jpg'
|
|
if head[:4]==b'RIFF' and head[8:12]==b'WEBP': return '.webp'
|
|
if head[:4]==b'RIFF' and head[8:12]==b'WAVE': return '.wav'
|
|
if head[:4]==b'OggS': return '.ogg'
|
|
if head[:4]==b'fLaC': return '.flac'
|
|
if head[4:8]==b'ftyp': return '.mp4'
|
|
if head[:4]==b'\x1aE\xdf\xa3': return '.webm'
|
|
return None
|
|
|
|
def pck_entries(data):
|
|
bounds(data,0,88)
|
|
magic,version,major,minor,patch=struct.unpack_from('<5I',data)
|
|
if magic!=0x43504447 or version not in (1,2): raise ValueError('unsupported Godot PCK version')
|
|
base=0; pos=84
|
|
if version==2:
|
|
bounds(data,0,100)
|
|
flags,base=struct.unpack_from('<IQ',data,20)
|
|
if flags: raise ValueError('encrypted or relative PCK directory unsupported')
|
|
pos=96
|
|
count=struct.unpack_from('<I',data,pos)[0];pos+=4
|
|
if count>1000000: raise ValueError('PCK entry count exceeds bound')
|
|
entries=[];seen=set()
|
|
for _ in range(count):
|
|
bounds(data,pos,4); n=struct.unpack_from('<I',data,pos)[0];pos+=4
|
|
if not 0<n<=16384: raise ValueError('invalid PCK path length')
|
|
bounds(data,pos,n+32+(4 if version==2 else 0))
|
|
name=safe_name(data[pos:pos+n].rstrip(b'\0').decode('utf-8'));pos+=n
|
|
offset,size=struct.unpack_from('<QQ',data,pos);md5=data[pos+16:pos+32];pos+=32
|
|
flags=0
|
|
if version==2: flags=struct.unpack_from('<I',data,pos)[0];pos+=4
|
|
if flags: raise ValueError('encrypted PCK entry unsupported')
|
|
start=base+offset; bounds(data,start,size)
|
|
if name in seen: raise ValueError('duplicate normalized PCK path')
|
|
seen.add(name);entries.append((name,start,size,md5))
|
|
for _,start,_,_ in entries:
|
|
if start<pos: raise ValueError('PCK payload overlaps directory')
|
|
return entries
|
|
|
|
def decompress_resource(blob,limit):
|
|
if blob[:4]!=b'RSCC':return blob
|
|
bounds(blob,0,16);mode,block_size,total=struct.unpack_from('<3I',blob,4)
|
|
if not block_size or total>limit or total>64*1024*1024:raise ValueError('compressed resource exceeds bound')
|
|
count=(total+block_size-1)//block_size;bounds(blob,16,count*4)
|
|
sizes=struct.unpack_from(f'<{count}I',blob,16);pos=16+count*4;decoded=bytearray()
|
|
for size in sizes:
|
|
bounds(blob,pos,size);block=blob[pos:pos+size];pos+=size
|
|
expected=min(block_size,total-len(decoded))
|
|
if mode==2:
|
|
from compression.zstd import ZstdDecompressor
|
|
decoder=ZstdDecompressor();part=decoder.decompress(block,max_length=expected+1)
|
|
if not decoder.eof or decoder.unused_data:raise ValueError('invalid zstd resource block')
|
|
elif mode in (1,3):
|
|
decoder=zlib.decompressobj(15 if mode==1 else 31);part=decoder.decompress(block,expected+1)
|
|
if not decoder.eof or decoder.unused_data:raise ValueError('invalid deflate resource block')
|
|
else:raise ValueError('unsupported compressed resource codec')
|
|
if len(part)!=expected:raise ValueError('resource decompression length mismatch')
|
|
decoded.extend(part)
|
|
return bytes(decoded)
|
|
|
|
def embedded_font(blob):
|
|
for signature in (b'\x00\x01\x00\x00',b'OTTO'):
|
|
start=0
|
|
while (i:=blob.find(signature,start))>=0:
|
|
start=i+1
|
|
if i+12>len(blob):continue
|
|
count=struct.unpack_from('>H',blob,i+4)[0]
|
|
if not 1<=count<=256 or i+12+count*16>len(blob):continue
|
|
end=i+12+count*16;valid=True
|
|
for n in range(count):
|
|
offset,length=struct.unpack_from('>II',blob,i+12+n*16+8)
|
|
if offset<12+count*16 or i+offset+length>len(blob):valid=False;break
|
|
end=max(end,i+offset+length)
|
|
if valid:return blob[i:end]
|
|
return None
|
|
|
|
def extract_pck(data,out,prefix,source):
|
|
entries=pck_entries(data)
|
|
for name,start,size,md5 in entries:
|
|
blob=data[start:start+size]
|
|
if hashlib.md5(blob).digest()!=md5: raise ValueError('PCK entry checksum mismatch')
|
|
original=out.write(f'{prefix}/original/{name}',blob)
|
|
preview=None;status='preserved'
|
|
if blob[:4]==b'RSCC':
|
|
blob=decompress_resource(blob,out.left)
|
|
out.write(f'{prefix}/decompressed/{name}',blob)
|
|
if name.lower().endswith('.fontdata'):
|
|
font=embedded_font(blob)
|
|
if font is None:status='unsupported_font_resource'
|
|
else:preview=out.write(f'{prefix}/fonts/{name}.ttf',font);status='decoded'
|
|
elif name.lower().endswith('.ctex'):
|
|
# Godot compressed texture payload. Preserve the entire source record;
|
|
# only a fully bounded RIFF chunk is eligible for this preview path.
|
|
if blob[:4]!=b'GST2': status='unsupported_texture_header'
|
|
else:
|
|
start_webp=blob.find(b'RIFF',32)
|
|
if start_webp>=0 and blob[start_webp+8:start_webp+12]==b'WEBP':
|
|
length=struct.unpack_from('<I',blob,start_webp+4)[0]+8
|
|
bounds(blob,start_webp,length)
|
|
preview=out.write(f'{prefix}/previews/{name}.webp',blob[start_webp:start_webp+length]);status='decoded'
|
|
else: status='unsupported_texture_codec'
|
|
elif sniff(blob[:16]):
|
|
preview=out.write(f'{prefix}/previews/{name}{sniff(blob[:16])}',blob);status='decoded'
|
|
|
|
out.record(source=source,entry=name,offset=start,length=size,original=original,preview=preview,status=status,name_confidence='confirmed_container_path')
|
|
|
|
def png(width,height,pixels):
|
|
def chunk(tag,payload):return struct.pack('>I',len(payload))+tag+payload+struct.pack('>I',zlib.crc32(tag+payload)&0xffffffff)
|
|
gray=pixels.translate(bytes(v*17 if v<=15 else v for v in range(256)))
|
|
alpha=pixels.translate(bytes(0 if v==255 else 255 for v in range(256)))
|
|
stride=width*4+1;raw=bytearray(stride*height)
|
|
for y in range(height):
|
|
row=bytearray(width*4);values=gray[y*width:(y+1)*width]
|
|
row[0::4]=values;row[1::4]=values;row[2::4]=values;row[3::4]=alpha[y*width:(y+1)*width]
|
|
raw[y*stride+1:(y+1)*stride]=row
|
|
return b'\x89PNG\r\n\x1a\n'+chunk(b'IHDR',struct.pack('>IIBBBBB',width,height,8,6,0,0,0))+chunk(b'IDAT',zlib.compress(raw))+chunk(b'IEND',b'')
|
|
|
|
def radium_header(data):
|
|
bounds(data,0,104);h=struct.unpack_from('<13Q',data)
|
|
if h[0]<104 or any(x>=len(data) for x in h[:10]): raise ValueError('unrecognized Radium qword layout')
|
|
count=h[12]&0xffffffff
|
|
if count>1000000 or h[12]>>32: raise ValueError('unsupported Radium sound count')
|
|
length=(count*24+15)&~15; bounds(data,h[8],length+4)
|
|
expected=struct.unpack_from('<I',data,h[8]+length)[0]
|
|
if zlib.crc32(data[h[8]:h[8]+length])&0xffffffff!=expected: raise ValueError('Radium sound table CRC mismatch')
|
|
return h
|
|
|
|
def extract_radium(data,out,prefix,source,source_hash,settings):
|
|
h=radium_header(data);count=h[12]&0xffffffff
|
|
profiles=settings.get('pcm_profiles',{})
|
|
profile=profiles.get(source_hash,{})
|
|
rate=KNOWN_PCM_RATES.get(source_hash) or profile.get('sample_rate')
|
|
if rate is not None and (not isinstance(rate,int) or not 8000<=rate<=192000): raise ValueError('invalid PCM profile rate')
|
|
for i in range(count):
|
|
start,flags,samples=struct.unpack_from('<3Q',data,h[8]+i*24)
|
|
channels=(((flags>>25)&1)<<3)|(((flags>>30)&1)<<4)|(((flags>>17)&1)<<1)|(((flags>>31)&1)<<2)|((flags>>7)&1)
|
|
length=channels*(samples&0xffffffff)*2
|
|
bounds(data,start,length)
|
|
if start+length>h[8]: raise ValueError('PCM overlaps sound directory')
|
|
next_start=struct.unpack_from('<Q',data,h[8]+(i+1)*24)[0] if i+1<count else start+length
|
|
if next_start!=start+length: raise ValueError('PCM range does not match next record')
|
|
original=out.write(f'{prefix}/original/sound-{i:05}.pcm',data[start:start+length])
|
|
preview=None;status='unsupported_pcm_flags'
|
|
if channels in (1,2):
|
|
status='missing_verified_sample_rate'
|
|
if rate:
|
|
import io
|
|
wav=io.BytesIO()
|
|
with wave.open(wav,'wb') as w:w.setnchannels(channels);w.setsampwidth(2);w.setframerate(rate);w.writeframes(data[start:start+length])
|
|
preview=out.write(f'{prefix}/sounds/sound-{i:05}.wav',wav.getvalue());status='decoded'
|
|
out.record(source=source,section=8,index=i,offset=start,length=length,channels=channels,sample_rate=rate,duration_seconds=(samples&0xffffffff)/rate if rate else None,original=original,preview=preview,status=status)
|
|
|
|
def extract_bitmaps(data,out,prefix,source,request=None):
|
|
from dmd_bitmap import decode
|
|
bounds(data,0,56);header=struct.unpack_from('<7Q',data)
|
|
start,end=header[3],header[2]
|
|
if not 56<=start<end<len(data) or (end-start)%8 or (end-start)//8>1000000:
|
|
raise ValueError('unrecognized DMD pointer table')
|
|
count=(end-start)//8;bounds(data,start,count*8)
|
|
pointers=struct.unpack_from(f'<{count}Q',data,start)
|
|
if any(p<end or p+13>len(data) or p%8 for p in pointers):raise ValueError('DMD pointer outside asset area')
|
|
ordered=sorted(set(pointers));nexts=dict(zip(ordered,ordered[1:]+[len(data)]));frames={}
|
|
workers=worker_count(request or {'cpu_threads':1},max(1,count//16));pending=deque()
|
|
def save_next():
|
|
i,offset,decoded,future=pending.popleft()
|
|
try:
|
|
original=out.write(f'{prefix}/original/bitmap-{i:05}.bin',data[offset:offset+decoded['consumed']])
|
|
indices=out.write(f'{prefix}/indices/bitmap-{i:05}.idx',decoded['pixels'])
|
|
preview=out.write(f'{prefix}/images/bitmap-{i:05}.png',future.result())
|
|
metadata={k:v for k,v in decoded.items() if k not in ('pixels','consumed')}
|
|
out.record(source=source,section=3,index=i,offset=offset,length=decoded['consumed'],original=original,indices=indices,preview=preview,status='decoded',profile='dmd-indexed/1',preview_accuracy='approximate_palette',**metadata)
|
|
except (ValueError,struct.error) as error:out.record(source=source,section=3,index=i,offset=offset,status='coverage_failure',error=str(error))
|
|
report('Decoding DMD bitmaps',i+1,count,'frames',f'{workers} image workers · {source}')
|
|
with ThreadPoolExecutor(max_workers=workers) as pool:
|
|
for i,offset in enumerate(pointers):
|
|
try:
|
|
# Delta frames depend on the preceding keyframe. Keep index decoding
|
|
# ordered; compress the independent PNG previews concurrently.
|
|
decoded=decode(memoryview(data)[offset:nexts[offset]],frames)
|
|
pending.append((i,offset,decoded,pool.submit(png,decoded['width'],decoded['height'],decoded['pixels'])))
|
|
if len(pending)>=workers:save_next()
|
|
except (ValueError,struct.error) as error:
|
|
while pending:save_next()
|
|
out.record(source=source,section=3,index=i,offset=offset,status='coverage_failure',error=str(error))
|
|
while pending:save_next()
|
|
|
|
def extract_spike1(data,out,prefix,source):
|
|
from vendor.spike1_parser import parse_master
|
|
import io
|
|
parsed=parse_master(data)
|
|
for record in parsed['records']:
|
|
for index,(offset,frames,channels,divisor) in enumerate(record['tracks']):
|
|
length=frames*channels*2
|
|
bounds(data,offset+8,length)
|
|
stem=f"sound-{record['idx']:05}-track-{index+1}"
|
|
original=out.write(f'{prefix}/original/{stem}.pcm',data[offset+8:offset+8+length])
|
|
buf=io.BytesIO()
|
|
with wave.open(buf,'wb') as w:
|
|
w.setnchannels(channels);w.setsampwidth(2);w.setframerate(44100//divisor);w.writeframes(data[offset+8:offset+8+length])
|
|
preview=out.write(f'{prefix}/sounds/{stem}.wav',buf.getvalue())
|
|
out.record(source=source,index=record['idx'],track=index+1,offset=offset+8,length=length,
|
|
source_header_offset=offset,channels=channels,sample_rate=44100//divisor,original=original,preview=preview,status='decoded',profile='spike1-plaintext-pcm')
|
|
out.record(source=source,status='partial_container',profile='spike1-plaintext-pcm',reason='Audio master records decoded; non-audio sections remain unverified')
|
|
|
|
from progress import report
|
|
|
|
def run(request):
|
|
if request['protocol']!=1: raise ValueError('unsupported protocol')
|
|
source=pathlib.Path(request['input_dir']);output=pathlib.Path(request['output_dir']);settings=request.get('settings') or {}
|
|
out=Output(output,request['workspace_bytes']//2)
|
|
paths=[p for p in sorted(source.rglob('*')) if p.is_file() and not p.is_symlink()]
|
|
for index,path in enumerate(paths):
|
|
report('Decoding assets',index,len(paths),'files',path.name)
|
|
if not path.is_file() or path.is_symlink(): continue
|
|
rel=path.relative_to(source).as_posix()
|
|
with path.open('rb') as f: head=f.read(104)
|
|
ext=sniff(head); relevant=ext or head[:4]==b'GDPC' or path.name.startswith('image') and path.suffix=='.bin' or path.suffix in ('.asset','.radium')
|
|
if not relevant: continue
|
|
with path.open('rb') as f: source_hash=hashlib.file_digest(f,'sha256').hexdigest()
|
|
prefix=f'assets/{rel}'
|
|
first_record=len(out.records)
|
|
try:
|
|
with path.open('rb') as f, mmap.mmap(f.fileno(),0,access=mmap.ACCESS_READ) as data:
|
|
if head[:4]==b'GDPC':extract_pck(data,out,prefix,rel)
|
|
elif ext:
|
|
original=out.write(f'{prefix}/original{ext}',data)
|
|
out.record(source=rel,source_sha256=source_hash,offset=0,length=len(data),original=original,preview=original,status='decoded',name_confidence='confirmed_container_path')
|
|
elif path.name.startswith('image') and path.suffix=='.bin':
|
|
try:extract_bitmaps(data,out,prefix,rel,request)
|
|
except (ValueError,struct.error) as error:out.record(source=rel,section=3,status='coverage_failure',error=str(error))
|
|
from vendor.spike1_parser import find_header2, Spike1Error
|
|
try:find_header2(data);spike1=True
|
|
except Spike1Error:spike1=False
|
|
if spike1:extract_spike1(data,out,prefix,rel)
|
|
else:
|
|
effective=dict(settings)
|
|
if source_hash not in KNOWN_PCM_RATES and source_hash not in settings.get('pcm_profiles',{}):
|
|
from pcm_profile import alsa_rate
|
|
binary=path.parent/'game'
|
|
profile=alsa_rate(binary) if binary.is_file() else None
|
|
if profile:
|
|
effective['pcm_profiles']={**settings.get('pcm_profiles',{}),source_hash:profile}
|
|
out.record(source=rel,status='profile_evidence',profile=profile)
|
|
extract_radium(data,out,prefix,rel,source_hash,effective)
|
|
else:
|
|
from scene_media import extract_scene
|
|
extract_scene(path,data,out,prefix,rel)
|
|
except (ValueError,struct.error,UnicodeError,ImportError) as error:out.record(source=rel,source_sha256=source_hash,status='coverage_failure',error=str(error))
|
|
for record in out.records[first_record:]:record.setdefault('source_sha256',source_hash)
|
|
failures=[r for r in out.records if r['status'] not in ('decoded','preserved','scene_metadata_preserved','profile_evidence')]
|
|
out.write('media-evidence.json',json.dumps({'schema':1,'processing_revision':REVISION,'coverage':'partial' if failures else 'complete','assets':out.records},indent=2).encode())
|
|
files=[p.relative_to(output).as_posix() for p in sorted(output.rglob('*')) if p.is_file()]
|
|
return {'protocol':1,'layer':'derived','coverage':'partial' if failures else 'complete','files':files,'warnings':[f'{len(failures)} media coverage failures or approximate previews; original containers retained.'] if failures else []}
|
|
if __name__=='__main__':
|
|
req=json.loads(pathlib.Path(sys.argv[1]).read_text())
|
|
try:result=run(req)
|
|
except Exception as error:
|
|
pathlib.Path(req['result_file']).write_text(json.dumps({'protocol':1,'error':str(error)}));raise
|
|
pathlib.Path(req['result_file']).write_text(json.dumps(result))
|