Verstack Local
f73167de85
Package Apple Silicon VM exports and cached runtime switching with local cabinet controls and conagent provisioning. Preserve experimental SPIKE 2 emulation. Improve preparation concurrency, Ghidra checkpoints, signature importing, sound indexing and client-side spectrograms. Include regression tests and validation notes.
295 lines
18 KiB
Python
295 lines
18 KiB
Python
#!/usr/bin/env python3
|
|
"""Detect nested archives and SD-card filesystems in userspace, entirely in RAM."""
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import re
|
|
import shutil
|
|
import stat
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import zipfile
|
|
import zlib
|
|
from progress import report
|
|
from parallelism import worker_count
|
|
import fat_files
|
|
|
|
IMAGEUSB_MAGIC = 'imageUSB'.encode('utf-16le') + bytes(16)
|
|
|
|
|
|
def safe_name(name):
|
|
p=pathlib.PurePosixPath(name)
|
|
if p.is_absolute() or '..' in p.parts or '\\' in name or any(c in name for c in '\0\r\n') or not p.parts:
|
|
raise ValueError('Unsafe archive member path')
|
|
return p
|
|
|
|
def container(path):
|
|
with path.open('rb') as f:head=f.read(4096)
|
|
return head[:32] == IMAGEUSB_MAGIC or zipfile.is_zipfile(path) or head[:4] in (b'SPKS',b'hsqs') or head[:6]==b'LUKS\xba\xbe' or head[1080:1082]==b'\x53\xef' or head[510:512]==b'\x55\xaa' or head[:2]==b'\x1f\x8b' and path.suffix.lower()=='.spk'
|
|
|
|
|
|
def sparse_copy(reader, target, length, budget, label):
|
|
"""Zero-filled SD-card padding becomes holes, not allocated tmpfs pages."""
|
|
if length < 0 or length > 128*1024**3:
|
|
raise ValueError('Image exceeds the 128 GiB logical image limit')
|
|
target.parent.mkdir(parents=True,exist_ok=True)
|
|
completed=allocated=0
|
|
with target.open('xb') as output:
|
|
while completed < length:
|
|
data=reader.read(min(1024*1024,length-completed))
|
|
if not data:raise ValueError('Truncated image or archive entry')
|
|
completed+=len(data)
|
|
if data.count(0)==len(data):output.seek(len(data),1)
|
|
else:
|
|
allocated+=len(data)
|
|
if allocated>budget:raise ValueError('Unpacked content exceeds available RAM')
|
|
output.write(data)
|
|
report('Unpacking',completed,length,'bytes',label)
|
|
output.truncate(length)
|
|
return allocated
|
|
|
|
|
|
def partitions(image, image_offset=0):
|
|
size=image.stat().st_size-image_offset
|
|
with image.open('rb') as f:
|
|
f.seek(image_offset)
|
|
head=f.read(4096)
|
|
if len(head)<512 or head[510:512]!=b'\x55\xaa':return []
|
|
f.seek(image_offset+512);gpt=f.read(512)
|
|
result=[]
|
|
if gpt[:8]==b'EFI PART':
|
|
header_size=struct.unpack_from('<I',gpt,12)[0]
|
|
if not 92<=header_size<=512:raise ValueError('Invalid GPT header size')
|
|
crc=struct.unpack_from('<I',gpt,16)[0];check=bytearray(gpt[:header_size]);check[16:20]=b'\0'*4
|
|
if zlib.crc32(check)!=crc:raise ValueError('GPT header checksum mismatch')
|
|
table,count,entry_size,table_crc=struct.unpack_from('<QIII',gpt,72)
|
|
if count>1024 or not 128<=entry_size<=4096 or table*512+count*entry_size>size:raise ValueError('Invalid GPT partition table bounds')
|
|
f.seek(image_offset+table*512);data=f.read(count*entry_size)
|
|
if zlib.crc32(data)!=table_crc:raise ValueError('GPT partition table checksum mismatch')
|
|
for index in range(count):
|
|
entry=data[index*entry_size:(index+1)*entry_size]
|
|
if entry[:16]==bytes(16):continue
|
|
first,last=struct.unpack_from('<QQ',entry,32)
|
|
if last<first:raise ValueError('Invalid GPT partition range')
|
|
result.append((index+1,first*512,(last-first+1)*512))
|
|
else:
|
|
extended=[];primary=[]
|
|
for index in range(4):
|
|
entry=head[446+index*16:462+index*16]
|
|
kind=entry[4];first,count=struct.unpack_from('<II',entry,8)
|
|
if not kind or not count:continue
|
|
if kind==0xee:raise ValueError('Protective MBR has no valid GPT header')
|
|
part=(index+1,first*512,count*512)
|
|
primary.append(part)
|
|
if kind in (5,15,0x85):extended.append(part)
|
|
else:result.append(part)
|
|
validate_partition_ranges(primary,size)
|
|
if len(extended)>1:raise ValueError('Multiple extended MBR containers are unsupported')
|
|
for _,base,length in extended:
|
|
limit=base+length;offset=base;seen=set();ebrs=[]
|
|
while True:
|
|
if offset in seen:raise ValueError('Cycle in extended MBR partition chain')
|
|
if len(seen)>=1024:raise ValueError('Extended MBR partition count limit exceeded')
|
|
if not base<=offset or offset+512>limit:raise ValueError('Extended MBR record is out of bounds')
|
|
seen.add(offset);ebrs.append((0,offset,512))
|
|
f.seek(image_offset+offset);ebr=f.read(512)
|
|
if len(ebr)!=512 or ebr[510:512]!=b'\x55\xaa':raise ValueError('Invalid extended MBR record signature')
|
|
if any(ebr[478:510]):raise ValueError('Unexpected extra extended MBR entries')
|
|
data=ebr[446:462];link=ebr[462:478]
|
|
kind=data[4];first,count=struct.unpack_from('<II',data,8)
|
|
if kind or first or count:
|
|
if not kind or not first or not count or kind in (5,15,0x85,0xee):raise ValueError('Invalid logical MBR partition')
|
|
# Logical data is relative to this EBR; the chain link
|
|
# is relative to the original extended container.
|
|
start=offset+first*512;length=count*512
|
|
if start+length>limit:raise ValueError('Logical MBR partition is out of bounds')
|
|
result.append((4+len(ebrs),start,length))
|
|
kind=link[4];first,count=struct.unpack_from('<II',link,8)
|
|
if not (kind or first or count):break
|
|
if kind not in (5,15,0x85) or not count:raise ValueError('Invalid extended MBR chain link')
|
|
offset=base+first*512
|
|
if offset+count*512>limit:raise ValueError('Extended MBR chain link is out of bounds')
|
|
# EBR sectors must not overlap any filesystem, including a
|
|
# partition described by an earlier record in the chain.
|
|
validate_partition_ranges(result+ebrs,size)
|
|
validate_partition_ranges(result,size)
|
|
return result
|
|
|
|
|
|
def validate_partition_ranges(parts,size):
|
|
end=512
|
|
for _,offset,length in sorted(parts,key=lambda p:p[1]):
|
|
if offset<end or length<=0 or offset+length>size:raise ValueError('Overlapping or out-of-bounds disk partitions')
|
|
end=offset+length
|
|
|
|
|
|
def ext_files(image,destination,settings,budget):
|
|
"""Walk inode numbers and dump only regular files; never follow filesystem symlinks."""
|
|
tool=settings.get('debugfs','/usr/sbin/debugfs')
|
|
pending=[(2,pathlib.Path())];seen=set();used=0;files=0;links=[]
|
|
destination.mkdir(parents=True,exist_ok=True)
|
|
while pending:
|
|
inode,relative=pending.pop()
|
|
if inode in seen:continue
|
|
seen.add(inode)
|
|
process=subprocess.run([tool,'-R',f'ls -p <{inode}>',str(image)],capture_output=True,check=True,text=True)
|
|
rows=process.stdout.splitlines()
|
|
if not rows:raise ValueError('Filesystem directory could not be listed')
|
|
for line in rows:
|
|
if not line.startswith('/'):continue
|
|
fields=line.split('/')
|
|
if len(fields)<7:raise ValueError('Unsupported filesystem filename')
|
|
child,mode,name=fields[1],fields[2],fields[5]
|
|
if name in ('.','..') or child=='0':continue
|
|
safe_name(name)
|
|
if '/' in name or '"' in name:raise ValueError('Unsupported filesystem filename')
|
|
path=relative/name;kind=int(mode,8)
|
|
if stat.S_ISDIR(kind):
|
|
if len(seen)+len(pending)>200000:raise ValueError('Filesystem directory limit exceeded')
|
|
pending.append((int(child),path));continue
|
|
if not stat.S_ISREG(kind):links.append({'path':path.as_posix(),'mode':mode});continue
|
|
size=int(fields[6] or 0);used+=size
|
|
if used>budget:raise ValueError('Filesystem files exceed the RAM extraction allowance')
|
|
dest=destination/path;dest.parent.mkdir(parents=True,exist_ok=True)
|
|
dump=subprocess.run([tool,'-R',f'dump <{int(child)}> "{dest}"',str(image)],capture_output=True)
|
|
if dump.returncode or not dest.is_file() or dest.stat().st_size!=size:raise ValueError(f'Could not read complete filesystem file: {path}')
|
|
files+=1
|
|
report('Reading filesystem',files,None,'files',path.as_posix())
|
|
return links
|
|
|
|
|
|
class Extractor:
|
|
def __init__(self,request):
|
|
self.request=request;self.output=pathlib.Path(request['output_dir']);self.settings=request.get('settings') or {};self.budget=request['workspace_bytes']//2
|
|
self.records=[];self.warnings=[];self.count=0
|
|
def run_file(self,path,destination,depth=0,disk_partition=False):
|
|
self.count+=1
|
|
if depth>8 or self.count>200000:raise ValueError('Nested archive or file count limit exceeded')
|
|
with path.open('rb') as stream:head=stream.read(4096)
|
|
if head[:32] == IMAGEUSB_MAGIC:
|
|
if len(head) < 512:raise ValueError('Truncated ImageUSB header')
|
|
length=struct.unpack_from('<Q',head,48)[0]
|
|
if length < 512 or length != path.stat().st_size-512:
|
|
raise ValueError('ImageUSB payload length mismatch')
|
|
parts=partitions(path,512)
|
|
if parts:
|
|
self.disk(path,destination,depth,parts,512)
|
|
else:
|
|
with tempfile.TemporaryDirectory(prefix='imageusb-',dir=self.output.parent) as tmp:
|
|
raw=pathlib.Path(tmp)/'sdcard.raw'
|
|
with path.open('rb') as reader:
|
|
reader.seek(512)
|
|
sparse_copy(reader,raw,length,self.budget,path.name)
|
|
self.run_file(raw,destination,depth+1)
|
|
self.records.append({'source':path.name,'format':'imageusb','header_bytes':512,'bytes':length})
|
|
elif zipfile.is_zipfile(path):
|
|
with zipfile.ZipFile(path) as archive:
|
|
entries=[e for e in archive.infolist() if not e.is_dir()]
|
|
if len(entries)>200000:raise ValueError('ZIP file count limit exceeded')
|
|
split=entries and all(re.fullmatch(r'.+\.spk\.\d{3}\.\d{3}',e.filename,re.I) for e in entries)
|
|
if split:return self.spk(path,destination,True)
|
|
names=set()
|
|
for entry in entries:
|
|
name=safe_name(entry.filename)
|
|
if str(name) in names:raise ValueError('Duplicate ZIP member path')
|
|
names.add(str(name))
|
|
if stat.S_ISLNK(entry.external_attr>>16):self.warnings.append('ZIP symlink retained only in original: '+entry.filename);continue
|
|
with tempfile.TemporaryDirectory(prefix='member-',dir=self.output.parent) as tmp:
|
|
staged=pathlib.Path(tmp)/name.name
|
|
with archive.open(entry) as reader:sparse_copy(reader,staged,entry.file_size,self.budget,entry.filename)
|
|
# A single archive-wrapped image/package is a transport wrapper,
|
|
# not a version-named directory in the working tree.
|
|
target=destination if len(entries)==1 and container(staged) else destination/name
|
|
self.run_file(staged,target,depth+1)
|
|
self.records.append({'source':path.name,'format':'zip','members':len(entries)})
|
|
elif head[:4] in (b'SPKS',b'hsqs') or (head[:2]==b'\x1f\x8b' and path.name.lower().endswith('.spk')):
|
|
self.spk(path,destination)
|
|
elif head[:6]==b'LUKS\xba\xbe':
|
|
self.luks(path,destination,depth,disk_partition)
|
|
elif len(head)>1082 and head[1080:1082]==b'\x53\xef':
|
|
with tempfile.TemporaryDirectory(prefix='filesystem-',dir=self.output.parent) as tmp:
|
|
tree=pathlib.Path(tmp)/'files'
|
|
links=ext_files(path,tree,self.settings,self.budget)
|
|
for member in sorted(tree.rglob('*')):
|
|
if member.is_file():self.run_file(member,destination/member.relative_to(tree),depth+1)
|
|
self.records.append({'source':path.name,'format':'ext','links_preserved_in_original':links})
|
|
if links:self.warnings.append('Filesystem links and special files remain in the original image')
|
|
elif fat_files.recognized(head):
|
|
with tempfile.TemporaryDirectory(prefix='fat-',dir=self.output.parent) as tmp:
|
|
tree=pathlib.Path(tmp)/'files'
|
|
fat_files.extract(path,tree,self.budget)
|
|
for member in sorted(tree.rglob('*')):
|
|
if member.is_file():self.run_file(member,destination/member.relative_to(tree),depth+1)
|
|
self.records.append({'source':path.name,'format':'fat'})
|
|
elif head[510:512]==b'\x55\xaa' and partitions(path):
|
|
self.disk(path,destination,depth,partitions(path))
|
|
else:
|
|
destination.parent.mkdir(parents=True,exist_ok=True)
|
|
if destination.exists():raise ValueError('Multiple inputs resolve to the same output path')
|
|
with path.open('rb') as stream:sparse_copy(stream,destination,path.stat().st_size,self.budget,path.name)
|
|
self.records.append({'source':path.name,'format':'file','bytes':path.stat().st_size})
|
|
if path.suffix.lower() in ('.raw','.img','.spk'):self.warnings.append('Unrecognized container retained as a file: '+path.name)
|
|
def disk(self,path,destination,depth,parts,image_offset=0):
|
|
# Read partitions directly from the wrapped image; do not stage a second
|
|
# whole SD card alongside its decrypted filesystem.
|
|
with path.open('rb') as source:
|
|
for index,offset,length in parts:
|
|
with tempfile.TemporaryDirectory(prefix='partition-',dir=self.output.parent) as tmp:
|
|
part=pathlib.Path(tmp)/f'partition-{index:02d}.img';source.seek(image_offset+offset)
|
|
sparse_copy(source,part,length,self.budget,part.name)
|
|
self.run_file(part,destination/f'partition-{index:02d}',depth+1,disk_partition=True)
|
|
self.records.append({'source':path.name,'format':'disk-image','image_offset':image_offset,'partitions':[{'index':i,'offset':o,'bytes':n} for i,o,n in parts]})
|
|
def spk(self,path,destination,split=False):
|
|
import spike_package
|
|
with tempfile.TemporaryDirectory(prefix='spk-',dir=self.output.parent) as tmp:
|
|
inputs=pathlib.Path(tmp)/'input';inputs.mkdir();alias=inputs/('package.spk.zip' if split else 'package.spk');os.link(path,alias)
|
|
destination.mkdir(parents=True,exist_ok=True)
|
|
result=spike_package.run({**self.request,'input_dir':str(inputs),'output_dir':str(destination),'settings':self.settings})
|
|
self.warnings.extend(result['warnings']);self.records.append({'source':path.name,'format':'spk'})
|
|
def luks(self,path,destination,depth,disk_partition=False):
|
|
import luks_extract,luks_check
|
|
disk_key=disk_partition and self.settings.get('disk_key_file')
|
|
key_file=disk_key or self.settings.get('key_file')
|
|
if not key_file:raise ValueError('Encrypted image detected, but no decryption key is configured')
|
|
metadata=json.loads(subprocess.check_output([self.settings.get('cryptsetup','/usr/sbin/cryptsetup'),'luksDump','--dump-json-metadata',str(path)]))
|
|
encoding=self.settings.get('disk_key_encoding' if disk_key else 'key_encoding','raw')
|
|
credentials=luks_check.credentials({'key_file':key_file,'key_encoding':encoding})
|
|
try:
|
|
for credential in credentials.values():
|
|
try:key=luks_extract.volume_key(path,credential);break
|
|
except ValueError as error:
|
|
if str(error)!='credential did not unlock this container':raise
|
|
else:raise ValueError(f'Configured key did not unlock {path.name}; check the disk key file and encoding')
|
|
finally:del credentials
|
|
with tempfile.TemporaryDirectory(prefix='decrypt-',dir=self.output.parent) as tmp:
|
|
plain=pathlib.Path(tmp)/'filesystem.ext4'
|
|
report('Decrypting filesystem',None,None,'bytes',path.name,force=True)
|
|
workers=min(16,worker_count(self.request,max(1,path.stat().st_size//(4*1024**2)),per_worker_bytes=128*1024**2))
|
|
luks_extract.decrypt(path,plain,key,metadata,workers=workers);del key
|
|
self.run_file(plain,destination,depth+1)
|
|
self.records.append({'source':path.name,'format':'luks-ext'})
|
|
|
|
|
|
def run(request):
|
|
if request['protocol']!=1:raise ValueError('Unsupported protocol')
|
|
worker=Extractor(request);source=pathlib.Path(request['input_dir'])
|
|
paths=[p for p in sorted(source.rglob('*')) if p.is_file() and not p.is_symlink()]
|
|
for index,path in enumerate(paths):
|
|
report('Detecting files',index,len(paths),'files',path.name,force=True)
|
|
target=worker.output if len(paths)==1 and container(path) else worker.output/path.relative_to(source)
|
|
worker.run_file(path,target)
|
|
report('Cataloging extracted files',len(paths),len(paths),'files',force=True)
|
|
with (worker.output/'import-evidence.json').open('x') as evidence:
|
|
json.dump({'schema':1,'detected':worker.records,'warnings':worker.warnings},evidence)
|
|
return {'protocol':1,'layer':'extracted','coverage':'partial' if worker.warnings else 'complete','warnings':worker.warnings,'files':[p.relative_to(worker.output).as_posix() for p in sorted(worker.output.rglob('*')) if p.is_file()]}
|
|
|
|
if __name__=='__main__':
|
|
request=json.loads(pathlib.Path(sys.argv[1]).read_text())
|
|
try:result=run(request)
|
|
except Exception as error:
|
|
pathlib.Path(request['result_file']).write_text(json.dumps({'protocol':1,'error':str(error)}));raise
|
|
pathlib.Path(request['result_file']).write_text(json.dumps(result))
|