80 lines
4.4 KiB
Python
80 lines
4.4 KiB
Python
"""Bounded gzip/tar reader for early SPIKE packages; links are metadata only."""
|
|
import gzip, hashlib, json, pathlib, tarfile
|
|
|
|
def run(request, package):
|
|
output = pathlib.Path(request['output_dir'])
|
|
destination = output / 'package-0000'
|
|
destination.mkdir()
|
|
inventory, seen, used = [], set(), 0
|
|
budget = int(request['workspace_bytes']) // 2
|
|
# Stream all gzip bytes, including trailer, so CRC and truncation are checked.
|
|
with gzip.open(package, 'rb') as compressed:
|
|
with tarfile.open(fileobj=compressed, mode='r|') as archive:
|
|
for member in archive:
|
|
name = member.name.removeprefix('./').rstrip('/')
|
|
parts = pathlib.PurePosixPath(name)
|
|
if not name or parts.is_absolute() or '..' in parts.parts or '\\' in name:
|
|
raise ValueError('unsafe tar entry path')
|
|
if name in seen:
|
|
raise ValueError('duplicate tar entry')
|
|
seen.add(name)
|
|
if len(seen) > 500000:
|
|
raise ValueError('tar entry limit exceeded')
|
|
record = {'entry': name, 'bytes': member.size, 'mode': member.mode,
|
|
'uid': member.uid, 'gid': member.gid, 'mtime': member.mtime}
|
|
target = destination / name
|
|
if member.isdir():
|
|
target.mkdir(parents=True, exist_ok=True); record['kind'] = 'directory'
|
|
elif member.isfile():
|
|
if member.size < 0 or used + member.size > budget:
|
|
raise ValueError('tar output exceeds workspace budget')
|
|
used += member.size
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
digest = hashlib.sha256(); remaining = member.size
|
|
with archive.extractfile(member) as source, target.open('xb') as sink:
|
|
while remaining:
|
|
data = source.read(min(remaining, 1024 * 1024))
|
|
if not data: raise ValueError('truncated tar payload')
|
|
sink.write(data); digest.update(data); remaining -= len(data)
|
|
record.update(kind='file', sha256=digest.hexdigest())
|
|
elif member.issym() or member.islnk():
|
|
record.update(kind='symlink' if member.issym() else 'hardlink', target=member.linkname)
|
|
else:
|
|
raise ValueError('unsupported tar record type')
|
|
inventory.append(record)
|
|
tail = 0
|
|
while data := compressed.read(1024 * 1024):
|
|
tail += len(data)
|
|
if tail > 16 * 1024 * 1024 or any(data):
|
|
raise ValueError('unexpected trailing tar data')
|
|
(output / 'package-evidence.json').write_text(json.dumps({'schema': 1, 'wrapper': 'gzip/tar', 'inventory': inventory}))
|
|
return {'protocol': 1, 'layer': 'extracted', 'coverage': 'complete',
|
|
'files': [p.relative_to(output).as_posix() for p in sorted(output.rglob('*')) if p.is_file()],
|
|
'warnings': ['Filesystem links preserved as metadata; nested media requires decoding.']}
|
|
|
|
def split_prefix(package, directory, budget):
|
|
"""Validate a single gzip member and split an appended SPKS payload losslessly."""
|
|
import zlib, shutil
|
|
decoder = zlib.decompressobj(31)
|
|
compressed = directory / 'bootstrap.tar.gz'
|
|
payload = directory / 'payload.spk'
|
|
used = offset = 0
|
|
with package.open('rb') as source, compressed.open('xb') as prefix:
|
|
while not decoder.eof:
|
|
chunk = source.read(1024 * 1024)
|
|
if not chunk: raise ValueError('truncated gzip prefix')
|
|
pending = chunk
|
|
while pending and not decoder.eof:
|
|
data = decoder.decompress(pending, 1024 * 1024)
|
|
used += len(data)
|
|
if used > budget: raise ValueError('gzip prefix exceeds workspace budget')
|
|
pending = decoder.unconsumed_tail
|
|
consumed = len(chunk) - len(decoder.unused_data)
|
|
prefix.write(chunk[:consumed]); offset += consumed
|
|
with payload.open('xb') as sink:
|
|
sink.write(decoder.unused_data); shutil.copyfileobj(source, sink, 1024 * 1024)
|
|
if payload.stat().st_size:
|
|
with payload.open('rb') as f:
|
|
if f.read(4) != b'SPKS': raise ValueError('unsupported payload after gzip prefix')
|
|
return compressed, payload, offset
|