989 lines
38 KiB
Python
989 lines
38 KiB
Python
#!/usr/bin/env python3
|
|
"""Parse the Radium mmap image header and section 8 chunk table.
|
|
|
|
This is a static parser for /games/pokemon_pro/image.bin. It implements the
|
|
section offsets and section 8 row fields observed in Ghidra from FUN_008736c0,
|
|
FUN_00873290, and FUN_007a46e0. It does not execute target binaries.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import collections
|
|
import csv
|
|
import json
|
|
import math
|
|
import re
|
|
import struct
|
|
import wave
|
|
import zlib
|
|
from pathlib import Path
|
|
|
|
|
|
HEADER_QWORDS = 13
|
|
SECTION8_ROW_SIZE = 24
|
|
SECTION4_ROW_SIZE = 32
|
|
OBJECT_TABLE_OFFSET = 0x32521C0
|
|
OBJECT_TABLE_ENTRY_SIZE = 0x20
|
|
OBJECT_TABLE_COUNT = 0x3F1
|
|
OPCODE_SIZE_TABLE_OFFSET = 0x3079E78
|
|
OPCODE_COUNT = 0x13
|
|
SECTION8_KEY_MASK = 0xFFFC0003FFFFFFFF
|
|
|
|
|
|
def u32(data: bytes, offset: int) -> int:
|
|
return struct.unpack_from("<I", data, offset)[0]
|
|
|
|
|
|
def u64(data: bytes, offset: int) -> int:
|
|
return struct.unpack_from("<Q", data, offset)[0]
|
|
|
|
|
|
def entropy(sample: bytes) -> float:
|
|
if not sample:
|
|
return 0.0
|
|
counts = [0] * 256
|
|
for b in sample:
|
|
counts[b] += 1
|
|
total = len(sample)
|
|
value = 0.0
|
|
for count in counts:
|
|
if count:
|
|
p = count / total
|
|
value -= p * math.log2(p)
|
|
return value
|
|
|
|
|
|
def derive_section8_fields(q1: int, q2: int, image_index: int) -> dict[str, int]:
|
|
"""Replicate the packed flag math in FUN_007a46e0.
|
|
|
|
The loader stores q2 low32 as a count and later opcode 0x0b computes the
|
|
byte length as bvar4 * q2_low32 * 2.
|
|
"""
|
|
|
|
masked_key = q1 & SECTION8_KEY_MASK
|
|
local_key = (image_index << 34) | masked_key
|
|
low32 = masked_key & 0xFFFFFFFF
|
|
uvar5 = (
|
|
(((local_key >> 25) & 1) << 3)
|
|
| (((local_key >> 30) & 1) << 4)
|
|
| (((local_key >> 17) & 1) << 1)
|
|
| (((local_key >> 31) & 1) << 2)
|
|
)
|
|
channel_or_width = uvar5 | ((low32 >> 7) & 1)
|
|
if ((low32 >> 6) & 1) == 0:
|
|
storage_kind = 1
|
|
preload_width = channel_or_width
|
|
else:
|
|
storage_kind = 2
|
|
preload_width = uvar5 >> 1
|
|
sample_count = q2 & 0xFFFFFFFF
|
|
pcm_bytes = channel_or_width * sample_count * 2
|
|
return {
|
|
"masked_key": masked_key,
|
|
"local_key": local_key,
|
|
"uvar5": uvar5,
|
|
"channel_or_width": channel_or_width,
|
|
"storage_kind": storage_kind,
|
|
"preload_width": preload_width,
|
|
"sample_count": sample_count,
|
|
"pcm_bytes": pcm_bytes,
|
|
}
|
|
|
|
|
|
def parse_section8(data: bytes, image_index: int, table_offset: int, count: int) -> list[dict[str, object]]:
|
|
rows: list[dict[str, object]] = []
|
|
for index in range(count):
|
|
row_offset = table_offset + index * SECTION8_ROW_SIZE
|
|
q0, q1, q2 = struct.unpack_from("<QQQ", data, row_offset)
|
|
next_q0 = None
|
|
next_delta = None
|
|
if index + 1 < count:
|
|
next_q0 = u64(data, table_offset + (index + 1) * SECTION8_ROW_SIZE)
|
|
next_delta = next_q0 - q0
|
|
fields = derive_section8_fields(q1, q2, image_index)
|
|
pcm_bytes = fields["pcm_bytes"]
|
|
first = b""
|
|
if q0 < len(data):
|
|
first = data[q0 : min(q0 + 16, len(data))]
|
|
sample = b""
|
|
if q0 < len(data):
|
|
end = min(q0 + max(0, min(pcm_bytes or next_delta or 0, 65536)), len(data))
|
|
sample = data[q0:end]
|
|
rows.append(
|
|
{
|
|
"index": index,
|
|
"row_offset": row_offset,
|
|
"q0_offset": q0,
|
|
"q1": q1,
|
|
"q2": q2,
|
|
"next_q0_offset": next_q0,
|
|
"next_delta": next_delta,
|
|
"first16_hex": first.hex(),
|
|
"sample_entropy": round(entropy(sample), 4),
|
|
"length_matches_next_delta": next_delta == pcm_bytes if next_delta is not None else None,
|
|
**fields,
|
|
}
|
|
)
|
|
return rows
|
|
|
|
|
|
def parse_section4(data: bytes, table_offset: int, count: int) -> list[dict[str, int]]:
|
|
rows: list[dict[str, int]] = []
|
|
for index in range(count):
|
|
row_offset = table_offset + index * SECTION4_ROW_SIZE
|
|
a, b, c, d = struct.unpack_from("<QQQQ", data, row_offset)
|
|
rows.append(
|
|
{
|
|
"index": index,
|
|
"row_offset": row_offset,
|
|
"a": a,
|
|
"b": b,
|
|
"c": c,
|
|
"d": d,
|
|
"a_minus_b": a - b,
|
|
}
|
|
)
|
|
return rows
|
|
|
|
|
|
def write_tsv(path: Path, rows: list[dict[str, object]], fieldnames: list[str]) -> None:
|
|
with path.open("w", newline="") as f:
|
|
writer = csv.DictWriter(f, fieldnames=fieldnames, delimiter="\t", extrasaction="ignore")
|
|
writer.writeheader()
|
|
for row in rows:
|
|
writer.writerow(row)
|
|
|
|
|
|
def payload_field_summary(payload: bytes) -> dict[str, str]:
|
|
"""Return compact generic little-endian views for a command payload."""
|
|
|
|
u16_values = [
|
|
str(struct.unpack_from("<H", payload, offset)[0])
|
|
for offset in range(0, len(payload) - 1, 2)
|
|
]
|
|
i16_values = [
|
|
str(struct.unpack_from("<h", payload, offset)[0])
|
|
for offset in range(0, len(payload) - 1, 2)
|
|
]
|
|
u32_values = [
|
|
str(struct.unpack_from("<I", payload, offset)[0])
|
|
for offset in range(0, len(payload) - 3, 4)
|
|
]
|
|
i32_values = [
|
|
str(struct.unpack_from("<i", payload, offset)[0])
|
|
for offset in range(0, len(payload) - 3, 4)
|
|
]
|
|
return {
|
|
"payload_hex": payload.hex(),
|
|
"payload_u8_csv": ",".join(str(value) for value in payload),
|
|
"payload_u16le_csv": ",".join(u16_values),
|
|
"payload_i16le_csv": ",".join(i16_values),
|
|
"payload_u32le_csv": ",".join(u32_values),
|
|
"payload_i32le_csv": ",".join(i32_values),
|
|
}
|
|
|
|
|
|
def summarize_opcode_payloads(opcode_rows: list[dict[str, object]]) -> list[dict[str, object]]:
|
|
rows: list[dict[str, object]] = []
|
|
by_opcode: dict[int, list[dict[str, object]]] = collections.defaultdict(list)
|
|
for row in opcode_rows:
|
|
by_opcode[int(row["opcode"])].append(row)
|
|
|
|
def top_values(values: list[str], limit: int = 8) -> str:
|
|
counter = collections.Counter(value for value in values if value != "")
|
|
return ";".join(f"{value}:{count}" for value, count in counter.most_common(limit))
|
|
|
|
for opcode in sorted(by_opcode):
|
|
entries = by_opcode[opcode]
|
|
payloads = [str(row["payload_hex"]) for row in entries]
|
|
first_u8 = []
|
|
second_u8 = []
|
|
first_u16 = []
|
|
second_u16 = []
|
|
first_u32 = []
|
|
for row in entries:
|
|
u8_values = str(row["payload_u8_csv"]).split(",") if row["payload_u8_csv"] else []
|
|
u16_values = str(row["payload_u16le_csv"]).split(",") if row["payload_u16le_csv"] else []
|
|
u32_values = str(row["payload_u32le_csv"]).split(",") if row["payload_u32le_csv"] else []
|
|
first_u8.append(u8_values[0] if len(u8_values) > 0 else "")
|
|
second_u8.append(u8_values[1] if len(u8_values) > 1 else "")
|
|
first_u16.append(u16_values[0] if len(u16_values) > 0 else "")
|
|
second_u16.append(u16_values[1] if len(u16_values) > 1 else "")
|
|
first_u32.append(u32_values[0] if len(u32_values) > 0 else "")
|
|
|
|
rows.append(
|
|
{
|
|
"opcode": opcode,
|
|
"opcode_hex": f"0x{opcode:02x}",
|
|
"payload_length": entries[0]["payload_length"],
|
|
"command_count": len(entries),
|
|
"unique_payload_count": len(set(payloads)),
|
|
"top_payload_hex": top_values(payloads),
|
|
"top_u8_0": top_values(first_u8),
|
|
"top_u8_1": top_values(second_u8),
|
|
"top_u16le_0": top_values(first_u16),
|
|
"top_u16le_1": top_values(second_u16),
|
|
"top_u32le_0": top_values(first_u32),
|
|
}
|
|
)
|
|
return rows
|
|
|
|
|
|
def read_u32_list(data: bytes, offset: int, max_items: int = 4096) -> list[int]:
|
|
values: list[int] = []
|
|
if offset <= 0 or offset >= len(data):
|
|
return values
|
|
for index in range(max_items):
|
|
item_offset = offset + index * 4
|
|
if item_offset + 4 > len(data):
|
|
break
|
|
value = u32(data, item_offset)
|
|
if value == 0:
|
|
break
|
|
values.append(value)
|
|
return values
|
|
|
|
|
|
def read_opcode_lengths(game_data: bytes, opcode_size_table_offset: int) -> list[int]:
|
|
return [u64(game_data, opcode_size_table_offset + opcode * 0x10) for opcode in range(OPCODE_COUNT)]
|
|
|
|
|
|
def build_object_table_by_local_id(
|
|
game_data: bytes,
|
|
image_index: int,
|
|
object_table_offset: int,
|
|
object_table_count: int,
|
|
) -> dict[int, dict[str, object]]:
|
|
object_table_by_local_id: dict[int, dict[str, object]] = {}
|
|
for table_index in range(object_table_count):
|
|
entry_offset = object_table_offset + table_index * OBJECT_TABLE_ENTRY_SIZE
|
|
if entry_offset + OBJECT_TABLE_ENTRY_SIZE > len(game_data):
|
|
break
|
|
list_ptr = u64(game_data, entry_offset)
|
|
table_flag = game_data[entry_offset + 0x0D]
|
|
for object_id in read_u32_list(game_data, list_ptr):
|
|
object_image_id = object_id >> 16
|
|
if object_image_id != image_index:
|
|
continue
|
|
object_local_id = object_id & 0xFFFF
|
|
object_table_by_local_id.setdefault(
|
|
object_local_id,
|
|
{
|
|
"object_table_index": table_index,
|
|
"object_table_flag": table_flag,
|
|
"object_id": object_id,
|
|
},
|
|
)
|
|
return object_table_by_local_id
|
|
|
|
|
|
def get_section5_shape(header: list[int], image_data: bytes) -> tuple[int, int, int]:
|
|
section2_offset = header[2]
|
|
section5_offset = header[5]
|
|
variant_count = image_data[section2_offset + 7]
|
|
section5_next_offset = header[4]
|
|
if section5_next_offset > section5_offset and variant_count:
|
|
local_object_count = (section5_next_offset - section5_offset) // (variant_count * 8)
|
|
else:
|
|
local_object_count = header[11] >> 32
|
|
return section5_offset, variant_count, local_object_count
|
|
|
|
|
|
def parse_command_sound_refs(
|
|
image_data: bytes,
|
|
game_data: bytes,
|
|
section8_rows: list[dict[str, object]],
|
|
header: list[int],
|
|
image_index: int,
|
|
opcode_size_table_offset: int,
|
|
object_table_offset: int,
|
|
object_table_count: int,
|
|
) -> list[dict[str, object]]:
|
|
"""Resolve opcode 0x0b command stream references to section 8 rows."""
|
|
|
|
opcode_lengths = read_opcode_lengths(game_data, opcode_size_table_offset)
|
|
key_to_row = {int(row["local_key"]): row for row in section8_rows}
|
|
section5_offset, variant_count, local_object_count = get_section5_shape(header, image_data)
|
|
refs: list[dict[str, object]] = []
|
|
seen_streams: set[tuple[int, int, int]] = set()
|
|
object_table_by_local_id = build_object_table_by_local_id(
|
|
game_data, image_index, object_table_offset, object_table_count
|
|
)
|
|
|
|
for object_local_id in range(local_object_count):
|
|
metadata = object_table_by_local_id.get(object_local_id, {})
|
|
object_id = metadata.get("object_id", (image_index << 16) | object_local_id)
|
|
for variant in range(variant_count):
|
|
section5_entry = section5_offset + (object_local_id * variant_count + variant) * 8
|
|
if section5_entry + 8 > len(image_data):
|
|
continue
|
|
stream_offset = u64(image_data, section5_entry)
|
|
if stream_offset == 0 or stream_offset >= len(image_data):
|
|
continue
|
|
stream_key = (object_local_id, variant, stream_offset)
|
|
if stream_key in seen_streams:
|
|
continue
|
|
seen_streams.add(stream_key)
|
|
cursor = stream_offset
|
|
opcode_count = 0
|
|
while cursor < len(image_data) and opcode_count < 8192:
|
|
opcode = image_data[cursor]
|
|
if opcode == 0:
|
|
break
|
|
if opcode == 0x0B:
|
|
if cursor + 12 > len(image_data):
|
|
break
|
|
command_channel = image_data[cursor + 1]
|
|
raw_key = u64(image_data, cursor + 4)
|
|
local_key = (raw_key & SECTION8_KEY_MASK) | (image_index << 34)
|
|
row = key_to_row.get(local_key)
|
|
refs.append(
|
|
{
|
|
"object_table_index": metadata.get("object_table_index", ""),
|
|
"object_table_flag": metadata.get("object_table_flag", ""),
|
|
"object_id": object_id,
|
|
"object_local_id": object_local_id,
|
|
"variant": variant,
|
|
"stream_offset": stream_offset,
|
|
"command_offset": cursor,
|
|
"command_channel": command_channel,
|
|
"raw_key": raw_key,
|
|
"local_key": local_key,
|
|
"section8_index": row["index"] if row else "",
|
|
"q0_offset": row["q0_offset"] if row else "",
|
|
"channel_or_width": row["channel_or_width"] if row else "",
|
|
"sample_count": row["sample_count"] if row else "",
|
|
"pcm_bytes": row["pcm_bytes"] if row else "",
|
|
}
|
|
)
|
|
cursor += 12
|
|
elif opcode < OPCODE_COUNT:
|
|
cursor += 1 + opcode_lengths[opcode]
|
|
else:
|
|
break
|
|
opcode_count += 1
|
|
return refs
|
|
|
|
|
|
def parse_command_streams(
|
|
image_data: bytes,
|
|
game_data: bytes,
|
|
section8_rows: list[dict[str, object]],
|
|
header: list[int],
|
|
image_index: int,
|
|
opcode_size_table_offset: int,
|
|
object_table_offset: int,
|
|
object_table_count: int,
|
|
) -> tuple[list[dict[str, object]], list[dict[str, object]], list[dict[str, object]]]:
|
|
"""Decode section 5 command streams generically using the Ghidra opcode-size table."""
|
|
|
|
opcode_lengths = read_opcode_lengths(game_data, opcode_size_table_offset)
|
|
key_to_row = {int(row["local_key"]): row for row in section8_rows}
|
|
section5_offset, variant_count, local_object_count = get_section5_shape(header, image_data)
|
|
object_table_by_local_id = build_object_table_by_local_id(
|
|
game_data, image_index, object_table_offset, object_table_count
|
|
)
|
|
|
|
stream_rows: list[dict[str, object]] = []
|
|
opcode_rows: list[dict[str, object]] = []
|
|
hist: dict[int, dict[str, object]] = {
|
|
opcode: {
|
|
"opcode": opcode,
|
|
"opcode_hex": f"0x{opcode:02x}",
|
|
"known_payload_length": opcode_lengths[opcode],
|
|
"command_count": 0,
|
|
"stream_count": 0,
|
|
"sound_ref_count": 0,
|
|
}
|
|
for opcode in range(OPCODE_COUNT)
|
|
}
|
|
malformed_opcode_count = 0
|
|
|
|
for object_local_id in range(local_object_count):
|
|
metadata = object_table_by_local_id.get(object_local_id, {})
|
|
object_id = metadata.get("object_id", (image_index << 16) | object_local_id)
|
|
for variant in range(variant_count):
|
|
section5_entry = section5_offset + (object_local_id * variant_count + variant) * 8
|
|
if section5_entry + 8 > len(image_data):
|
|
continue
|
|
stream_offset = u64(image_data, section5_entry)
|
|
if stream_offset == 0:
|
|
continue
|
|
stream_row = {
|
|
"object_table_index": metadata.get("object_table_index", ""),
|
|
"object_table_flag": metadata.get("object_table_flag", ""),
|
|
"object_id": object_id,
|
|
"object_local_id": object_local_id,
|
|
"variant": variant,
|
|
"stream_offset": stream_offset,
|
|
"stream_end_offset": "",
|
|
"byte_length": "",
|
|
"opcode_count": 0,
|
|
"sound_ref_count": 0,
|
|
"terminator": "",
|
|
"malformed_reason": "",
|
|
"first_opcodes": "",
|
|
}
|
|
if stream_offset >= len(image_data):
|
|
stream_row["malformed_reason"] = "stream_offset_out_of_range"
|
|
stream_rows.append(stream_row)
|
|
continue
|
|
|
|
stream_opcodes: list[int] = []
|
|
stream_opcode_set: set[int] = set()
|
|
cursor = stream_offset
|
|
ordinal = 0
|
|
while cursor < len(image_data) and ordinal < 8192:
|
|
opcode = image_data[cursor]
|
|
if opcode == 0:
|
|
stream_row["terminator"] = "0x00"
|
|
cursor += 1
|
|
break
|
|
if opcode >= OPCODE_COUNT:
|
|
malformed_opcode_count += 1
|
|
stream_row["malformed_reason"] = f"unknown_opcode_0x{opcode:02x}"
|
|
break
|
|
|
|
payload_length = opcode_lengths[opcode]
|
|
total_length = 1 + payload_length
|
|
sound_command_channel = ""
|
|
sound_raw_key = ""
|
|
sound_local_key = ""
|
|
sound_section8_index = ""
|
|
if opcode == 0x0B:
|
|
payload_length = 11
|
|
total_length = 12
|
|
if cursor + total_length > len(image_data):
|
|
stream_row["malformed_reason"] = "truncated_sound_opcode"
|
|
break
|
|
sound_command_channel = image_data[cursor + 1]
|
|
sound_raw_key = u64(image_data, cursor + 4)
|
|
sound_local_key = (int(sound_raw_key) & SECTION8_KEY_MASK) | (image_index << 34)
|
|
section8_row = key_to_row.get(sound_local_key)
|
|
sound_section8_index = section8_row["index"] if section8_row else ""
|
|
elif cursor + total_length > len(image_data):
|
|
stream_row["malformed_reason"] = "truncated_opcode"
|
|
break
|
|
|
|
payload = image_data[cursor + 1 : cursor + total_length]
|
|
opcode_rows.append(
|
|
{
|
|
"object_table_index": metadata.get("object_table_index", ""),
|
|
"object_table_flag": metadata.get("object_table_flag", ""),
|
|
"object_id": object_id,
|
|
"object_local_id": object_local_id,
|
|
"variant": variant,
|
|
"stream_offset": stream_offset,
|
|
"command_offset": cursor,
|
|
"ordinal": ordinal,
|
|
"opcode": opcode,
|
|
"opcode_hex": f"0x{opcode:02x}",
|
|
"payload_length": payload_length,
|
|
"total_length": total_length,
|
|
"sound_command_channel": sound_command_channel,
|
|
"sound_raw_key": sound_raw_key,
|
|
"sound_local_key": sound_local_key,
|
|
"sound_section8_index": sound_section8_index,
|
|
**payload_field_summary(payload),
|
|
}
|
|
)
|
|
hist[opcode]["command_count"] = int(hist[opcode]["command_count"]) + 1
|
|
if opcode == 0x0B:
|
|
hist[opcode]["sound_ref_count"] = int(hist[opcode]["sound_ref_count"]) + 1
|
|
stream_row["sound_ref_count"] = int(stream_row["sound_ref_count"]) + 1
|
|
stream_opcode_set.add(opcode)
|
|
stream_opcodes.append(opcode)
|
|
cursor += total_length
|
|
ordinal += 1
|
|
|
|
if ordinal >= 8192 and stream_row["malformed_reason"] == "":
|
|
stream_row["malformed_reason"] = "opcode_limit_reached"
|
|
stream_row["stream_end_offset"] = cursor
|
|
stream_row["byte_length"] = cursor - stream_offset
|
|
stream_row["opcode_count"] = ordinal
|
|
stream_row["first_opcodes"] = ",".join(f"0x{op:02x}" for op in stream_opcodes[:16])
|
|
for opcode in stream_opcode_set:
|
|
hist[opcode]["stream_count"] = int(hist[opcode]["stream_count"]) + 1
|
|
stream_rows.append(stream_row)
|
|
|
|
hist_rows = [row for row in hist.values() if int(row["command_count"]) > 0]
|
|
if malformed_opcode_count:
|
|
hist_rows.append(
|
|
{
|
|
"opcode": "",
|
|
"opcode_hex": "unknown",
|
|
"known_payload_length": "",
|
|
"command_count": malformed_opcode_count,
|
|
"stream_count": "",
|
|
"sound_ref_count": "",
|
|
}
|
|
)
|
|
return stream_rows, opcode_rows, hist_rows
|
|
|
|
|
|
def write_wav_previews(
|
|
data: bytes,
|
|
rows: list[dict[str, object]],
|
|
wav_dir: Path,
|
|
sample_rate: int,
|
|
max_wavs: int,
|
|
) -> list[dict[str, object]]:
|
|
wav_dir.mkdir(parents=True, exist_ok=True)
|
|
written: list[dict[str, object]] = []
|
|
for row in rows:
|
|
if len(written) >= max_wavs:
|
|
break
|
|
channels = int(row["channel_or_width"])
|
|
if channels not in (1, 2):
|
|
continue
|
|
q0 = int(row["q0_offset"])
|
|
pcm_bytes = int(row["pcm_bytes"])
|
|
pcm = data[q0 : q0 + pcm_bytes]
|
|
out_path = wav_dir / f"section8_{int(row['index']):04d}_{channels}ch_{sample_rate}hz.wav"
|
|
with wave.open(str(out_path), "wb") as wav:
|
|
wav.setnchannels(channels)
|
|
wav.setsampwidth(2)
|
|
wav.setframerate(sample_rate)
|
|
wav.writeframes(pcm)
|
|
written.append(
|
|
{
|
|
"index": row["index"],
|
|
"path": out_path.as_posix(),
|
|
"q0_offset": q0,
|
|
"channels": channels,
|
|
"sample_rate": sample_rate,
|
|
"pcm_bytes": pcm_bytes,
|
|
"sample_count": row["sample_count"],
|
|
}
|
|
)
|
|
return written
|
|
|
|
|
|
def slug_part(value: object) -> str:
|
|
text = str(value).lower()
|
|
text = re.sub(r"[^a-z0-9]+", "_", text).strip("_")
|
|
return text or "unknown"
|
|
|
|
|
|
def build_sound_export_rows(
|
|
section8_rows: list[dict[str, object]],
|
|
command_refs: list[dict[str, object]],
|
|
sample_rate: int,
|
|
sound_dir: Path,
|
|
) -> list[dict[str, object]]:
|
|
refs_by_section: dict[int, list[dict[str, object]]] = {}
|
|
for ref in command_refs:
|
|
if ref["section8_index"] == "":
|
|
continue
|
|
refs_by_section.setdefault(int(ref["section8_index"]), []).append(ref)
|
|
|
|
export_rows: list[dict[str, object]] = []
|
|
for row in section8_rows:
|
|
index = int(row["index"])
|
|
refs = sorted(
|
|
refs_by_section.get(index, []),
|
|
key=lambda ref: (
|
|
int(ref["object_local_id"]),
|
|
int(ref["variant"]),
|
|
int(ref["command_channel"]),
|
|
int(ref["command_offset"]),
|
|
),
|
|
)
|
|
primary = refs[0] if refs else None
|
|
channels = int(row["channel_or_width"])
|
|
sample_count = int(row["sample_count"])
|
|
duration = sample_count / sample_rate if sample_rate else 0
|
|
unique_object_ids = sorted({int(ref["object_id"]) for ref in refs})
|
|
unique_variants = sorted({int(ref["variant"]) for ref in refs})
|
|
unique_command_channels = sorted({int(ref["command_channel"]) for ref in refs})
|
|
|
|
if primary:
|
|
base_name = (
|
|
f"obj{int(primary['object_local_id']):04d}"
|
|
f"_v{int(primary['variant'])}"
|
|
f"_cmd{int(primary['command_channel'])}"
|
|
f"_s{index:04d}"
|
|
)
|
|
else:
|
|
base_name = f"unreferenced_s{index:04d}"
|
|
sound_name = f"{base_name}_{channels}ch_{sample_rate}hz"
|
|
wav_path = sound_dir / f"{sound_name}.wav"
|
|
|
|
export_rows.append(
|
|
{
|
|
"sound_name": sound_name,
|
|
"wav_path": wav_path.as_posix(),
|
|
"section8_index": index,
|
|
"q0_offset": row["q0_offset"],
|
|
"channels": channels,
|
|
"sample_rate": sample_rate,
|
|
"sample_count": sample_count,
|
|
"duration_seconds": round(duration, 6),
|
|
"pcm_bytes": row["pcm_bytes"],
|
|
"q1": row["q1"],
|
|
"q2": row["q2"],
|
|
"local_key": row["local_key"],
|
|
"command_ref_count": len(refs),
|
|
"object_ids": ";".join(str(value) for value in unique_object_ids),
|
|
"variants": ";".join(str(value) for value in unique_variants),
|
|
"command_channels": ";".join(str(value) for value in unique_command_channels),
|
|
"primary_object_id": primary["object_id"] if primary else "",
|
|
"primary_object_local_id": primary["object_local_id"] if primary else "",
|
|
"primary_variant": primary["variant"] if primary else "",
|
|
"primary_command_channel": primary["command_channel"] if primary else "",
|
|
"primary_stream_offset": primary["stream_offset"] if primary else "",
|
|
"primary_command_offset": primary["command_offset"] if primary else "",
|
|
"first16_hex": row["first16_hex"],
|
|
}
|
|
)
|
|
return export_rows
|
|
|
|
|
|
def write_sound_wavs(data: bytes, export_rows: list[dict[str, object]], force: bool) -> int:
|
|
written = 0
|
|
for row in export_rows:
|
|
channels = int(row["channels"])
|
|
if channels not in (1, 2):
|
|
continue
|
|
wav_path = Path(str(row["wav_path"]))
|
|
if wav_path.exists() and not force:
|
|
continue
|
|
wav_path.parent.mkdir(parents=True, exist_ok=True)
|
|
q0 = int(row["q0_offset"])
|
|
pcm_bytes = int(row["pcm_bytes"])
|
|
pcm = data[q0 : q0 + pcm_bytes]
|
|
with wave.open(str(wav_path), "wb") as wav:
|
|
wav.setnchannels(channels)
|
|
wav.setsampwidth(2)
|
|
wav.setframerate(int(row["sample_rate"]))
|
|
wav.writeframes(pcm)
|
|
written += 1
|
|
return written
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--image", type=Path, default=Path("games/pokemon_pro/image.bin"))
|
|
parser.add_argument("--game-binary", type=Path, default=Path("games/pokemon_pro/game"))
|
|
parser.add_argument("--image-index", type=int, default=0)
|
|
parser.add_argument("--out-dir", type=Path, default=Path("analysis/radium-image"))
|
|
parser.add_argument("--skip-command-refs", action="store_true")
|
|
parser.add_argument("--write-wav-previews", action="store_true")
|
|
parser.add_argument("--max-wavs", type=int, default=16)
|
|
parser.add_argument("--write-all-wavs", action="store_true")
|
|
parser.add_argument("--force-wavs", action="store_true")
|
|
parser.add_argument("--sound-dir", type=Path, default=Path("analysis/radium-image/sounds"))
|
|
parser.add_argument("--sample-rate", type=int, default=44100)
|
|
parser.add_argument("--opcode-size-table-offset", type=lambda value: int(value, 0), default=OPCODE_SIZE_TABLE_OFFSET)
|
|
parser.add_argument("--object-table-offset", type=lambda value: int(value, 0), default=OBJECT_TABLE_OFFSET)
|
|
parser.add_argument("--object-table-count", type=int, default=OBJECT_TABLE_COUNT)
|
|
args = parser.parse_args()
|
|
|
|
data = args.image.read_bytes()
|
|
args.out_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
header = [u64(data, i * 8) for i in range(HEADER_QWORDS)]
|
|
section_count_0 = header[10] & 0xFFFFFFFF
|
|
section_count_1 = header[10] >> 32
|
|
section_count_2 = header[11] & 0xFFFFFFFF
|
|
section_count_3 = header[11] >> 32
|
|
section_count_4 = header[12] & 0xFFFFFFFF
|
|
section_count_5 = header[12] >> 32
|
|
|
|
section8_offset = header[8]
|
|
section8_count = section_count_4
|
|
section8_aligned_len = (section8_count * SECTION8_ROW_SIZE + 15) & ~15
|
|
section8_crc_stored = u32(data, section8_offset + section8_aligned_len)
|
|
section8_crc_calc = zlib.crc32(data[section8_offset : section8_offset + section8_aligned_len]) & 0xFFFFFFFF
|
|
|
|
header_summary = {
|
|
"image_path": str(args.image),
|
|
"image_size": len(data),
|
|
"image_index": args.image_index,
|
|
"header_qwords": [f"0x{x:016x}" for x in header],
|
|
"section_offsets": {f"section_{i}": header[i] for i in range(10)},
|
|
"counts_from_header": {
|
|
"count_0_header10_low32": section_count_0,
|
|
"count_1_header10_high32": section_count_1,
|
|
"count_2_header11_low32": section_count_2,
|
|
"count_3_header11_high32": section_count_3,
|
|
"count_4_header12_low32": section_count_4,
|
|
"count_5_header12_high32": section_count_5,
|
|
},
|
|
"section8_crc": {
|
|
"table_offset": section8_offset,
|
|
"row_count": section8_count,
|
|
"row_bytes": section8_count * SECTION8_ROW_SIZE,
|
|
"aligned_bytes": section8_aligned_len,
|
|
"stored": f"0x{section8_crc_stored:08x}",
|
|
"calculated": f"0x{section8_crc_calc:08x}",
|
|
"matches": section8_crc_stored == section8_crc_calc,
|
|
},
|
|
}
|
|
|
|
section4_rows = parse_section4(data, header[4], section_count_0)
|
|
section8_rows = parse_section8(data, args.image_index, section8_offset, section8_count)
|
|
pcm_candidates = [
|
|
row
|
|
for row in section8_rows
|
|
if int(row["channel_or_width"]) > 0
|
|
and int(row["sample_count"]) > 0
|
|
and int(row["q0_offset"]) + int(row["pcm_bytes"]) <= len(data)
|
|
]
|
|
|
|
exact_matches = sum(1 for row in section8_rows if row["length_matches_next_delta"] is True)
|
|
bounded_pcm = sum(
|
|
1
|
|
for row in section8_rows
|
|
if int(row["pcm_bytes"]) > 0 and int(row["q0_offset"]) + int(row["pcm_bytes"]) <= len(data)
|
|
)
|
|
header_summary["section8_pcm_length_summary"] = {
|
|
"rows": len(section8_rows),
|
|
"rows_with_nonzero_bounded_pcm_length": bounded_pcm,
|
|
"rows_where_pcm_len_equals_next_q0_delta": exact_matches,
|
|
"last_row_excluded_from_next_delta_count": 1 if section8_rows else 0,
|
|
}
|
|
|
|
command_refs: list[dict[str, object]] = []
|
|
command_streams: list[dict[str, object]] = []
|
|
command_opcodes: list[dict[str, object]] = []
|
|
command_opcode_histogram: list[dict[str, object]] = []
|
|
command_opcode_payload_summary: list[dict[str, object]] = []
|
|
if not args.skip_command_refs and args.game_binary.exists():
|
|
game_data = args.game_binary.read_bytes()
|
|
command_refs = parse_command_sound_refs(
|
|
data,
|
|
game_data,
|
|
section8_rows,
|
|
header,
|
|
args.image_index,
|
|
args.opcode_size_table_offset,
|
|
args.object_table_offset,
|
|
args.object_table_count,
|
|
)
|
|
command_streams, command_opcodes, command_opcode_histogram = parse_command_streams(
|
|
data,
|
|
game_data,
|
|
section8_rows,
|
|
header,
|
|
args.image_index,
|
|
args.opcode_size_table_offset,
|
|
args.object_table_offset,
|
|
args.object_table_count,
|
|
)
|
|
command_opcode_payload_summary = summarize_opcode_payloads(command_opcodes)
|
|
resolved_refs = [row for row in command_refs if row["section8_index"] != ""]
|
|
header_summary["command_sound_ref_summary"] = {
|
|
"game_binary": str(args.game_binary),
|
|
"opcode_size_table_offset": f"0x{args.opcode_size_table_offset:x}",
|
|
"object_table_offset": f"0x{args.object_table_offset:x}",
|
|
"object_table_count": args.object_table_count,
|
|
"refs": len(command_refs),
|
|
"resolved_refs": len(resolved_refs),
|
|
"unique_object_ids": len({row["object_id"] for row in command_refs}),
|
|
"unique_section8_indices": len({row["section8_index"] for row in resolved_refs}),
|
|
}
|
|
header_summary["command_stream_summary"] = {
|
|
"streams": len(command_streams),
|
|
"opcode_rows": len(command_opcodes),
|
|
"malformed_streams": sum(1 for row in command_streams if row["malformed_reason"] != ""),
|
|
"unique_opcodes": len({row["opcode"] for row in command_opcodes}),
|
|
"streams_with_sound_refs": sum(1 for row in command_streams if int(row["sound_ref_count"]) > 0),
|
|
}
|
|
|
|
(args.out_dir / "header.json").write_text(json.dumps(header_summary, indent=2) + "\n")
|
|
write_tsv(
|
|
args.out_dir / "section4_table.tsv",
|
|
section4_rows,
|
|
["index", "row_offset", "a", "b", "c", "d", "a_minus_b"],
|
|
)
|
|
section8_fields = [
|
|
"index",
|
|
"row_offset",
|
|
"q0_offset",
|
|
"next_q0_offset",
|
|
"next_delta",
|
|
"q1",
|
|
"q2",
|
|
"masked_key",
|
|
"local_key",
|
|
"uvar5",
|
|
"channel_or_width",
|
|
"storage_kind",
|
|
"preload_width",
|
|
"sample_count",
|
|
"pcm_bytes",
|
|
"length_matches_next_delta",
|
|
"sample_entropy",
|
|
"first16_hex",
|
|
]
|
|
write_tsv(args.out_dir / "section8_entries.tsv", section8_rows, section8_fields)
|
|
write_tsv(args.out_dir / "pcm_candidates.tsv", pcm_candidates, section8_fields)
|
|
if command_refs:
|
|
write_tsv(
|
|
args.out_dir / "command_sound_refs.tsv",
|
|
command_refs,
|
|
[
|
|
"object_table_index",
|
|
"object_table_flag",
|
|
"object_id",
|
|
"object_local_id",
|
|
"variant",
|
|
"stream_offset",
|
|
"command_offset",
|
|
"command_channel",
|
|
"raw_key",
|
|
"local_key",
|
|
"section8_index",
|
|
"q0_offset",
|
|
"channel_or_width",
|
|
"sample_count",
|
|
"pcm_bytes",
|
|
],
|
|
)
|
|
if command_streams:
|
|
write_tsv(
|
|
args.out_dir / "command_stream_summary.tsv",
|
|
command_streams,
|
|
[
|
|
"object_table_index",
|
|
"object_table_flag",
|
|
"object_id",
|
|
"object_local_id",
|
|
"variant",
|
|
"stream_offset",
|
|
"stream_end_offset",
|
|
"byte_length",
|
|
"opcode_count",
|
|
"sound_ref_count",
|
|
"terminator",
|
|
"malformed_reason",
|
|
"first_opcodes",
|
|
],
|
|
)
|
|
write_tsv(
|
|
args.out_dir / "command_opcode_rows.tsv",
|
|
command_opcodes,
|
|
[
|
|
"object_table_index",
|
|
"object_table_flag",
|
|
"object_id",
|
|
"object_local_id",
|
|
"variant",
|
|
"stream_offset",
|
|
"command_offset",
|
|
"ordinal",
|
|
"opcode",
|
|
"opcode_hex",
|
|
"payload_length",
|
|
"total_length",
|
|
"sound_command_channel",
|
|
"sound_raw_key",
|
|
"sound_local_key",
|
|
"sound_section8_index",
|
|
"payload_hex",
|
|
"payload_u8_csv",
|
|
"payload_u16le_csv",
|
|
"payload_i16le_csv",
|
|
"payload_u32le_csv",
|
|
"payload_i32le_csv",
|
|
],
|
|
)
|
|
write_tsv(
|
|
args.out_dir / "command_opcode_payload_summary.tsv",
|
|
command_opcode_payload_summary,
|
|
[
|
|
"opcode",
|
|
"opcode_hex",
|
|
"payload_length",
|
|
"command_count",
|
|
"unique_payload_count",
|
|
"top_payload_hex",
|
|
"top_u8_0",
|
|
"top_u8_1",
|
|
"top_u16le_0",
|
|
"top_u16le_1",
|
|
"top_u32le_0",
|
|
],
|
|
)
|
|
write_tsv(
|
|
args.out_dir / "command_opcode_histogram.tsv",
|
|
command_opcode_histogram,
|
|
[
|
|
"opcode",
|
|
"opcode_hex",
|
|
"known_payload_length",
|
|
"command_count",
|
|
"stream_count",
|
|
"sound_ref_count",
|
|
],
|
|
)
|
|
sound_export_rows = build_sound_export_rows(section8_rows, command_refs, args.sample_rate, args.sound_dir)
|
|
sound_export_fields = [
|
|
"sound_name",
|
|
"wav_path",
|
|
"section8_index",
|
|
"q0_offset",
|
|
"channels",
|
|
"sample_rate",
|
|
"sample_count",
|
|
"duration_seconds",
|
|
"pcm_bytes",
|
|
"q1",
|
|
"q2",
|
|
"local_key",
|
|
"command_ref_count",
|
|
"object_ids",
|
|
"variants",
|
|
"command_channels",
|
|
"primary_object_id",
|
|
"primary_object_local_id",
|
|
"primary_variant",
|
|
"primary_command_channel",
|
|
"primary_stream_offset",
|
|
"primary_command_offset",
|
|
"first16_hex",
|
|
]
|
|
write_tsv(args.out_dir / "sound_export_manifest.tsv", sound_export_rows, sound_export_fields)
|
|
(args.out_dir / "sound_export_manifest.json").write_text(json.dumps(sound_export_rows, indent=2) + "\n")
|
|
header_summary["sound_export_summary"] = {
|
|
"manifest_rows": len(sound_export_rows),
|
|
"referenced_rows": sum(1 for row in sound_export_rows if int(row["command_ref_count"]) > 0),
|
|
"unreferenced_rows": sum(1 for row in sound_export_rows if int(row["command_ref_count"]) == 0),
|
|
"sample_rate": args.sample_rate,
|
|
"sound_dir": args.sound_dir.as_posix(),
|
|
}
|
|
(args.out_dir / "header.json").write_text(json.dumps(header_summary, indent=2) + "\n")
|
|
|
|
if args.write_all_wavs:
|
|
written = write_sound_wavs(data, sound_export_rows, args.force_wavs)
|
|
print(f"all sound wavs written/skipped: written={written} total={len(sound_export_rows)}")
|
|
if args.write_wav_previews:
|
|
preview_rows = [row for row in pcm_candidates if row["length_matches_next_delta"] is True]
|
|
wav_manifest = write_wav_previews(
|
|
data,
|
|
preview_rows,
|
|
args.out_dir / "wav-preview",
|
|
args.sample_rate,
|
|
args.max_wavs,
|
|
)
|
|
(args.out_dir / "wav-preview-manifest.json").write_text(json.dumps(wav_manifest, indent=2) + "\n")
|
|
print(f"wav previews: {len(wav_manifest)}")
|
|
|
|
print(f"wrote {args.out_dir}")
|
|
print(f"section8 crc: stored=0x{section8_crc_stored:08x} calculated=0x{section8_crc_calc:08x}")
|
|
print(f"section8 rows: {len(section8_rows)}")
|
|
print(f"bounded pcm candidates: {len(pcm_candidates)}")
|
|
print(f"pcm_len == next_delta: {exact_matches}")
|
|
if command_refs:
|
|
print(f"command sound refs: {len(command_refs)}")
|
|
if command_streams:
|
|
print(f"command streams: {len(command_streams)}")
|
|
print(f"command opcodes: {len(command_opcodes)}")
|
|
return 0 if section8_crc_stored == section8_crc_calc else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|