398 lines
16 KiB
Python
398 lines
16 KiB
Python
#!/usr/bin/env python3
|
|
"""Build best-available named aliases for every exported Radium sound."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import csv
|
|
import hashlib
|
|
import os
|
|
import re
|
|
from pathlib import Path
|
|
|
|
|
|
FIELDS = [
|
|
"bank",
|
|
"name_source",
|
|
"confidence",
|
|
"canonical_name",
|
|
"alias_wav_path",
|
|
"source_wav_path",
|
|
"source_sound_name",
|
|
"section8_index",
|
|
"object_id",
|
|
"duration_seconds",
|
|
"channels",
|
|
"sample_rate",
|
|
"command_ref_count",
|
|
"transcript",
|
|
"source_detail",
|
|
"link_type",
|
|
]
|
|
|
|
|
|
def read_tsv(path: Path) -> list[dict[str, str]]:
|
|
if not path.exists():
|
|
return []
|
|
with path.open(newline="", encoding="utf-8") as f:
|
|
return list(csv.DictReader(f, delimiter="\t"))
|
|
|
|
|
|
def slug(value: str, max_len: int = 90) -> str:
|
|
text = value.upper().replace("&", " AND ")
|
|
text = re.sub(r"[^A-Z0-9]+", "_", text).strip("_")
|
|
text = re.sub(r"_+", "_", text)
|
|
if not text:
|
|
text = "SOUND"
|
|
return text[:max_len].rstrip("_") or "SOUND"
|
|
|
|
|
|
def ensure_unique(path: Path, used_names: set[str]) -> Path:
|
|
if path.name not in used_names:
|
|
used_names.add(path.name)
|
|
return path
|
|
stem = path.stem
|
|
suffix = path.suffix
|
|
counter = 2
|
|
while True:
|
|
candidate = path.with_name(f"{stem}_{counter:02d}{suffix}")
|
|
if candidate.name not in used_names:
|
|
used_names.add(candidate.name)
|
|
return candidate
|
|
counter += 1
|
|
|
|
|
|
def link_alias(source_path: Path, alias_path: Path, symlink_target: Path) -> str:
|
|
alias_path.parent.mkdir(parents=True, exist_ok=True)
|
|
if alias_path.exists() or alias_path.is_symlink():
|
|
try:
|
|
if alias_path.samefile(source_path):
|
|
return "hardlink" if not alias_path.is_symlink() else "symlink"
|
|
except OSError:
|
|
pass
|
|
return "existing"
|
|
try:
|
|
os.link(source_path, alias_path)
|
|
return "hardlink"
|
|
except OSError:
|
|
alias_path.symlink_to(symlink_target)
|
|
return "symlink"
|
|
|
|
|
|
def sha256_file(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as f:
|
|
for chunk in iter(lambda: f.read(1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def write_tsv(path: Path, rows: list[dict[str, object]]) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
with path.open("w", newline="", encoding="utf-8") as f:
|
|
writer = csv.DictWriter(f, fieldnames=FIELDS, delimiter="\t", extrasaction="ignore")
|
|
writer.writeheader()
|
|
writer.writerows(rows)
|
|
|
|
|
|
def build_main_rows(args: argparse.Namespace, used_names: set[str]) -> list[dict[str, object]]:
|
|
sounds = read_tsv(args.sound_manifest)
|
|
native_by_source = {row["source_wav_path"]: row for row in read_tsv(args.native_manifest)}
|
|
pokedex_by_source = {row["source_wav_path"]: row for row in read_tsv(args.pokedex_manifest)}
|
|
asr_by_source = {
|
|
row["wav_path"]: row
|
|
for row in read_tsv(args.asr_manifest)
|
|
if row.get("accepted") == "yes" and row.get("draft_label")
|
|
}
|
|
asr_review_by_source = {
|
|
row["wav_path"]: row
|
|
for row in read_tsv(args.asr_review_manifest)
|
|
if row.get("review_label")
|
|
}
|
|
catalog_by_source = {row["wav_path"]: row for row in read_tsv(args.catalog_asr_manifest)}
|
|
catalog_offset_by_object = {
|
|
int(row["candidate_object_id"]): row
|
|
for row in read_tsv(args.catalog_offset_manifest)
|
|
if row.get("candidate_status") == "candidate_unpromoted"
|
|
and row.get("candidate_object_id", "").isdigit()
|
|
}
|
|
|
|
decisions: list[dict[str, object]] = []
|
|
for sound in sorted(sounds, key=lambda row: int(row["section8_index"])):
|
|
source_wav = sound["wav_path"]
|
|
native = native_by_source.get(source_wav)
|
|
pokedex = pokedex_by_source.get(source_wav)
|
|
catalog = catalog_by_source.get(source_wav)
|
|
asr = asr_by_source.get(source_wav)
|
|
asr_review = asr_review_by_source.get(source_wav)
|
|
object_id = sound["primary_object_local_id"] or "unreferenced"
|
|
catalog_offset = (
|
|
catalog_offset_by_object.get(int(object_id))
|
|
if str(object_id).isdigit()
|
|
else None
|
|
)
|
|
source_detail = ""
|
|
transcript = ""
|
|
if native:
|
|
canonical_name = native["name"]
|
|
name_source = "confirmed_native"
|
|
confidence = "confirmed"
|
|
label = slug(canonical_name)
|
|
source_detail = f"register={native['register_call_ghidra']} string={native['string_ghidra']}"
|
|
elif pokedex:
|
|
role = pokedex["role"]
|
|
if role == "description":
|
|
role_label = "description" if pokedex["role_variant"] == "1" else f"description alt {pokedex['role_variant']}"
|
|
elif role == "category":
|
|
role_label = f"{pokedex['csv_category']} category"
|
|
else:
|
|
role_label = "name"
|
|
canonical_name = f"Pokedex {int(pokedex['pokemon_number']):04d} {pokedex['pokemon_name']} {role_label}"
|
|
name_source = "pokedex_voice_alignment"
|
|
confidence = "aligned"
|
|
label = slug(canonical_name)
|
|
transcript = pokedex["transcript"]
|
|
source_detail = (
|
|
f"pokemon_data={pokedex['pokemon_number']} role={pokedex['role']} "
|
|
f"variant={pokedex['role_variant']} action={pokedex['alignment_action']} "
|
|
f"score={pokedex['alignment_score']}"
|
|
)
|
|
elif catalog:
|
|
canonical_name = catalog["catalog_constant"]
|
|
name_source = "catalog_asr_match"
|
|
confidence = "catalog_inferred"
|
|
label = slug(canonical_name)
|
|
transcript = catalog["transcript"]
|
|
source_detail = (
|
|
f"match_kind={catalog['match_kind']} match_phrase={catalog['match_phrase']} "
|
|
f"model={catalog['model']} avg_logprob={catalog['avg_logprob']} "
|
|
f"no_speech={catalog['no_speech_prob']}"
|
|
)
|
|
elif asr:
|
|
canonical_name = asr["transcript"]
|
|
name_source = "asr_draft"
|
|
confidence = "draft"
|
|
label = slug(asr["draft_label"].removeprefix("ASR_"))
|
|
transcript = asr["transcript"]
|
|
source_detail = (
|
|
f"model={asr['model']} avg_logprob={asr['avg_logprob']} "
|
|
f"no_speech={asr['no_speech_prob']}"
|
|
)
|
|
elif catalog_offset:
|
|
canonical_name = f"Catalog-offset candidate {catalog_offset['catalog_constant']}"
|
|
name_source = "catalog_offset_candidate"
|
|
confidence = "candidate_unpromoted"
|
|
label = slug(catalog_offset["catalog_constant"])
|
|
source_detail = (
|
|
f"block={catalog_offset['block_name']} range={catalog_offset['range_kind']} "
|
|
f"catalog_index={catalog_offset['catalog_index']} offset={catalog_offset['offset']} "
|
|
"not_native_confirmed"
|
|
)
|
|
elif asr_review:
|
|
canonical_name = asr_review["transcript"]
|
|
name_source = "asr_review_candidate"
|
|
confidence = "low_confidence_asr"
|
|
label = slug(asr_review["review_label"].removeprefix("ASR_REVIEW_"))
|
|
transcript = asr_review["transcript"]
|
|
source_detail = (
|
|
f"review_rule={asr_review['review_rule']} model={asr_review['model']} "
|
|
f"avg_logprob={asr_review['avg_logprob']} no_speech={asr_review['no_speech_prob']} "
|
|
f"compression={asr_review['compression_ratio']}"
|
|
)
|
|
else:
|
|
canonical_name = f"Object {object_id} section {int(sound['section8_index']):04d}"
|
|
name_source = "object_ref"
|
|
confidence = "structural"
|
|
label = f"OBJ{int(object_id):04d}" if str(object_id).isdigit() else slug(str(object_id))
|
|
source_detail = "section5_object_reference"
|
|
|
|
decisions.append(
|
|
{
|
|
"sound": sound,
|
|
"source_wav": source_wav,
|
|
"canonical_name": canonical_name,
|
|
"name_source": name_source,
|
|
"confidence": confidence,
|
|
"label": label,
|
|
"transcript": transcript,
|
|
"source_detail": source_detail,
|
|
}
|
|
)
|
|
|
|
main_by_hash: dict[str, dict[str, object]] = {}
|
|
for decision in decisions:
|
|
if decision["name_source"] in {"object_ref", "catalog_offset_candidate", "asr_review_candidate"}:
|
|
continue
|
|
main_by_hash.setdefault(sha256_file(Path(str(decision["source_wav"]))), decision)
|
|
|
|
for decision in decisions:
|
|
if decision["name_source"] != "object_ref":
|
|
continue
|
|
hash_match = main_by_hash.get(sha256_file(Path(str(decision["source_wav"]))))
|
|
if not hash_match:
|
|
continue
|
|
decision["name_source"] = "main_duplicate_hash_match"
|
|
decision["confidence"] = f"inherited_{hash_match['confidence']}"
|
|
decision["canonical_name"] = f"Duplicate of {hash_match['canonical_name']}"
|
|
decision["label"] = slug(str(decision["canonical_name"]))
|
|
decision["transcript"] = hash_match["transcript"]
|
|
decision["source_detail"] = (
|
|
f"sha256_match_main_object={hash_match['sound']['primary_object_local_id']} "
|
|
f"main_source={hash_match['name_source']} main_section8={hash_match['sound']['section8_index']}"
|
|
)
|
|
|
|
rows: list[dict[str, object]] = []
|
|
for decision in decisions:
|
|
sound = decision["sound"]
|
|
source_wav = str(decision["source_wav"])
|
|
canonical_name = str(decision["canonical_name"])
|
|
name_source = str(decision["name_source"])
|
|
confidence = str(decision["confidence"])
|
|
label = str(decision["label"])
|
|
transcript = str(decision["transcript"])
|
|
source_detail = str(decision["source_detail"])
|
|
alias_name = (
|
|
f"main__{name_source}__{label}"
|
|
f"__obj{int(sound['primary_object_local_id'] or 0):04d}"
|
|
f"__s{int(sound['section8_index']):04d}"
|
|
f"__{sound['channels']}ch_{sound['sample_rate']}hz.wav"
|
|
)
|
|
alias_path = ensure_unique(args.main_alias_dir / alias_name, used_names)
|
|
source_path = Path(source_wav)
|
|
link_type = link_alias(source_path, alias_path, Path("..") / "sounds" / source_path.name)
|
|
rows.append(
|
|
{
|
|
"bank": "main",
|
|
"name_source": name_source,
|
|
"confidence": confidence,
|
|
"canonical_name": canonical_name,
|
|
"alias_wav_path": alias_path.as_posix(),
|
|
"source_wav_path": source_wav,
|
|
"source_sound_name": sound["sound_name"],
|
|
"section8_index": sound["section8_index"],
|
|
"object_id": sound["primary_object_local_id"],
|
|
"duration_seconds": sound["duration_seconds"],
|
|
"channels": sound["channels"],
|
|
"sample_rate": sound["sample_rate"],
|
|
"command_ref_count": sound["command_ref_count"],
|
|
"transcript": transcript,
|
|
"source_detail": source_detail,
|
|
"link_type": link_type,
|
|
}
|
|
)
|
|
return rows
|
|
|
|
|
|
def build_menu_rows(
|
|
args: argparse.Namespace, used_names: set[str], main_rows: list[dict[str, object]]
|
|
) -> list[dict[str, object]]:
|
|
rows: list[dict[str, object]] = []
|
|
main_by_hash: dict[str, dict[str, object]] = {}
|
|
for row in main_rows:
|
|
if row["name_source"] in {"object_ref", "catalog_offset_candidate", "asr_review_candidate"}:
|
|
continue
|
|
main_by_hash[sha256_file(Path(str(row["source_wav_path"])))] = row
|
|
for sound in sorted(read_tsv(args.spike_menu_manifest), key=lambda row: int(row["section8_index"])):
|
|
object_id = sound["primary_object_local_id"] or "unreferenced"
|
|
source_path = Path(sound["wav_path"])
|
|
main_match = main_by_hash.get(sha256_file(source_path))
|
|
if main_match:
|
|
canonical_name = f"SPIKE menu copy of {main_match['canonical_name']}"
|
|
name_source = "main_hash_match"
|
|
confidence = f"inherited_{main_match['confidence']}"
|
|
label = slug(canonical_name)
|
|
transcript = main_match["transcript"]
|
|
source_detail = (
|
|
f"sha256_match_main_object={main_match['object_id']} "
|
|
f"main_source={main_match['name_source']} main_section8={main_match['section8_index']}"
|
|
)
|
|
else:
|
|
canonical_name = f"SPIKE menu object {object_id} section {int(sound['section8_index']):04d}"
|
|
name_source = "spike_menu_object_ref"
|
|
confidence = "structural"
|
|
label = f"SPIKE_MENU_OBJ{int(object_id):04d}" if str(object_id).isdigit() else slug(str(object_id))
|
|
transcript = ""
|
|
source_detail = "spike_menu_section5_object_reference"
|
|
alias_name = (
|
|
f"spike_menu__{name_source}__{label}"
|
|
f"__s{int(sound['section8_index']):04d}"
|
|
f"__{sound['channels']}ch_{sound['sample_rate']}hz.wav"
|
|
)
|
|
alias_path = ensure_unique(args.spike_menu_alias_dir / alias_name, used_names)
|
|
link_type = link_alias(source_path, alias_path, Path("..") / "sounds" / source_path.name)
|
|
rows.append(
|
|
{
|
|
"bank": "spike_menu",
|
|
"name_source": name_source,
|
|
"confidence": confidence,
|
|
"canonical_name": canonical_name,
|
|
"alias_wav_path": alias_path.as_posix(),
|
|
"source_wav_path": sound["wav_path"],
|
|
"source_sound_name": sound["sound_name"],
|
|
"section8_index": sound["section8_index"],
|
|
"object_id": object_id,
|
|
"duration_seconds": sound["duration_seconds"],
|
|
"channels": sound["channels"],
|
|
"sample_rate": sound["sample_rate"],
|
|
"command_ref_count": sound["command_ref_count"],
|
|
"transcript": transcript,
|
|
"source_detail": source_detail,
|
|
"link_type": link_type,
|
|
}
|
|
)
|
|
return rows
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--sound-manifest", type=Path, default=Path("analysis/radium-image/sound_export_manifest.tsv"))
|
|
parser.add_argument("--native-manifest", type=Path, default=Path("analysis/radium-image/native_named_sound_manifest.tsv"))
|
|
parser.add_argument(
|
|
"--pokedex-manifest", type=Path, default=Path("analysis/radium-image/pokedex_voice_label_manifest.tsv")
|
|
)
|
|
parser.add_argument(
|
|
"--sound-test-manifest", type=Path, default=Path("analysis/radium-image/native_sound_test_label_manifest.tsv")
|
|
)
|
|
parser.add_argument("--asr-manifest", type=Path, default=Path("analysis/radium-image/asr_draft_sound_labels.tsv"))
|
|
parser.add_argument(
|
|
"--asr-review-manifest",
|
|
type=Path,
|
|
default=Path("analysis/radium-image/asr_review_sound_label_candidates.tsv"),
|
|
)
|
|
parser.add_argument(
|
|
"--catalog-asr-manifest",
|
|
type=Path,
|
|
default=Path("analysis/radium-image/asr_catalog_sound_label_candidates.tsv"),
|
|
)
|
|
parser.add_argument(
|
|
"--catalog-offset-manifest",
|
|
type=Path,
|
|
default=Path("analysis/radium-image/catalog_offset_candidate_labels.tsv"),
|
|
)
|
|
parser.add_argument(
|
|
"--spike-menu-manifest", type=Path, default=Path("analysis/radium-spike-menu-image/sound_export_manifest.tsv")
|
|
)
|
|
parser.add_argument("--main-alias-dir", type=Path, default=Path("analysis/radium-image/sounds-best-named"))
|
|
parser.add_argument(
|
|
"--spike-menu-alias-dir", type=Path, default=Path("analysis/radium-spike-menu-image/sounds-best-named")
|
|
)
|
|
parser.add_argument("--out", type=Path, default=Path("analysis/all-sound-best-name-manifest.tsv"))
|
|
args = parser.parse_args()
|
|
|
|
used_names: set[str] = set()
|
|
main_rows = build_main_rows(args, used_names)
|
|
rows = main_rows + build_menu_rows(args, used_names, main_rows)
|
|
write_tsv(args.out, rows)
|
|
counts: dict[str, int] = {}
|
|
for row in rows:
|
|
key = f"{row['bank']}:{row['name_source']}"
|
|
counts[key] = counts.get(key, 0) + 1
|
|
print(f"wrote {args.out} rows={len(rows)}")
|
|
for key in sorted(counts):
|
|
print(f"{key}={counts[key]}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|