119 lines
3.9 KiB
Python
119 lines
3.9 KiB
Python
#!/usr/bin/env python3
|
|
"""Build low-confidence ASR review candidates for still-structural sounds.
|
|
|
|
These rows are not native symbols. They are deliberately weaker than accepted
|
|
ASR draft labels and are intended to replace otherwise bare object references
|
|
only when the transcript passes conservative review filters.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import csv
|
|
import re
|
|
from pathlib import Path
|
|
|
|
|
|
FIELDS = [
|
|
"sound_name",
|
|
"wav_path",
|
|
"section8_index",
|
|
"object_id",
|
|
"duration_seconds",
|
|
"channels",
|
|
"transcript",
|
|
"review_label",
|
|
"avg_logprob",
|
|
"no_speech_prob",
|
|
"compression_ratio",
|
|
"segment_count",
|
|
"model",
|
|
"review_rule",
|
|
]
|
|
|
|
|
|
def read_tsv(path: Path) -> list[dict[str, str]]:
|
|
if not path.exists():
|
|
return []
|
|
with path.open(newline="", encoding="utf-8") as f:
|
|
return list(csv.DictReader(f, delimiter="\t"))
|
|
|
|
|
|
def slug(value: str, max_words: int = 8) -> str:
|
|
words = re.findall(r"[A-Za-z0-9]+", value.upper())
|
|
return "_".join(words[:max_words]) or "ASR_REVIEW"
|
|
|
|
|
|
def repeated_noise(text: str) -> bool:
|
|
letters = re.sub(r"[^A-Z]+", "", text.upper())
|
|
if len(letters) > 20 and len(set(letters)) <= 2:
|
|
return True
|
|
words = re.findall(r"[A-Za-z0-9]+", text.upper())
|
|
return len(words) > 8 and len(set(words)) <= 2
|
|
|
|
|
|
def review_rule(row: dict[str, str]) -> str:
|
|
transcript = row["transcript"].strip()
|
|
if not transcript or repeated_noise(transcript) or len(transcript) > 80:
|
|
return ""
|
|
words = re.findall(r"[A-Za-z0-9]+", transcript)
|
|
if not words:
|
|
return ""
|
|
if len(words) == 1 and len(words[0]) == 1 and not words[0].isdigit():
|
|
return ""
|
|
avg_logprob = float(row["avg_logprob"])
|
|
no_speech = float(row["no_speech_prob"])
|
|
compression = float(row["compression_ratio"])
|
|
if compression > 2.0:
|
|
return ""
|
|
if avg_logprob >= -1.0 and no_speech <= 0.12:
|
|
return "avg_logprob_ge_-1.0_and_no_speech_le_0.12"
|
|
if avg_logprob >= -0.92 and no_speech <= 0.18 and len(words) <= 4:
|
|
return "short_transcript_avg_logprob_ge_-0.92_and_no_speech_le_0.18"
|
|
return ""
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--asr-manifest", type=Path, default=Path("analysis/radium-image/asr_draft_sound_labels.tsv"))
|
|
parser.add_argument("--out", type=Path, default=Path("analysis/radium-image/asr_review_sound_label_candidates.tsv"))
|
|
args = parser.parse_args()
|
|
|
|
rows: list[dict[str, str]] = []
|
|
for row in read_tsv(args.asr_manifest):
|
|
if row.get("accepted") != "no":
|
|
continue
|
|
rule = review_rule(row)
|
|
if not rule:
|
|
continue
|
|
rows.append(
|
|
{
|
|
"sound_name": row["sound_name"],
|
|
"wav_path": row["wav_path"],
|
|
"section8_index": row["section8_index"],
|
|
"object_id": row["object_id"],
|
|
"duration_seconds": row["duration_seconds"],
|
|
"channels": row["channels"],
|
|
"transcript": row["transcript"],
|
|
"review_label": f"ASR_REVIEW_{slug(row['transcript'])}",
|
|
"avg_logprob": row["avg_logprob"],
|
|
"no_speech_prob": row["no_speech_prob"],
|
|
"compression_ratio": row["compression_ratio"],
|
|
"segment_count": row["segment_count"],
|
|
"model": row["model"],
|
|
"review_rule": rule,
|
|
}
|
|
)
|
|
|
|
args.out.parent.mkdir(parents=True, exist_ok=True)
|
|
with args.out.open("w", newline="", encoding="utf-8") as f:
|
|
writer = csv.DictWriter(f, fieldnames=FIELDS, delimiter="\t", extrasaction="ignore")
|
|
writer.writeheader()
|
|
writer.writerows(sorted(rows, key=lambda item: int(item["section8_index"])))
|
|
print(f"wrote {args.out} rows={len(rows)}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|