From bb1c45a19fd25653b3fde5a8854220dabeac8ccb Mon Sep 17 00:00:00 2001 From: Honney OC Date: Sun, 11 Oct 2026 17:02:13 +0000 Subject: [PATCH] chore: sanitize personal paths, add README --- README.md | 238 ++++++++++++++++++++++++++++++++++++++++++++++++++ extract.py | 4 +- sync_audio.py | 2 +- 3 files changed, 241 insertions(+), 3 deletions(-) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..5a44e85 --- /dev/null +++ b/README.md @@ -0,0 +1,238 @@ +# movie_sync + +Sync two releases/cuts of the same movie (different language dub, different +master, different container delay) by measuring content-to-content offset and +muxing a synced output. + +Pipeline: pick a window → extract lossless clips → measure audio/video lag → +mux delayed audio → verify A/B. + +``` +file_a ─┐ + ├─► extract.py ─► extracted/{stem}_{video,audio,proxy} ─► sync_audio_test.py (AST) ─┐ +file_b ─┘ ├─► lag (seconds) ─► apply_sync.py ─► output/{stem}_synced.mkv ─► verify_sync.py + sync_video_test.py (VST) ─┘ +``` + +Lag convention (used everywhere, feeds `apply_sync --lag` directly): +`lag > 0` means **B is delayed vs A** (B starts later). `lag < 0` means B is +early. All estimators (`sync_audio.correlate`, VST methods) share this sign. + +## Requirements + +- Python 3.11+ (`tomllib` from stdlib; 3.14 also works) +- `ffmpeg` + `ffprobe` on `PATH` (extraction, proxy, mux, all probing) +- `mpv` for `verify_sync.py` video playback (IPC-controlled, `--no-audio`) +- Python deps (installed via `make install`): `torch`, `torchaudio`, + `soundfile`, `scipy`, `numpy`, `pyqtgraph`, `PyQt6` +- Optional: `sounddevice` (audio playback in AST/verify), + `openai-whisper` (only for `rename_audio.py`), `python-mpv` + +No GPU is required. `torch.cuda` (ROCm or CUDA) is used when available, +otherwise everything falls back to NumPy CPU. + +## Install + +```bash +make install # auto-detect: ROCm > CUDA > CPU, create .venv, install deps +make install-cpu # force CPU torch +make install-rocm # force ROCm torch (e.g. RX 7900 XTX) +make install-cuda # force CUDA torch +``` + +Activate manually with `source .venv/bin/activate`. Python entry points also +work via `make ` without activating. + +## Configuration + +All paths/settings live in `config.toml` (repo root). Copy the shipped +template and set your two sources: + +```toml +[extract] +file_a = "/path/to/movie_a.mkv" +file_b = "/path/to/movie_b.mp4" +duration = 30 # analysis window, seconds (the ONLY duration needed) +start = 0.0 # start offset, seconds (ffmpeg -ss before -i) +out_dir = "extracted" +target_fps = "24000/1001" +target_fps_eps = 0.02 +proxy = true # build {stem}_proxy.mkv (192px gray FFV1) for VST +proxy_width = 192 + +[audio] +sample_rate = 48000 +channels = 1 +normalize = true # two-pass EBU R128 loudnorm +target_lufs = -14 +target_tp = -1.0 +target_lra = 11 +# stream = null # null = prompt, int = index, "skip" = no audio +volume = 70 + +[sync] +method = "phat" # "phat" (GCC-PHAT, robust) or "standard" +use_gpu = "auto" # "auto" | "cpu" | "cuda" (ROCm+CUDA both via torch.cuda) +out_dir = "output" + +[video] +grid = "32x18" +bars = true # auto-crop static letterbox/pillarbox before scaling +max_lag = 0.0 # search half-window, seconds (0 = full range) +prefer_proxy = true +decode_width = 192 +``` + +Use generic local filenames (`movie_a`, `movie_b`) — never commit real +paths or copyrighted titles. + +## Workflow + +### 1. Pick a window (optional GUI) + +```bash +make picker +# .venv/bin/python picker.py --config config.toml [--a … --b …] +``` + +Shows RMS waveforms + thumbnails for `file_a`, draggable start/duration +region, audio-stream selector. `Save to config.toml` writes back +`[extract] start/duration` (preserves other sections). + +### 2. Extract analysis clips + +```bash +make extract # normal: FFV1 video + PCM WAV + gray proxy +make extract-loud # same + agate transient gating (explosions/dialogue only) +make proxy # rebuild {stem}_proxy.mkv from local masters only (no source access) +``` + +Per input, `extract.py` writes into `extracted/`: + +- `{stem}_video.mkv` — FFV1 lossless intra-only, `yuv444p`, CFR-normalized to + `target_fps` only if probed fps differs (avoids needless re-timing) +- `{stem}_audio.wav` — `pcm_f32le`, 48 kHz mono, two-pass `loudnorm` + (skipped with fallback on silent/`-inf` segments) +- `{stem}_proxy.mkv` — 192 px `gray8` FFV1 from the **local master**, for fast VST decode + +Stems: differing filenames → bare stem; identical filenames → parent dirs +are walked up until they diverge. Multi-audio files prompt for +`0:a:N` (or set `[audio] stream`); the choice is saved to `.stream_state` +(gitignored) for later `verify_sync --mode full` mapping. + +FPS note: if both inputs mismatch the target and differ from each other +(e.g. PAL 25 vs NTSC film 23.976), both are normalized to the target and a +warning suggests compensating audio tempo as well. + +### 3. Measure lag + +Audio (primary): + +```bash +make ast +# LC_NUMERIC=C PYTHONPATH=. .venv/bin/python sync_audio_test.py --config config.toml +``` + +PyQtGraph GUI: overlay + per-track waveforms + GCC-PHAT xcorr, slider / +seconds / samples lag controls, Auto peak, Swap A/B, Export JSON, stereo +split playback (A left / B+lag right), `Apply sync` (background ffmpeg with +frame progress bar) and `Verify (short)` launcher. `make ast` auto-runs +`make extract` first if the expected WAVs are missing. + +Video (cross-check / lab): + +```bash +make vst +# .venv/bin/python sync_video_test.py --config config.toml +# headless: .venv/bin/python sync_video_test.py --no-gui [--audio-lag 0.042] [--export report.json] +``` + +Reads **only** `extracted/` (proxy preferred, master fallback — never the +originals). One decode + bar-crop + grid, then one batched per-pixel FFT. +Methods: M1 global-luma, M2 3×3 grid-diff, M3 pixel-diff-sum (main), +M4 median (logo-robust), M5 PHAT, M6 scene-cut spikes, M7 raw (no diff). +GUI shows overlaid xcorr curves, per-pixel heatmap, 3×3 zone energy, frame +scrub A|B, method table with median agreement, chunk-fit drift check +(slope ≈ 0 confirms pure offset, no speed mismatch), JSON export. + +Library use: + +```python +from sync_audio import load_wav, correlate +a, sr = load_wav("extracted/movie_a_audio.wav") +b, sr = load_wav("extracted/movie_b_audio.wav") +lag_samples, peak, corr = correlate(a, b, sr=48000, method="phat", device="auto") +``` + +### 4. Mux synced output + +```bash +make sync LAG=0.042 +# .venv/bin/python apply_sync.py --config config.toml --lag 0.042 [--a … --b … --out … --progress] +``` + +`apply_sync.py` builds: `ffmpeg -i A -i B -filter_complex … -map 0:v -map 0:a +-map [dN] -c:v copy -c:a: copy -c:a: out.mkv`. + +- `lag ≥ 0`: `adelay` (silence pad at B start); `lag < 0`: `atrim+asetpts` cut. +- All A audio streams are copied; all B audio streams are re-encoded with + their original codec where possible and appended after A's streams. +- Single undetermined-language B stream is tagged `ger`; existing + language/title/forced flags are preserved. `--progress` streams + `frame=/progress=end` on stdout for the AST background worker. + +### 5. Verify + +```bash +make verify LAG=0.042 +# .venv/bin/python verify_sync.py --config config.toml --lag 0.042 --mode short +# --mode short: extracted 30 s parts (video + wav_a + wav_b) +# --mode full: synced output MKV (audio streams re-extracted from it) +``` + +mpv window (video, IPC seek/pause) + PyQtGraph overlay/dual waveforms with +playhead, timeline seek synced to mpv, A/B show toggles, per-track volume, +stereo or solo playback. Full mode resolves output stream indices as +`A: sel_a`, `B: n_a_streams + sel_b` using `.stream_state`. + +### Utility: audio language tagging + +```bash +.venv/bin/python rename_audio.py movie.mkv # dry run +.venv/bin/python rename_audio.py movie.mkv --apply [--stream 0,2] [--model base] [--segment 60] [--start 0] +``` + +Probes audio streams, runs Whisper language detection (confidence + top-5), +then rewrites MKV language/title metadata with stream copy (no re-encode). + +## Files + +| File | Purpose | +|---|---| +| `config.toml` | Single source of truth: inputs, window, audio, sync, video settings | +| `Makefile` | `install*`, `extract`, `extract-loud`, `proxy`, `picker`, `ast`, `vst`, `sync LAG=…`, `verify LAG=…`, `clean` | +| `extract.py` | Lossless clip extraction (FFV1 + PCM + proxy), loudnorm, stream picker, start validation | +| `picker.py` | Qt picker GUI: waveforms, thumbnails, start/duration → `config.toml` | +| `sync_audio.py` | Core lib: `load_wav`, `correlate` (GCC-PHAT torch/CPU), `lag_to_seconds`, `device_info` | +| `sync_audio_test.py` | AST GUI: waveforms + xcorr + tuning + playback + background apply + verify launcher | +| `sync_video_test.py` | VST lab: 7 video methods, heatmap, scrub, chunk-fit, headless `--no-gui` reports | +| `apply_sync.py` | Mux `A video + A audio + delayed B audio` with `--lag`, `--progress` protocol | +| `verify_sync.py` | mpv + dual-waveform verify player (`short` preview / `full` synced MKV) | +| `rename_audio.py` | Whisper language detect + MKV metadata rewrite (dry-run by default) | +| `waveform_cache.py` | `cache/waveforms/*.npy` disk cache for loaded WAVs (mtime-keyed) | +| `extracted/` | Generated clips (`*_video.mkv`, `*_audio.wav`, `*_proxy.mkv`); contents gitignored | +| `cache/` | Waveform + decoded-video caches; gitignored | +| `output/` | Default `apply_sync` destination (`{stem}_synced.mkv`); gitignored via `cache/`-style flow, create as needed | + +Generated/ignored: `extracted/*.{mkv,wav,mp3}`, `cache/`, `.stream_state`, +`.venv/`, `__pycache__/`, `output/` (create per `sync.out_dir`). + +## Notes + +- Content-to-content sync: container `start_time`/edit-list mux delay is + ignored; alignment is achieved by padding silence on mux. +- Analysis codecs are deliberately CPU-only (FFV1/PCM have no GPU + implementation; VAAPI/AMF are lossy delivery codecs, wrong for measurement). +- PAL-speedup cases (25 fps vs 23.976) need audio tempo compensation beyond + frame-rate normalization — `extract.py` warns when it detects this. +- `LC_NUMERIC=C` is set for Qt/ffmpeg decimal parsing in `ast`/`vst`/`verify`. diff --git a/extract.py b/extract.py index 19997f7..e4991f8 100644 --- a/extract.py +++ b/extract.py @@ -106,8 +106,8 @@ if _cfg_inputs is not None: inputs = _cfg_inputs else: inputs = [ - "/path/to/movie_b.mp4", - "/path/to/movie_a.mkv" + "/path/to/movie_a.mp4", + "/path/to/movie_b.mkv" ] INPUT_FILES = [] for input_file in inputs: diff --git a/sync_audio.py b/sync_audio.py index 45c9f57..a316e26 100644 --- a/sync_audio.py +++ b/sync_audio.py @@ -8,7 +8,7 @@ Extracted wavs are pcm_f32le 48k mono (extract.py:41), so correlation is phase-a Usage: from sync_audio import load_wav, correlate, find_lag - a, sr = load_wav("extracted/movie_b_audio.wav") + a, sr = load_wav("extracted/movie_a_audio.wav") lag_samples, peak, corr = correlate(a, b, sr=48000, method="phat", device="auto") """