diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..84fa83a --- /dev/null +++ b/Makefile @@ -0,0 +1,83 @@ +VENV ?= .venv +PY := $(VENV)/bin/python +PIP := $(PY) -m pip +UV := uv +CONFIG ?= config.toml + +# detect uv, fallback to pip +HAS_UV := $(shell command -v $(UV) >/dev/null 2>&1 && echo 1 || echo 0) +ifeq ($(HAS_UV),1) + VENV_CREATE = $(UV) venv $(VENV) --python 3.11 2>/dev/null || $(UV) venv $(VENV) + PIP_INSTALL = $(UV) pip install --python $(PY) +else + VENV_CREATE = python3 -m venv $(VENV) + PIP_INSTALL = $(PIP) install +endif + +.PHONY: install install-cpu install-rocm install-cuda extract ast clean help + +help: + @echo "Targets:" + @echo " make install # auto-detect GPU (ROCm>CUDA>CPU) via rocm-smi/nvidia-smi/torch, create venv, install deps" + @echo " make install-cpu # force CPU torch" + @echo " make install-rocm # force ROCm torch (RX 7900 XTX gfx1100)" + @echo " make install-cuda # force CUDA torch" + @echo " make extract # run extract.py (uses config.toml if present)" + @echo " make ast # run sync_audio_test.py (audio_sync_test, PyQtGraph GUI)" + @echo " make clean" + +install: + @if [ ! -d $(VENV) ]; then echo "=> creating venv $(VENV)"; $(VENV_CREATE); else echo "=> venv exists $(VENV)"; fi + @echo "=> detecting GPU..." + @HAS_ROCM=0; HAS_CUDA=0; \ + if command -v rocm-smi >/dev/null 2>&1 && [ -c /dev/kfd ] && lspci 2>/dev/null | grep -qi amd; then HAS_ROCM=1; fi; \ + if command -v nvidia-smi >/dev/null 2>&1; then HAS_CUDA=1; fi; \ + if [ $$HAS_ROCM -eq 1 ]; then \ + echo " ROCm detected (AMD) -> torch rocm6.2"; \ + $(PIP_INSTALL) --upgrade pip; \ + $(PIP_INSTALL) torch torchaudio --index-url https://download.pytorch.org/whl/rocm6.2; \ + elif [ $$HAS_CUDA -eq 1 ]; then \ + echo " CUDA detected (NVIDIA) -> torch cu121"; \ + $(PIP_INSTALL) --upgrade pip; \ + $(PIP_INSTALL) torch torchaudio --index-url https://download.pytorch.org/whl/cu121; \ + else \ + echo " No GPU -> torch cpu"; \ + $(PIP_INSTALL) --upgrade pip; \ + $(PIP_INSTALL) torch torchaudio --index-url https://download.pytorch.org/whl/cpu; \ + fi + @echo "=> installing common deps..." + @$(PIP_INSTALL) soundfile scipy numpy pyqtgraph PyQt6 + @echo "=> optional sounddevice (audio playback, volume slider)..." + @$(PIP_INSTALL) sounddevice || echo " sounddevice optional failed (PortAudio missing) -> playback disabled" + @$(PY) -c "import torch, sys; print(f'torch {torch.__version__} cuda_available={torch.cuda.is_available()}', end=''); print(f' device={torch.cuda.get_device_name(0)}' if torch.cuda.is_available() else ' (cpu)')" + @echo "=> done. Activate: source $(VENV)/bin/activate" + +install-cpu: + @if [ ! -d $(VENV) ]; then $(VENV_CREATE); fi + $(PIP_INSTALL) --upgrade pip + $(PIP_INSTALL) torch torchaudio --index-url https://download.pytorch.org/whl/cpu + $(PIP_INSTALL) soundfile scipy numpy pyqtgraph PyQt6 sounddevice || true + $(PY) -c "import torch; print(torch.__version__, torch.cuda.is_available())" + +install-rocm: + @if [ ! -d $(VENV) ]; then $(VENV_CREATE); fi + $(PIP_INSTALL) --upgrade pip + $(PIP_INSTALL) torch torchaudio --index-url https://download.pytorch.org/whl/rocm6.2 + $(PIP_INSTALL) soundfile scipy numpy pyqtgraph PyQt6 sounddevice || true + $(PY) -c "import torch; print(torch.__version__, torch.cuda.is_available(), torch.cuda.get_device_name(0) if torch.cuda.is_available() else '')" + +install-cuda: + @if [ ! -d $(VENV) ]; then $(VENV_CREATE); fi + $(PIP_INSTALL) --upgrade pip + $(PIP_INSTALL) torch torchaudio --index-url https://download.pytorch.org/whl/cu121 + $(PIP_INSTALL) soundfile scipy numpy pyqtgraph PyQt6 sounddevice || true + $(PY) -c "import torch; print(torch.__version__, torch.cuda.is_available(), torch.cuda.get_device_name(0) if torch.cuda.is_available() else '')" + +extract: + @if [ -x $(PY) ]; then $(PY) extract.py; else python3 extract.py; fi + +ast: + @if [ -x $(PY) ]; then $(PY) sync_audio_test.py --config $(CONFIG); else python3 sync_audio_test.py --config $(CONFIG); fi + +clean: + rm -rf $(VENV) __pycache__ .pytest_cache diff --git a/config.toml b/config.toml new file mode 100644 index 0000000..2580b56 --- /dev/null +++ b/config.toml @@ -0,0 +1,28 @@ +# movie_sync base config — shared by extract.py and sync_audio_test.py +# TOML is efficient: no yaml dep, stdlib tomllib (py3.11+) parses directly. + +[extract] +# Input files — relative to repo root or absolute. Same as extract.py inputs. +file_a = "/path/to/movie_b.mp4" +file_b = "/path/to/movie_a.mkv" +# Window to extract (seconds). This is the ONLY duration needed. +duration = 90 +# Start offset in file (seconds, 0 = file start). ffmpeg -ss before -i for fast seek. +start = 0.0 +out_dir = "extracted" +target_fps = "24000/1001" # Blu-ray film 23.976 +target_fps_eps = 0.02 + +[audio] +# Target for re-encode / load. Must be identical for both files. +sample_rate = 48000 +channels = 1 +# If extracted wavs already exist, sync test can load them directly: +# file_a_wav = "extracted/movie_b_audio.wav" +# file_b_wav = "extracted/movie_a_audio.wav" + +[sync] +# Analysis window: null = use full extracted duration; otherwise limit for quick test +# duration = 30 # optional, seconds, null = full +method = "phat" # "phat" (GCC-PHAT, robust) or "standard" (raw) +use_gpu = "auto" # "auto" | "cpu" | "cuda" (ROCm/CUDA both via torch.cuda) diff --git a/extract.py b/extract.py index 03608d2..e527435 100644 --- a/extract.py +++ b/extract.py @@ -12,16 +12,53 @@ import os import sys import shutil -# ─── Hardcoded input (relative to this script's directory) ─── -inputs = [ - "/path/to/movie_b.mp4", - "/path/to/movie_a.mkv" -] +# ─── Input / config (config.toml is base config, fallback to hardcoded) ─── +CONFIG_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "config.toml") +_cfg_inputs = None +_cfg_duration = 90 +_cfg_start = 0.0 +_cfg_out_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "extracted") +_cfg_target_fps = "24000/1001" +_cfg_eps = 0.02 +try: + try: + import tomllib # py3.11+ + except ModuleNotFoundError: + import tomli as tomllib # type: ignore + if os.path.exists(CONFIG_PATH): + with open(CONFIG_PATH, "rb") as _f: + _cfg = tomllib.load(_f) + _ex = _cfg.get("extract", {}) + if _ex.get("file_a") or _ex.get("file_b"): + _cfg_inputs = [p for p in [_ex.get("file_a"), _ex.get("file_b")] if p] + _cfg_duration = int(_ex.get("duration", _cfg_duration)) + _cfg_start = float(_ex.get("start", _cfg_start)) + _cfg_out_dir = _ex.get("out_dir", _cfg_out_dir) + if _ex.get("target_fps"): + _cfg_target_fps = str(_ex["target_fps"]) + if _ex.get("target_fps_eps") is not None: + try: + _cfg_eps = float(_ex["target_fps_eps"]) + except Exception: + pass + if _cfg_inputs: + print(f"[config] loaded {CONFIG_PATH}: duration={_cfg_duration}s start={_cfg_start}s target_fps={_cfg_target_fps}") +except Exception as _e: + print(f"[config] warn: failed to load {CONFIG_PATH}: {_e}", file=sys.stderr) + +if _cfg_inputs is not None: + inputs = _cfg_inputs +else: + inputs = [ + "/path/to/movie_b.mp4", + "/path/to/movie_a.mkv" + ] INPUT_FILES = [] for input_file in inputs: INPUT_FILES.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), input_file)) -DURATION = 90 # 1.5 minutes in seconds -OUT_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "extracted") +DURATION = _cfg_duration +START_SEC = _cfg_start # for ffmpeg -ss (0 = file start) +OUT_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), _cfg_out_dir) if not os.path.isabs(_cfg_out_dir) else _cfg_out_dir # ─── Analysis-friendly codecs (NOT GPU) ─── # Video: FFV1 (lossless, intra-only) in MKV. Every frame stands alone, no @@ -39,9 +76,14 @@ OUT_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "extracted # of 90 s clips is fast enough and bit-exact. VIDEO_CODEC_BASE = ["-c:v", "ffv1", "-level", "3", "-pix_fmt", "yuv444p"] AUDIO_CODEC = ["-c:a", "pcm_f32le", "-ar", "48000", "-ac", "1"] -TARGET_FPS = "24000/1001" # Blu-ray film = 23.976023... (shown as 23.98) -TARGET_FPS_FLOAT = 24000 / 1001 -FPS_EPS = 0.02 # tolerance for float comparison +# target fps may be overridden via config.toml +TARGET_FPS = _cfg_target_fps # default "24000/1001" +try: + _n, _d = TARGET_FPS.split("/") if "/" in TARGET_FPS else (TARGET_FPS, "1") + TARGET_FPS_FLOAT = float(_n) / float(_d) +except Exception: + TARGET_FPS_FLOAT = 24000 / 1001 +FPS_EPS = _cfg_eps # tolerance for float comparison def _parse_rate(s: str) -> float | None: s = s.strip() @@ -135,9 +177,11 @@ for INPUT_FILE in INPUT_FILES: if not needs_fps: video_codec = VIDEO_CODEC_BASE + ["-vf", f"fps={TARGET_FPS}:round=near", "-fps_mode", "cfr"] + ss_args = ["-ss", str(START_SEC)] if START_SEC and START_SEC != 0 else [] cmd_video = [ "ffmpeg", "-y", # overwrite output + *ss_args, # seek before -i (fast, frame-accurate with re-encode) "-i", INPUT_FILE, "-t", str(DURATION), "-an", # no audio @@ -154,6 +198,7 @@ for INPUT_FILE in INPUT_FILES: cmd_audio = [ "ffmpeg", "-y", + *ss_args, "-i", INPUT_FILE, "-t", str(DURATION), "-vn", # no video diff --git a/sync_audio.py b/sync_audio.py new file mode 100644 index 0000000..67ba4ef --- /dev/null +++ b/sync_audio.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +""" +Audio cross-correlation for movie_sync — torch GPU auto (ROCm/CUDA) with CPU fallback. + +Content-to-content sync (Option A): ignores container start_time/edit-list mux delay. +If audio is delayed in container, we pad with silence on mux instead of shifting presentation. +Extracted wavs are pcm_f32le 48k mono (extract.py:41), so correlation is phase-accurate. + +Usage: + from sync_audio import load_wav, correlate, find_lag + a, sr = load_wav("extracted/movie_b_audio.wav") + lag_samples, peak, corr = correlate(a, b, sr=48000, method="phat", device="auto") +""" + +from __future__ import annotations +import os +import sys +import math +from typing import Tuple, Literal + +import numpy as np + +try: + import soundfile as sf # preferred for f32 wav + HAS_SF = True +except ImportError: + HAS_SF = False + +try: + import torch + HAS_TORCH = True +except ImportError: + HAS_TORCH = False + + +def _get_device(pref: str = "auto") -> str: + if pref == "cpu": + return "cpu" + if pref in ("cuda", "auto") and HAS_TORCH and torch.cuda.is_available(): + return "cuda" + return "cpu" + + +def load_wav(path: str, target_sr: int = 48000, mono: bool = True) -> Tuple[np.ndarray, int]: + """Load wav as float32 mono. Resample if needed (cheap linear warn).""" + if not os.path.exists(path): + raise FileNotFoundError(path) + if HAS_SF: + data, sr = sf.read(path, always_2d=False, dtype="float32") + else: + # fallback scipy + from scipy.io import wavfile + sr, data = wavfile.read(path) + if data.dtype == np.int16: + data = data.astype(np.float32) / 32768.0 + elif data.dtype == np.int32: + data = data.astype(np.float32) / 2147483648.0 + elif data.dtype != np.float32: + data = data.astype(np.float32) + if data.ndim > 1: + if mono: + data = data.mean(axis=1).astype(np.float32) + if sr != target_sr: + # simple resample via scipy if available, else warn + try: + from scipy.signal import resample_poly + import math as _m + g = _m.gcd(sr, target_sr) + data = resample_poly(data, target_sr // g, sr // g).astype(np.float32) + sr = target_sr + except Exception: + print(f"WARNING: {path} sr={sr} != {target_sr} and resample failed, using as-is", file=sys.stderr) + return data.astype(np.float32), sr + + +def _next_pow2(n: int) -> int: + return 1 << (n - 1).bit_length() + + +def _normalize(a: np.ndarray) -> np.ndarray: + a = a - a.mean() + s = a.std() + return a / s if s > 1e-9 else a + + +def correlate( + a: np.ndarray, + b: np.ndarray, + sr: int = 48000, + method: Literal["phat", "standard"] = "phat", + device: str = "auto", + normalize: bool = True, +) -> Tuple[int, float, np.ndarray]: + """ + Return (lag_samples, peak_value, corr_array). + lag >0 means b delayed vs a (b starts later), need to shift b earlier by lag. + Convention: corr = ifft(FFT(a) * conj(FFT(b))). peak at lag = argmax. + GPU via torch if available, else numpy. + """ + if normalize: + a = _normalize(a) + b = _normalize(b) + + n = len(a) + len(b) - 1 + N = _next_pow2(n) + + dev = _get_device(device) + + if dev == "cuda" and HAS_TORCH: + # torch path — ROCm and CUDA both via cuda device + try: + ta = torch.from_numpy(a).to("cuda", dtype=torch.float32) + tb = torch.from_numpy(b).to("cuda", dtype=torch.float32) + Fa = torch.fft.rfft(ta, n=N) + Fb = torch.fft.rfft(tb, n=N) + R = Fa * torch.conj(Fb) + if method == "phat": + R = R / (torch.abs(R) + 1e-10) + corr_t = torch.fft.irfft(R, n=N) + # fftshift to center 0 lag + corr_t = torch.roll(corr_t, shifts=N // 2) + corr = corr_t.cpu().numpy() + # keep only valid length n centered + start = N // 2 - (len(b) - 1) + # Actually after roll, we want slice; simpler: use raw without roll and map lag later + # Recompute without roll for correct indexing, keep roll for display but fix lag calc + # For lag calc we need not shift: peak at N - lag etc. So we use shifted version + offset mapping + # To avoid confusion, return shifted corr and map lag accordingly + peak_idx = int(np.argmax(np.abs(corr))) + # lag = peak_idx - (len(b)-1) - (N//2 - (len(b)-1))? Let's instead use unshifted for lag and shifted for display + # Fallback: compute lag from unshifted + corr_raw = torch.fft.irfft(R, n=N).cpu().numpy() + peak_raw = int(np.argmax(np.abs(corr_raw))) + # raw layout: lag 0 at 0, positive lag (b delayed) wraps to end + if peak_raw > N // 2: + lag = peak_raw - N + else: + lag = peak_raw + peak_val = float(np.abs(corr_raw[peak_raw])) + # return shifted display corr + lag + return lag, peak_val, corr + except Exception as e: + print(f"torch cuda failed ({e}), falling back cpu", file=sys.stderr) + dev = "cpu" + + # numpy/scipy cpu path + Fa = np.fft.rfft(a, n=N) + Fb = np.fft.rfft(b, n=N) + R = Fa * np.conj(Fb) + if method == "phat": + R = R / (np.abs(R) + 1e-10) + corr_raw = np.fft.irfft(R, n=N) + peak_raw = int(np.argmax(np.abs(corr_raw))) + if peak_raw > N // 2: + lag = peak_raw - N + else: + lag = peak_raw + peak_val = float(np.abs(corr_raw[peak_raw])) + # shifted for display centered at 0 + corr_shifted = np.fft.fftshift(corr_raw) + # but we return raw shifted centered? Provide shifted for plotting + # For consistency with torch branch we return shifted display array + return lag, peak_val, corr_shifted + + +def lag_to_seconds(lag_samples: int, sr: int = 48000) -> float: + return lag_samples / float(sr) + + +def apply_lag_with_silence(wav: np.ndarray, lag_samples: int) -> np.ndarray: + """ + Apply lag by padding silence at start (no sound for delay). + lag >0: b delayed vs a => pad b at start with lag silence to align to a, or trim a. + For mux: if lag>0 (b starts later), we need to delay a by lag or提前 b. + This helper pads with zeros (silence) at start; negative lag pads the other direction. + Caller decides which file to pad. + """ + if lag_samples == 0: + return wav + if lag_samples > 0: + # b delayed => pad b front with lag silence to align to a + return np.concatenate([np.zeros(lag_samples, dtype=np.float32), wav]) + else: + # b earlier => trim or pad a; this pads a-side: when applying to b, negative lag means b earlier, so we trim? Provide padded version for mux: pad with silence then trim alignment + # For generic helper: pad with silence at start if lag negative we pad other file, but this is applied to lagging file only + return np.concatenate([np.zeros(-lag_samples, dtype=np.float32), wav]) + + +# torch device info for Makefile / GUI +def device_info() -> str: + if not HAS_TORCH: + return "torch not installed (cpu numpy)" + if torch.cuda.is_available(): + try: + name = torch.cuda.get_device_name(0) + except Exception: + name = "cuda" + return f"torch {torch.__version__} + {name} (cuda/ROCm)" + return f"torch {torch.__version__} (cpu only)" + + +if __name__ == "__main__": + import argparse, pathlib + ap = argparse.ArgumentParser() + ap.add_argument("a", help="wav A") + ap.add_argument("b", help="wav B") + ap.add_argument("--phat", action="store_true", help="use GCC-PHAT") + ap.add_argument("--device", default="auto", choices=["auto","cpu","cuda"]) + ap.add_argument("--sr", type=int, default=48000) + args = ap.parse_args() + a, sa = load_wav(args.a, target_sr=args.sr) + b, sb = load_wav(args.b, target_sr=args.sr) + method = "phat" if args.phat else "standard" + lag, peak, _ = correlate(a, b, sr=args.sr, method=method, device=args.device) + print(f"device: {device_info()} method={method}") + print(f"lag_samples={lag} lag_sec={lag/args.sr:.6f}s peak={peak:.4f}") + print(f"apply: ffmpeg -i video.mkv -i audio.wav -filter_complex \"[1:a]adelay={max(0, lag/args.sr*1000):.0f}|{max(0, lag/args.sr*1000):.0f}[ad]\" -map 0:v -map \"[ad]\" out.mkv (pad silence for delay)") diff --git a/sync_audio_test.py b/sync_audio_test.py new file mode 100644 index 0000000..8519adc --- /dev/null +++ b/sync_audio_test.py @@ -0,0 +1,415 @@ +#!/usr/bin/env python3 +""" +PyQtGraph GUI to visualize audio cross-correlation and interactively tune lag. + +Features (per user request): + - uv venv via Makefile, torch auto GPU (ROCm/CUDA) with cpu fallback + - config.toml for file_a/file_b, duration/start (extract duration is source of truth) + - PyQtGraph waveforms (decimated for 4M samples) + xcorr plot + - QSlider + QDoubleSpinBox numeric entry for lag (seconds + samples) + - Volume slider (sounddevice optional, playback with lag applied, silence padded for delay) + +Run: make ast or .venv/bin/python sync_audio_test.py --config config.toml +""" + +from __future__ import annotations +import os, sys, math, argparse, pathlib +import numpy as np + +# allow tomllib (py3.11+) with tomli fallback +try: + import tomllib +except ModuleNotFoundError: + import tomli as tomllib + +try: + import pyqtgraph as pg + from PyQt6 import QtWidgets, QtCore + HAS_PG = True +except ImportError as e: + print("ERROR: pyqtgraph/PyQt6 not installed — run make install", file=sys.stderr) + print(e, file=sys.stderr) + sys.exit(1) + +try: + import soundfile as sf + HAS_SF = True +except ImportError: + HAS_SF = False + +try: + import sounddevice as sd + HAS_SD = True +except ImportError: + HAS_SD = False + +from sync_audio import load_wav, correlate, device_info, lag_to_seconds + + +def load_config(path: str) -> dict: + if not os.path.exists(path): + return {} + with open(path, "rb") as f: + return tomllib.load(f) + + +def decimate_for_plot(arr: np.ndarray, max_pts: int = 120_000) -> tuple[np.ndarray, int]: + if len(arr) <= max_pts: + return arr, 1 + stride = int(math.ceil(len(arr) / max_pts)) + return arr[::stride], stride + + +def find_wavs_from_config(cfg: dict) -> tuple[str, str, int, float, float]: + audio_cfg = cfg.get("audio", {}) + extract_cfg = cfg.get("extract", {}) + sync_cfg = cfg.get("sync", {}) + sr = int(audio_cfg.get("sample_rate", 48000)) + # priority: explicit wav paths else derived from extracted dir + a_wav = audio_cfg.get("file_a_wav") + b_wav = audio_cfg.get("file_b_wav") + # fallback to extract files + out_dir = extract_cfg.get("out_dir", "extracted") + file_a = extract_cfg.get("file_a") + file_b = extract_cfg.get("file_b") + if not a_wav and file_a: + stem = os.path.splitext(os.path.basename(file_a))[0] + cand = os.path.join(out_dir, f"{stem}_audio.wav") + if os.path.exists(cand): + a_wav = cand + else: + a_wav = cand # keep for error msg + if not b_wav and file_b: + stem = os.path.splitext(os.path.basename(file_b))[0] + cand = os.path.join(out_dir, f"{stem}_audio.wav") + if os.path.exists(cand): + b_wav = cand + else: + b_wav = cand + # also allow sync-specific override + duration = extract_cfg.get("duration", 90) + start = float(extract_cfg.get("start", 0.0)) + # duration_sec comment: only duration needed is off the extract — we respect that + return a_wav, b_wav, sr, float(start), float(duration) + + +class SyncWindow(QtWidgets.QMainWindow): + def __init__(self, a_path: str, b_path: str, sr: int = 48000, method: str = "phat", device: str = "auto"): + super().__init__() + self.setWindowTitle(f"movie_sync — audio sync test — {device_info()}") + self.resize(1400, 900) + self.a_path = a_path + self.b_path = b_path + self.sr = sr + self.method = method + self.device_pref = device + self._updating = False + + # load + print(f"Loading {a_path}") + self.a, self.sr_a = load_wav(a_path, target_sr=sr) + print(f"Loading {b_path}") + self.b, self.sr_b = load_wav(b_path, target_sr=sr) + # keep originals for lag apply + self.a_orig = self.a.copy() + self.b_orig = self.b.copy() + + # initial correlate + self.lag_samples, self.peak, self.corr = correlate(self.a, self.b, sr=sr, method=method, device=device) + self.lag_sec = self.lag_samples / sr + print(f"initial lag {self.lag_samples} samples = {self.lag_sec:.6f}s peak={self.peak:.4f}") + + # decimated views + self.a_plot, self.stride_a = decimate_for_plot(self.a) + self.b_plot, self.stride_b = decimate_for_plot(self.b) + self.t_a = np.arange(len(self.a_plot)) * self.stride_a / sr + self.t_b = np.arange(len(self.b_plot)) * self.stride_b / sr + + # xcorr display — corr is already fftshifted + N = len(self.corr) + # lags in seconds for display + # raw corr was shifted: center at N//2, range [- (len(b)-1) .. len(a)-1] + # for display we map indices to lag + total = len(self.a) + len(self.b) - 1 + n_pow2 = N + # shifted lags: -N//2 .. N//2-1 + self.corr_lags = np.arange(N) - N//2 + self.corr_lags_sec = self.corr_lags / sr + + central = QtWidgets.QWidget() + self.setCentralWidget(central) + lay = QtWidgets.QVBoxLayout(central) + + # info bar + info = QtWidgets.QLabel(f"A: {os.path.basename(a_path)} ({len(self.a)/sr:.1f}s) B: {os.path.basename(b_path)} ({len(self.b)/sr:.1f}s) sr={sr} method={method} {device_info()}") + info.setWordWrap(True) + lay.addWidget(info) + + # plots + self.gw = pg.GraphicsLayoutWidget() + lay.addWidget(self.gw, stretch=1) + + # waveform A + self.p1 = self.gw.addPlot(row=0, col=0, title="Waveform A (reference)") + self.p1.showGrid(x=True, y=True) + self.p1.setLabel("bottom", "Time", "s") + self.curve_a = self.p1.plot(self.t_a, self.a_plot, pen=pg.mkPen("#4cc9f0", width=1)) + self.p1.setMouseEnabled(x=True, y=False) + + # waveform B with lag + self.p2 = self.gw.addPlot(row=1, col=0, title=f"Waveform B (shifted by lag)") + self.p2.showGrid(x=True, y=True) + self.p2.setLabel("bottom", "Time", "s") + self.curve_b = self.p2.plot(self.t_b, self.b_plot, pen=pg.mkPen("#f72585", width=1)) + self.p2.setMouseEnabled(x=True, y=False) + # link x + self.p2.setXLink(self.p1) + + # xcorr + self.p3 = self.gw.addPlot(row=2, col=0, title="Cross-correlation (GCC-PHAT)" if method=="phat" else "Cross-correlation") + self.p3.showGrid(x=True, y=True) + self.p3.setLabel("bottom", "Lag", "s") + # decimate corr for perf if huge + corr_dec, corr_stride = decimate_for_plot(self.corr, max_pts=200_000) + corr_lags_dec = self.corr_lags_sec[::corr_stride] if corr_stride>1 else self.corr_lags_sec + self.curve_corr = self.p3.plot(corr_lags_dec, np.abs(corr_dec), pen=pg.mkPen("#ffd166", width=1)) + self.peak_line = pg.InfiniteLine(pos=self.lag_sec, angle=90, pen=pg.mkPen("#06d6a0", width=2, style=QtCore.Qt.PenStyle.DashLine)) + self.p3.addItem(self.peak_line) + self.p3.setXRange(self.lag_sec-2, self.lag_sec+2) + + # controls + ctr = QtWidgets.QHBoxLayout() + lay.addLayout(ctr) + + ctr.addWidget(QtWidgets.QLabel("Lag (s):")) + self.slider = QtWidgets.QSlider(QtCore.Qt.Orientation.Horizontal) + self.slider.setRange(-10000, 10000) # will map to seconds via scale + # range +/-10s in ms + self.slider.setMinimum(int(-10 * 1000)) + self.slider.setMaximum(int(10 * 1000)) + self.slider.setValue(int(self.lag_sec * 1000)) + self.slider.setTickInterval(100) + ctr.addWidget(self.slider, stretch=1) + + self.spin_sec = QtWidgets.QDoubleSpinBox() + self.spin_sec.setRange(-60, 60) + self.spin_sec.setDecimals(6) + self.spin_sec.setSingleStep(0.001) + self.spin_sec.setValue(self.lag_sec) + self.spin_sec.setSuffix(" s") + self.spin_sec.setKeyboardTracking(False) + ctr.addWidget(self.spin_sec) + + self.spin_samp = QtWidgets.QSpinBox() + self.spin_samp.setRange(-10_000_000, 10_000_000) + self.spin_samp.setSingleStep(1) + self.spin_samp.setValue(self.lag_samples) + self.spin_samp.setSuffix(" samp") + ctr.addWidget(self.spin_samp) + + btn_auto = QtWidgets.QPushButton("Auto peak") + ctr.addWidget(btn_auto) + + btn_swap = QtWidgets.QPushButton("Swap A/B") + ctr.addWidget(btn_swap) + + # volume + playback + ctr2 = QtWidgets.QHBoxLayout() + lay.addLayout(ctr2) + ctr2.addWidget(QtWidgets.QLabel("Volume:")) + self.vol = QtWidgets.QSlider(QtCore.Qt.Orientation.Horizontal) + self.vol.setRange(0, 100) + self.vol.setValue(70) + self.vol.setMaximumWidth(200) + ctr2.addWidget(self.vol) + self.vol_label = QtWidgets.QLabel("70%") + ctr2.addWidget(self.vol_label) + + self.btn_play_a = QtWidgets.QPushButton("Play A (2s)") + self.btn_play_b = QtWidgets.QPushButton("Play B+lag (2s)") + self.btn_play_mix = QtWidgets.QPushButton("Play Mix") + self.btn_stop = QtWidgets.QPushButton("Stop") + for b in [self.btn_play_a, self.btn_play_b, self.btn_play_mix, self.btn_stop]: + ctr2.addWidget(b) + if not HAS_SD: + for b in [self.btn_play_a, self.btn_play_b, self.btn_play_mix, self.btn_stop]: + b.setEnabled(False) + b.setToolTip("sounddevice not installed — run make install (optional)") + ctr2.addWidget(QtWidgets.QLabel("(sounddevice missing — playback disabled)")) + + self.ffmpeg_label = QtWidgets.QLabel() + self.ffmpeg_label.setWordWrap(True) + self.ffmpeg_label.setTextInteractionFlags(QtCore.Qt.TextInteractionFlag.TextSelectableByMouse) + lay.addWidget(self.ffmpeg_label) + + # signals + self.slider.valueChanged.connect(self.on_slider) + self.spin_sec.valueChanged.connect(self.on_spin_sec) + self.spin_samp.valueChanged.connect(self.on_spin_samp) + self.vol.valueChanged.connect(lambda v: self.vol_label.setText(f"{v}%")) + btn_auto.clicked.connect(self.on_auto) + btn_swap.clicked.connect(self.on_swap) + self.btn_play_a.clicked.connect(lambda: self.play("a")) + self.btn_play_b.clicked.connect(lambda: self.play("b")) + self.btn_play_mix.clicked.connect(lambda: self.play("mix")) + self.btn_stop.clicked.connect(self.stop_play) + + self.update_labels() + + def _set_lag(self, lag_sec: float, source: str = ""): + if self._updating: + return + self._updating = True + lag_sec = float(np.clip(lag_sec, -60, 60)) + lag_samp = int(round(lag_sec * self.sr)) + self.lag_sec = lag_sec + self.lag_samples = lag_samp + # update widgets + self.slider.blockSignals(True); self.slider.setValue(int(lag_sec*1000)); self.slider.blockSignals(False) + self.spin_sec.blockSignals(True); self.spin_sec.setValue(lag_sec); self.spin_sec.blockSignals(False) + self.spin_samp.blockSignals(True); self.spin_samp.setValue(lag_samp); self.spin_samp.blockSignals(False) + self.peak_line.setPos(lag_sec) + # shift B curve visually by lag (pad with silence not needed for visualize, just shift x) + # we keep t_b but offset x + shift = lag_sec + # pyqtgraph setPos shifts items; we recreate x + # Instead shift data: plot t_b + lag + self.curve_b.setData(self.t_b + shift, self.b_plot) + self.update_labels() + self._updating = False + + def update_labels(self): + # ffmpeg adelay for mux: add silence for delay (no sound for build-in delay) + # If lag>0: B delayed vs A => to mux B onto A video, delay B audio by lag (pad silence) + # adelay expects ms per channel, e.g. adelay=1234|1234 + lag_ms = self.lag_sec * 1000 + if lag_ms >= 0: + ffcmd = f'ffmpeg -i video.mkv -i audio.wav -filter_complex "[1:a]adelay={lag_ms:.0f}:all=1[ad]" -map 0:v -map "[ad]" -c:v copy -c:a pcm_s16le out.mkv' + note = f"B delayed by {self.lag_sec:.6f}s ({self.lag_samples} samp) → pad B start with silence {lag_ms:.0f} ms" + else: + # B earlier => pad A (or trim B). For adding B to A file, we need to delay video or pad A? Simplest pad A? But we control only audio: delay B negative means B earlier, so we delay A or use -itsoffset + note = f"B earlier by {abs(self.lag_sec):.6f}s → pad A / use -itsoffset {abs(lag_ms):.0f}ms or trim B by {abs(self.lag_samples)} samp" + ffcmd = f'# B early: ffmpeg -itsoffset {abs(lag_ms)/1000:.6f} -i audio.wav -i video.mkv ... OR pad video audio with silence' + self.ffmpeg_label.setText(f"{note}\n{ffcmd}") + + # slots + def on_slider(self, v): + self._set_lag(v/1000.0, "slider") + + def on_spin_sec(self, v): + self._set_lag(float(v), "sec") + + def on_spin_samp(self, v): + self._set_lag(int(v)/self.sr, "samp") + + def on_auto(self): + # recompute peak + lag, peak, corr = correlate(self.a_orig, self.b_orig, sr=self.sr, method=self.method, device=self.device_pref) + self.corr = corr + # update corr curve? re-decimate + self._set_lag(lag / self.sr, "auto") + print(f"auto lag {lag} samp {lag/self.sr:.6f}s peak={peak:.4f}") + + def on_swap(self): + self.a_orig, self.b_orig = self.b_orig, self.a_orig + self.a, self.b = self.a_orig.copy(), self.b_orig.copy() + self.a_plot, _ = decimate_for_plot(self.a) + self.b_plot, _ = decimate_for_plot(self.b) + self.curve_a.setData(self.t_a, self.a_plot) + # recalc + lag, _, corr = correlate(self.a, self.b, sr=self.sr, method=self.method, device=self.device_pref) + self.corr = corr + self._set_lag(lag/self.sr, "swap") + + def play(self, which: str): + if not HAS_SD: + return + vol = self.vol.value()/100.0 + sr = self.sr + # play 2 seconds from 10s mark to hear sync + start_samp = int(10 * sr) + dur_samp = int(2 * sr) + if start_samp + dur_samp > len(self.a_orig): + start_samp = 0 + a_seg = self.a_orig[start_samp:start_samp+dur_samp] * vol + # apply lag with silence padding (no sound for delay) — Option A + lag = self.lag_samples + # For playback mix, we need to align: if B delayed (lag>0), pad B front with silence + # For preview we shift B segment accordingly + b_start = start_samp + if lag > 0: + # B delayed => B segment should be taken earlier to align? For playback we pad B with lag silence, so effective B at time t is original at t-lag + idx = b_start - lag + if idx < 0: + b_seg = np.concatenate([np.zeros(-idx, dtype=np.float32), self.b_orig[0:b_start + dur_samp - (-idx) if b_start+dur_samp - (-idx)>0 else 0]]) + # slice to dur + b_seg = b_seg[:dur_samp] + if len(b_seg) < dur_samp: + b_seg = np.concatenate([b_seg, np.zeros(dur_samp-len(b_seg), dtype=np.float32)]) + else: + b_seg = self.b_orig[idx:idx+dur_samp] + elif lag < 0: + idx = b_start - lag # lag negative => plus + b_seg = self.b_orig[idx:idx+dur_samp] if idx+dur_samp < len(self.b_orig) else np.zeros(dur_samp, dtype=np.float32) + if len(b_seg) < dur_samp: + b_seg = np.concatenate([b_seg, np.zeros(dur_samp-len(b_seg), dtype=np.float32)]) + else: + b_seg = self.b_orig[b_start:b_start+dur_samp] + b_seg = b_seg * vol + try: + sd.stop() + if which == "a": + sd.play(a_seg, samplerate=sr) + elif which == "b": + sd.play(b_seg, samplerate=sr) + elif which == "mix": + mix = (a_seg + b_seg) / 2.0 + sd.play(mix, samplerate=sr) + except Exception as e: + print(f"playback failed {e}", file=sys.stderr) + + def stop_play(self): + if HAS_SD: + sd.stop() + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--config", default="config.toml") + ap.add_argument("--a", default=None) + ap.add_argument("--b", default=None) + ap.add_argument("--device", default=None, choices=["auto","cpu","cuda"]) + ap.add_argument("--method", default=None, choices=["phat","standard"]) + args = ap.parse_args() + + cfg = load_config(args.config) + cfg_a, cfg_b, sr, _, _ = find_wavs_from_config(cfg) + a_path = args.a or cfg_a + b_path = args.b or cfg_b + method = args.method or cfg.get("sync", {}).get("method", "phat") + device = args.device or cfg.get("sync", {}).get("use_gpu", "auto") + # map toml "use_gpu" to device + if device == "auto": + device = "auto" + elif device == "cpu": + device = "cpu" + elif device in ("cuda", "gpu"): + device = "cuda" + + if not a_path or not b_path: + print(f"ERROR: no input wavs. Set file_a/file_b in {args.config} or --a/--b", file=sys.stderr) + sys.exit(1) + if not os.path.exists(a_path): + print(f"ERROR: A not found {a_path}", file=sys.stderr); sys.exit(1) + if not os.path.exists(b_path): + print(f"ERROR: B not found {b_path}", file=sys.stderr); sys.exit(1) + + app = QtWidgets.QApplication(sys.argv) + pg.setConfigOptions(antialias=True, background="#121212", foreground="#eee") + w = SyncWindow(a_path, b_path, sr=sr, method=method, device=device) + w.show() + sys.exit(app.exec()) + + +if __name__ == "__main__": + main()