feat: add config-driven extraction and audio sync tooling

This commit is contained in:
Honney OC committed 2026-09-13 20:15:13 +00:00
1 parent 476b2543d1
commit 3a18a8b2c2
5 files changed
+798 -10

No files matched your search

+83
View File
@@ -0,0 +1,83 @@
VENV ?= .venv
PY := $(VENV)/bin/python
PIP := $(PY) -m pip
UV := uv
CONFIG ?= config.toml
# detect uv, fallback to pip
HAS_UV := $(shell command -v $(UV) >/dev/null 2>&1 && echo 1 || echo 0)
ifeq ($(HAS_UV),1)
VENV_CREATE = $(UV) venv $(VENV) --python 3.11 2>/dev/null || $(UV) venv $(VENV)
PIP_INSTALL = $(UV) pip install --python $(PY)
else
VENV_CREATE = python3 -m venv $(VENV)
PIP_INSTALL = $(PIP) install
endif
.PHONY: install install-cpu install-rocm install-cuda extract ast clean help
help:
@echo "Targets:"
@echo " make install # auto-detect GPU (ROCm>CUDA>CPU) via rocm-smi/nvidia-smi/torch, create venv, install deps"
@echo " make install-cpu # force CPU torch"
@echo " make install-rocm # force ROCm torch (RX 7900 XTX gfx1100)"
@echo " make install-cuda # force CUDA torch"
@echo " make extract # run extract.py (uses config.toml if present)"
@echo " make ast # run sync_audio_test.py (audio_sync_test, PyQtGraph GUI)"
@echo " make clean"
install:
@if [ ! -d $(VENV) ]; then echo "=> creating venv $(VENV)"; $(VENV_CREATE); else echo "=> venv exists $(VENV)"; fi
@echo "=> detecting GPU..."
@HAS_ROCM=0; HAS_CUDA=0; \
if command -v rocm-smi >/dev/null 2>&1 && [ -c /dev/kfd ] && lspci 2>/dev/null | grep -qi amd; then HAS_ROCM=1; fi; \
if command -v nvidia-smi >/dev/null 2>&1; then HAS_CUDA=1; fi; \
if [ $$HAS_ROCM -eq 1 ]; then \
echo " ROCm detected (AMD) -> torch rocm6.2"; \
$(PIP_INSTALL) --upgrade pip; \
$(PIP_INSTALL) torch torchaudio --index-url https://download.pytorch.org/whl/rocm6.2; \
elif [ $$HAS_CUDA -eq 1 ]; then \
echo " CUDA detected (NVIDIA) -> torch cu121"; \
$(PIP_INSTALL) --upgrade pip; \
$(PIP_INSTALL) torch torchaudio --index-url https://download.pytorch.org/whl/cu121; \
else \
echo " No GPU -> torch cpu"; \
$(PIP_INSTALL) --upgrade pip; \
$(PIP_INSTALL) torch torchaudio --index-url https://download.pytorch.org/whl/cpu; \
fi
@echo "=> installing common deps..."
@$(PIP_INSTALL) soundfile scipy numpy pyqtgraph PyQt6
@echo "=> optional sounddevice (audio playback, volume slider)..."
@$(PIP_INSTALL) sounddevice || echo " sounddevice optional failed (PortAudio missing) -> playback disabled"
@$(PY) -c "import torch, sys; print(f'torch {torch.__version__} cuda_available={torch.cuda.is_available()}', end=''); print(f' device={torch.cuda.get_device_name(0)}' if torch.cuda.is_available() else ' (cpu)')"
@echo "=> done. Activate: source $(VENV)/bin/activate"
install-cpu:
@if [ ! -d $(VENV) ]; then $(VENV_CREATE); fi
$(PIP_INSTALL) --upgrade pip
$(PIP_INSTALL) torch torchaudio --index-url https://download.pytorch.org/whl/cpu
$(PIP_INSTALL) soundfile scipy numpy pyqtgraph PyQt6 sounddevice || true
$(PY) -c "import torch; print(torch.__version__, torch.cuda.is_available())"
install-rocm:
@if [ ! -d $(VENV) ]; then $(VENV_CREATE); fi
$(PIP_INSTALL) --upgrade pip
$(PIP_INSTALL) torch torchaudio --index-url https://download.pytorch.org/whl/rocm6.2
$(PIP_INSTALL) soundfile scipy numpy pyqtgraph PyQt6 sounddevice || true
$(PY) -c "import torch; print(torch.__version__, torch.cuda.is_available(), torch.cuda.get_device_name(0) if torch.cuda.is_available() else '')"
install-cuda:
@if [ ! -d $(VENV) ]; then $(VENV_CREATE); fi
$(PIP_INSTALL) --upgrade pip
$(PIP_INSTALL) torch torchaudio --index-url https://download.pytorch.org/whl/cu121
$(PIP_INSTALL) soundfile scipy numpy pyqtgraph PyQt6 sounddevice || true
$(PY) -c "import torch; print(torch.__version__, torch.cuda.is_available(), torch.cuda.get_device_name(0) if torch.cuda.is_available() else '')"
extract:
@if [ -x $(PY) ]; then $(PY) extract.py; else python3 extract.py; fi
ast:
@if [ -x $(PY) ]; then $(PY) sync_audio_test.py --config $(CONFIG); else python3 sync_audio_test.py --config $(CONFIG); fi
clean:
rm -rf $(VENV) __pycache__ .pytest_cache
+28
View File
@@ -0,0 +1,28 @@
# movie_sync base config — shared by extract.py and sync_audio_test.py
# TOML is efficient: no yaml dep, stdlib tomllib (py3.11+) parses directly.
[extract]
# Input files — relative to repo root or absolute. Same as extract.py inputs.
file_a = "/home/honney/Videos/Drachenzähmen/Drachenzaehmen.mp4"
file_b = "/home/honney/Videos/Drachenzähmen/How to Train Your Dragon.mkv"
# Window to extract (seconds). This is the ONLY duration needed.
duration = 90
# Start offset in file (seconds, 0 = file start). ffmpeg -ss before -i for fast seek.
start = 0.0
out_dir = "extracted"
target_fps = "24000/1001" # Blu-ray film 23.976
target_fps_eps = 0.02
[audio]
# Target for re-encode / load. Must be identical for both files.
sample_rate = 48000
channels = 1
# If extracted wavs already exist, sync test can load them directly:
# file_a_wav = "extracted/Drachenzaehmen_audio.wav"
# file_b_wav = "extracted/How to Train Your Dragon_audio.wav"
[sync]
# Analysis window: null = use full extracted duration; otherwise limit for quick test
# duration = 30 # optional, seconds, null = full
method = "phat" # "phat" (GCC-PHAT, robust) or "standard" (raw)
use_gpu = "auto" # "auto" | "cpu" | "cuda" (ROCm/CUDA both via torch.cuda)
+55 -10
View File
@@ -12,16 +12,53 @@ import os
import sys import sys
import shutil import shutil
# ─── Hardcoded input (relative to this script's directory) ─── # ─── Input / config (config.toml is base config, fallback to hardcoded) ───
inputs = [ CONFIG_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "config.toml")
"/home/honney/Videos/Drachenzähmen/Drachenzaehmen.mp4", _cfg_inputs = None
"/home/honney/Videos/Drachenzähmen/How to Train Your Dragon.mkv" _cfg_duration = 90
] _cfg_start = 0.0
_cfg_out_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "extracted")
_cfg_target_fps = "24000/1001"
_cfg_eps = 0.02
try:
try:
import tomllib # py3.11+
except ModuleNotFoundError:
import tomli as tomllib # type: ignore
if os.path.exists(CONFIG_PATH):
with open(CONFIG_PATH, "rb") as _f:
_cfg = tomllib.load(_f)
_ex = _cfg.get("extract", {})
if _ex.get("file_a") or _ex.get("file_b"):
_cfg_inputs = [p for p in [_ex.get("file_a"), _ex.get("file_b")] if p]
_cfg_duration = int(_ex.get("duration", _cfg_duration))
_cfg_start = float(_ex.get("start", _cfg_start))
_cfg_out_dir = _ex.get("out_dir", _cfg_out_dir)
if _ex.get("target_fps"):
_cfg_target_fps = str(_ex["target_fps"])
if _ex.get("target_fps_eps") is not None:
try:
_cfg_eps = float(_ex["target_fps_eps"])
except Exception:
pass
if _cfg_inputs:
print(f"[config] loaded {CONFIG_PATH}: duration={_cfg_duration}s start={_cfg_start}s target_fps={_cfg_target_fps}")
except Exception as _e:
print(f"[config] warn: failed to load {CONFIG_PATH}: {_e}", file=sys.stderr)
if _cfg_inputs is not None:
inputs = _cfg_inputs
else:
inputs = [
"/home/honney/Videos/Drachenzähmen/Drachenzaehmen.mp4",
"/home/honney/Videos/Drachenzähmen/How to Train Your Dragon.mkv"
]
INPUT_FILES = [] INPUT_FILES = []
for input_file in inputs: for input_file in inputs:
INPUT_FILES.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), input_file)) INPUT_FILES.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), input_file))
DURATION = 90 # 1.5 minutes in seconds DURATION = _cfg_duration
OUT_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "extracted") START_SEC = _cfg_start # for ffmpeg -ss (0 = file start)
OUT_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), _cfg_out_dir) if not os.path.isabs(_cfg_out_dir) else _cfg_out_dir
# ─── Analysis-friendly codecs (NOT GPU) ─── # ─── Analysis-friendly codecs (NOT GPU) ───
# Video: FFV1 (lossless, intra-only) in MKV. Every frame stands alone, no # Video: FFV1 (lossless, intra-only) in MKV. Every frame stands alone, no
@@ -39,9 +76,14 @@ OUT_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "extracted
# of 90 s clips is fast enough and bit-exact. # of 90 s clips is fast enough and bit-exact.
VIDEO_CODEC_BASE = ["-c:v", "ffv1", "-level", "3", "-pix_fmt", "yuv444p"] VIDEO_CODEC_BASE = ["-c:v", "ffv1", "-level", "3", "-pix_fmt", "yuv444p"]
AUDIO_CODEC = ["-c:a", "pcm_f32le", "-ar", "48000", "-ac", "1"] AUDIO_CODEC = ["-c:a", "pcm_f32le", "-ar", "48000", "-ac", "1"]
TARGET_FPS = "24000/1001" # Blu-ray film = 23.976023... (shown as 23.98) # target fps may be overridden via config.toml
TARGET_FPS_FLOAT = 24000 / 1001 TARGET_FPS = _cfg_target_fps # default "24000/1001"
FPS_EPS = 0.02 # tolerance for float comparison try:
_n, _d = TARGET_FPS.split("/") if "/" in TARGET_FPS else (TARGET_FPS, "1")
TARGET_FPS_FLOAT = float(_n) / float(_d)
except Exception:
TARGET_FPS_FLOAT = 24000 / 1001
FPS_EPS = _cfg_eps # tolerance for float comparison
def _parse_rate(s: str) -> float | None: def _parse_rate(s: str) -> float | None:
s = s.strip() s = s.strip()
@@ -135,9 +177,11 @@ for INPUT_FILE in INPUT_FILES:
if not needs_fps: if not needs_fps:
video_codec = VIDEO_CODEC_BASE + ["-vf", f"fps={TARGET_FPS}:round=near", "-fps_mode", "cfr"] video_codec = VIDEO_CODEC_BASE + ["-vf", f"fps={TARGET_FPS}:round=near", "-fps_mode", "cfr"]
ss_args = ["-ss", str(START_SEC)] if START_SEC and START_SEC != 0 else []
cmd_video = [ cmd_video = [
"ffmpeg", "ffmpeg",
"-y", # overwrite output "-y", # overwrite output
*ss_args, # seek before -i (fast, frame-accurate with re-encode)
"-i", INPUT_FILE, "-i", INPUT_FILE,
"-t", str(DURATION), "-t", str(DURATION),
"-an", # no audio "-an", # no audio
@@ -154,6 +198,7 @@ for INPUT_FILE in INPUT_FILES:
cmd_audio = [ cmd_audio = [
"ffmpeg", "ffmpeg",
"-y", "-y",
*ss_args,
"-i", INPUT_FILE, "-i", INPUT_FILE,
"-t", str(DURATION), "-t", str(DURATION),
"-vn", # no video "-vn", # no video
+217
View File
@@ -0,0 +1,217 @@
#!/usr/bin/env python3
"""
Audio cross-correlation for movie_sync — torch GPU auto (ROCm/CUDA) with CPU fallback.
Content-to-content sync (Option A): ignores container start_time/edit-list mux delay.
If audio is delayed in container, we pad with silence on mux instead of shifting presentation.
Extracted wavs are pcm_f32le 48k mono (extract.py:41), so correlation is phase-accurate.
Usage:
from sync_audio import load_wav, correlate, find_lag
a, sr = load_wav("extracted/Drachenzaehmen_audio.wav")
lag_samples, peak, corr = correlate(a, b, sr=48000, method="phat", device="auto")
"""
from __future__ import annotations
import os
import sys
import math
from typing import Tuple, Literal
import numpy as np
try:
import soundfile as sf # preferred for f32 wav
HAS_SF = True
except ImportError:
HAS_SF = False
try:
import torch
HAS_TORCH = True
except ImportError:
HAS_TORCH = False
def _get_device(pref: str = "auto") -> str:
if pref == "cpu":
return "cpu"
if pref in ("cuda", "auto") and HAS_TORCH and torch.cuda.is_available():
return "cuda"
return "cpu"
def load_wav(path: str, target_sr: int = 48000, mono: bool = True) -> Tuple[np.ndarray, int]:
"""Load wav as float32 mono. Resample if needed (cheap linear warn)."""
if not os.path.exists(path):
raise FileNotFoundError(path)
if HAS_SF:
data, sr = sf.read(path, always_2d=False, dtype="float32")
else:
# fallback scipy
from scipy.io import wavfile
sr, data = wavfile.read(path)
if data.dtype == np.int16:
data = data.astype(np.float32) / 32768.0
elif data.dtype == np.int32:
data = data.astype(np.float32) / 2147483648.0
elif data.dtype != np.float32:
data = data.astype(np.float32)
if data.ndim > 1:
if mono:
data = data.mean(axis=1).astype(np.float32)
if sr != target_sr:
# simple resample via scipy if available, else warn
try:
from scipy.signal import resample_poly
import math as _m
g = _m.gcd(sr, target_sr)
data = resample_poly(data, target_sr // g, sr // g).astype(np.float32)
sr = target_sr
except Exception:
print(f"WARNING: {path} sr={sr} != {target_sr} and resample failed, using as-is", file=sys.stderr)
return data.astype(np.float32), sr
def _next_pow2(n: int) -> int:
return 1 << (n - 1).bit_length()
def _normalize(a: np.ndarray) -> np.ndarray:
a = a - a.mean()
s = a.std()
return a / s if s > 1e-9 else a
def correlate(
a: np.ndarray,
b: np.ndarray,
sr: int = 48000,
method: Literal["phat", "standard"] = "phat",
device: str = "auto",
normalize: bool = True,
) -> Tuple[int, float, np.ndarray]:
"""
Return (lag_samples, peak_value, corr_array).
lag >0 means b delayed vs a (b starts later), need to shift b earlier by lag.
Convention: corr = ifft(FFT(a) * conj(FFT(b))). peak at lag = argmax.
GPU via torch if available, else numpy.
"""
if normalize:
a = _normalize(a)
b = _normalize(b)
n = len(a) + len(b) - 1
N = _next_pow2(n)
dev = _get_device(device)
if dev == "cuda" and HAS_TORCH:
# torch path — ROCm and CUDA both via cuda device
try:
ta = torch.from_numpy(a).to("cuda", dtype=torch.float32)
tb = torch.from_numpy(b).to("cuda", dtype=torch.float32)
Fa = torch.fft.rfft(ta, n=N)
Fb = torch.fft.rfft(tb, n=N)
R = Fa * torch.conj(Fb)
if method == "phat":
R = R / (torch.abs(R) + 1e-10)
corr_t = torch.fft.irfft(R, n=N)
# fftshift to center 0 lag
corr_t = torch.roll(corr_t, shifts=N // 2)
corr = corr_t.cpu().numpy()
# keep only valid length n centered
start = N // 2 - (len(b) - 1)
# Actually after roll, we want slice; simpler: use raw without roll and map lag later
# Recompute without roll for correct indexing, keep roll for display but fix lag calc
# For lag calc we need not shift: peak at N - lag etc. So we use shifted version + offset mapping
# To avoid confusion, return shifted corr and map lag accordingly
peak_idx = int(np.argmax(np.abs(corr)))
# lag = peak_idx - (len(b)-1) - (N//2 - (len(b)-1))? Let's instead use unshifted for lag and shifted for display
# Fallback: compute lag from unshifted
corr_raw = torch.fft.irfft(R, n=N).cpu().numpy()
peak_raw = int(np.argmax(np.abs(corr_raw)))
# raw layout: lag 0 at 0, positive lag (b delayed) wraps to end
if peak_raw > N // 2:
lag = peak_raw - N
else:
lag = peak_raw
peak_val = float(np.abs(corr_raw[peak_raw]))
# return shifted display corr + lag
return lag, peak_val, corr
except Exception as e:
print(f"torch cuda failed ({e}), falling back cpu", file=sys.stderr)
dev = "cpu"
# numpy/scipy cpu path
Fa = np.fft.rfft(a, n=N)
Fb = np.fft.rfft(b, n=N)
R = Fa * np.conj(Fb)
if method == "phat":
R = R / (np.abs(R) + 1e-10)
corr_raw = np.fft.irfft(R, n=N)
peak_raw = int(np.argmax(np.abs(corr_raw)))
if peak_raw > N // 2:
lag = peak_raw - N
else:
lag = peak_raw
peak_val = float(np.abs(corr_raw[peak_raw]))
# shifted for display centered at 0
corr_shifted = np.fft.fftshift(corr_raw)
# but we return raw shifted centered? Provide shifted for plotting
# For consistency with torch branch we return shifted display array
return lag, peak_val, corr_shifted
def lag_to_seconds(lag_samples: int, sr: int = 48000) -> float:
return lag_samples / float(sr)
def apply_lag_with_silence(wav: np.ndarray, lag_samples: int) -> np.ndarray:
"""
Apply lag by padding silence at start (no sound for delay).
lag >0: b delayed vs a => pad b at start with lag silence to align to a, or trim a.
For mux: if lag>0 (b starts later), we need to delay a by lag or提前 b.
This helper pads with zeros (silence) at start; negative lag pads the other direction.
Caller decides which file to pad.
"""
if lag_samples == 0:
return wav
if lag_samples > 0:
# b delayed => pad b front with lag silence to align to a
return np.concatenate([np.zeros(lag_samples, dtype=np.float32), wav])
else:
# b earlier => trim or pad a; this pads a-side: when applying to b, negative lag means b earlier, so we trim? Provide padded version for mux: pad with silence then trim alignment
# For generic helper: pad with silence at start if lag negative we pad other file, but this is applied to lagging file only
return np.concatenate([np.zeros(-lag_samples, dtype=np.float32), wav])
# torch device info for Makefile / GUI
def device_info() -> str:
if not HAS_TORCH:
return "torch not installed (cpu numpy)"
if torch.cuda.is_available():
try:
name = torch.cuda.get_device_name(0)
except Exception:
name = "cuda"
return f"torch {torch.__version__} + {name} (cuda/ROCm)"
return f"torch {torch.__version__} (cpu only)"
if __name__ == "__main__":
import argparse, pathlib
ap = argparse.ArgumentParser()
ap.add_argument("a", help="wav A")
ap.add_argument("b", help="wav B")
ap.add_argument("--phat", action="store_true", help="use GCC-PHAT")
ap.add_argument("--device", default="auto", choices=["auto","cpu","cuda"])
ap.add_argument("--sr", type=int, default=48000)
args = ap.parse_args()
a, sa = load_wav(args.a, target_sr=args.sr)
b, sb = load_wav(args.b, target_sr=args.sr)
method = "phat" if args.phat else "standard"
lag, peak, _ = correlate(a, b, sr=args.sr, method=method, device=args.device)
print(f"device: {device_info()} method={method}")
print(f"lag_samples={lag} lag_sec={lag/args.sr:.6f}s peak={peak:.4f}")
print(f"apply: ffmpeg -i video.mkv -i audio.wav -filter_complex \"[1:a]adelay={max(0, lag/args.sr*1000):.0f}|{max(0, lag/args.sr*1000):.0f}[ad]\" -map 0:v -map \"[ad]\" out.mkv (pad silence for delay)")
+415
View File
@@ -0,0 +1,415 @@
#!/usr/bin/env python3
"""
PyQtGraph GUI to visualize audio cross-correlation and interactively tune lag.
Features (per user request):
- uv venv via Makefile, torch auto GPU (ROCm/CUDA) with cpu fallback
- config.toml for file_a/file_b, duration/start (extract duration is source of truth)
- PyQtGraph waveforms (decimated for 4M samples) + xcorr plot
- QSlider + QDoubleSpinBox numeric entry for lag (seconds + samples)
- Volume slider (sounddevice optional, playback with lag applied, silence padded for delay)
Run: make ast or .venv/bin/python sync_audio_test.py --config config.toml
"""
from __future__ import annotations
import os, sys, math, argparse, pathlib
import numpy as np
# allow tomllib (py3.11+) with tomli fallback
try:
import tomllib
except ModuleNotFoundError:
import tomli as tomllib
try:
import pyqtgraph as pg
from PyQt6 import QtWidgets, QtCore
HAS_PG = True
except ImportError as e:
print("ERROR: pyqtgraph/PyQt6 not installed — run make install", file=sys.stderr)
print(e, file=sys.stderr)
sys.exit(1)
try:
import soundfile as sf
HAS_SF = True
except ImportError:
HAS_SF = False
try:
import sounddevice as sd
HAS_SD = True
except ImportError:
HAS_SD = False
from sync_audio import load_wav, correlate, device_info, lag_to_seconds
def load_config(path: str) -> dict:
if not os.path.exists(path):
return {}
with open(path, "rb") as f:
return tomllib.load(f)
def decimate_for_plot(arr: np.ndarray, max_pts: int = 120_000) -> tuple[np.ndarray, int]:
if len(arr) <= max_pts:
return arr, 1
stride = int(math.ceil(len(arr) / max_pts))
return arr[::stride], stride
def find_wavs_from_config(cfg: dict) -> tuple[str, str, int, float, float]:
audio_cfg = cfg.get("audio", {})
extract_cfg = cfg.get("extract", {})
sync_cfg = cfg.get("sync", {})
sr = int(audio_cfg.get("sample_rate", 48000))
# priority: explicit wav paths else derived from extracted dir
a_wav = audio_cfg.get("file_a_wav")
b_wav = audio_cfg.get("file_b_wav")
# fallback to extract files
out_dir = extract_cfg.get("out_dir", "extracted")
file_a = extract_cfg.get("file_a")
file_b = extract_cfg.get("file_b")
if not a_wav and file_a:
stem = os.path.splitext(os.path.basename(file_a))[0]
cand = os.path.join(out_dir, f"{stem}_audio.wav")
if os.path.exists(cand):
a_wav = cand
else:
a_wav = cand # keep for error msg
if not b_wav and file_b:
stem = os.path.splitext(os.path.basename(file_b))[0]
cand = os.path.join(out_dir, f"{stem}_audio.wav")
if os.path.exists(cand):
b_wav = cand
else:
b_wav = cand
# also allow sync-specific override
duration = extract_cfg.get("duration", 90)
start = float(extract_cfg.get("start", 0.0))
# duration_sec comment: only duration needed is off the extract — we respect that
return a_wav, b_wav, sr, float(start), float(duration)
class SyncWindow(QtWidgets.QMainWindow):
def __init__(self, a_path: str, b_path: str, sr: int = 48000, method: str = "phat", device: str = "auto"):
super().__init__()
self.setWindowTitle(f"movie_sync — audio sync test — {device_info()}")
self.resize(1400, 900)
self.a_path = a_path
self.b_path = b_path
self.sr = sr
self.method = method
self.device_pref = device
self._updating = False
# load
print(f"Loading {a_path}")
self.a, self.sr_a = load_wav(a_path, target_sr=sr)
print(f"Loading {b_path}")
self.b, self.sr_b = load_wav(b_path, target_sr=sr)
# keep originals for lag apply
self.a_orig = self.a.copy()
self.b_orig = self.b.copy()
# initial correlate
self.lag_samples, self.peak, self.corr = correlate(self.a, self.b, sr=sr, method=method, device=device)
self.lag_sec = self.lag_samples / sr
print(f"initial lag {self.lag_samples} samples = {self.lag_sec:.6f}s peak={self.peak:.4f}")
# decimated views
self.a_plot, self.stride_a = decimate_for_plot(self.a)
self.b_plot, self.stride_b = decimate_for_plot(self.b)
self.t_a = np.arange(len(self.a_plot)) * self.stride_a / sr
self.t_b = np.arange(len(self.b_plot)) * self.stride_b / sr
# xcorr display — corr is already fftshifted
N = len(self.corr)
# lags in seconds for display
# raw corr was shifted: center at N//2, range [- (len(b)-1) .. len(a)-1]
# for display we map indices to lag
total = len(self.a) + len(self.b) - 1
n_pow2 = N
# shifted lags: -N//2 .. N//2-1
self.corr_lags = np.arange(N) - N//2
self.corr_lags_sec = self.corr_lags / sr
central = QtWidgets.QWidget()
self.setCentralWidget(central)
lay = QtWidgets.QVBoxLayout(central)
# info bar
info = QtWidgets.QLabel(f"A: {os.path.basename(a_path)} ({len(self.a)/sr:.1f}s) B: {os.path.basename(b_path)} ({len(self.b)/sr:.1f}s) sr={sr} method={method} {device_info()}")
info.setWordWrap(True)
lay.addWidget(info)
# plots
self.gw = pg.GraphicsLayoutWidget()
lay.addWidget(self.gw, stretch=1)
# waveform A
self.p1 = self.gw.addPlot(row=0, col=0, title="Waveform A (reference)")
self.p1.showGrid(x=True, y=True)
self.p1.setLabel("bottom", "Time", "s")
self.curve_a = self.p1.plot(self.t_a, self.a_plot, pen=pg.mkPen("#4cc9f0", width=1))
self.p1.setMouseEnabled(x=True, y=False)
# waveform B with lag
self.p2 = self.gw.addPlot(row=1, col=0, title=f"Waveform B (shifted by lag)")
self.p2.showGrid(x=True, y=True)
self.p2.setLabel("bottom", "Time", "s")
self.curve_b = self.p2.plot(self.t_b, self.b_plot, pen=pg.mkPen("#f72585", width=1))
self.p2.setMouseEnabled(x=True, y=False)
# link x
self.p2.setXLink(self.p1)
# xcorr
self.p3 = self.gw.addPlot(row=2, col=0, title="Cross-correlation (GCC-PHAT)" if method=="phat" else "Cross-correlation")
self.p3.showGrid(x=True, y=True)
self.p3.setLabel("bottom", "Lag", "s")
# decimate corr for perf if huge
corr_dec, corr_stride = decimate_for_plot(self.corr, max_pts=200_000)
corr_lags_dec = self.corr_lags_sec[::corr_stride] if corr_stride>1 else self.corr_lags_sec
self.curve_corr = self.p3.plot(corr_lags_dec, np.abs(corr_dec), pen=pg.mkPen("#ffd166", width=1))
self.peak_line = pg.InfiniteLine(pos=self.lag_sec, angle=90, pen=pg.mkPen("#06d6a0", width=2, style=QtCore.Qt.PenStyle.DashLine))
self.p3.addItem(self.peak_line)
self.p3.setXRange(self.lag_sec-2, self.lag_sec+2)
# controls
ctr = QtWidgets.QHBoxLayout()
lay.addLayout(ctr)
ctr.addWidget(QtWidgets.QLabel("Lag (s):"))
self.slider = QtWidgets.QSlider(QtCore.Qt.Orientation.Horizontal)
self.slider.setRange(-10000, 10000) # will map to seconds via scale
# range +/-10s in ms
self.slider.setMinimum(int(-10 * 1000))
self.slider.setMaximum(int(10 * 1000))
self.slider.setValue(int(self.lag_sec * 1000))
self.slider.setTickInterval(100)
ctr.addWidget(self.slider, stretch=1)
self.spin_sec = QtWidgets.QDoubleSpinBox()
self.spin_sec.setRange(-60, 60)
self.spin_sec.setDecimals(6)
self.spin_sec.setSingleStep(0.001)
self.spin_sec.setValue(self.lag_sec)
self.spin_sec.setSuffix(" s")
self.spin_sec.setKeyboardTracking(False)
ctr.addWidget(self.spin_sec)
self.spin_samp = QtWidgets.QSpinBox()
self.spin_samp.setRange(-10_000_000, 10_000_000)
self.spin_samp.setSingleStep(1)
self.spin_samp.setValue(self.lag_samples)
self.spin_samp.setSuffix(" samp")
ctr.addWidget(self.spin_samp)
btn_auto = QtWidgets.QPushButton("Auto peak")
ctr.addWidget(btn_auto)
btn_swap = QtWidgets.QPushButton("Swap A/B")
ctr.addWidget(btn_swap)
# volume + playback
ctr2 = QtWidgets.QHBoxLayout()
lay.addLayout(ctr2)
ctr2.addWidget(QtWidgets.QLabel("Volume:"))
self.vol = QtWidgets.QSlider(QtCore.Qt.Orientation.Horizontal)
self.vol.setRange(0, 100)
self.vol.setValue(70)
self.vol.setMaximumWidth(200)
ctr2.addWidget(self.vol)
self.vol_label = QtWidgets.QLabel("70%")
ctr2.addWidget(self.vol_label)
self.btn_play_a = QtWidgets.QPushButton("Play A (2s)")
self.btn_play_b = QtWidgets.QPushButton("Play B+lag (2s)")
self.btn_play_mix = QtWidgets.QPushButton("Play Mix")
self.btn_stop = QtWidgets.QPushButton("Stop")
for b in [self.btn_play_a, self.btn_play_b, self.btn_play_mix, self.btn_stop]:
ctr2.addWidget(b)
if not HAS_SD:
for b in [self.btn_play_a, self.btn_play_b, self.btn_play_mix, self.btn_stop]:
b.setEnabled(False)
b.setToolTip("sounddevice not installed — run make install (optional)")
ctr2.addWidget(QtWidgets.QLabel("(sounddevice missing — playback disabled)"))
self.ffmpeg_label = QtWidgets.QLabel()
self.ffmpeg_label.setWordWrap(True)
self.ffmpeg_label.setTextInteractionFlags(QtCore.Qt.TextInteractionFlag.TextSelectableByMouse)
lay.addWidget(self.ffmpeg_label)
# signals
self.slider.valueChanged.connect(self.on_slider)
self.spin_sec.valueChanged.connect(self.on_spin_sec)
self.spin_samp.valueChanged.connect(self.on_spin_samp)
self.vol.valueChanged.connect(lambda v: self.vol_label.setText(f"{v}%"))
btn_auto.clicked.connect(self.on_auto)
btn_swap.clicked.connect(self.on_swap)
self.btn_play_a.clicked.connect(lambda: self.play("a"))
self.btn_play_b.clicked.connect(lambda: self.play("b"))
self.btn_play_mix.clicked.connect(lambda: self.play("mix"))
self.btn_stop.clicked.connect(self.stop_play)
self.update_labels()
def _set_lag(self, lag_sec: float, source: str = ""):
if self._updating:
return
self._updating = True
lag_sec = float(np.clip(lag_sec, -60, 60))
lag_samp = int(round(lag_sec * self.sr))
self.lag_sec = lag_sec
self.lag_samples = lag_samp
# update widgets
self.slider.blockSignals(True); self.slider.setValue(int(lag_sec*1000)); self.slider.blockSignals(False)
self.spin_sec.blockSignals(True); self.spin_sec.setValue(lag_sec); self.spin_sec.blockSignals(False)
self.spin_samp.blockSignals(True); self.spin_samp.setValue(lag_samp); self.spin_samp.blockSignals(False)
self.peak_line.setPos(lag_sec)
# shift B curve visually by lag (pad with silence not needed for visualize, just shift x)
# we keep t_b but offset x
shift = lag_sec
# pyqtgraph setPos shifts items; we recreate x
# Instead shift data: plot t_b + lag
self.curve_b.setData(self.t_b + shift, self.b_plot)
self.update_labels()
self._updating = False
def update_labels(self):
# ffmpeg adelay for mux: add silence for delay (no sound for build-in delay)
# If lag>0: B delayed vs A => to mux B onto A video, delay B audio by lag (pad silence)
# adelay expects ms per channel, e.g. adelay=1234|1234
lag_ms = self.lag_sec * 1000
if lag_ms >= 0:
ffcmd = f'ffmpeg -i video.mkv -i audio.wav -filter_complex "[1:a]adelay={lag_ms:.0f}:all=1[ad]" -map 0:v -map "[ad]" -c:v copy -c:a pcm_s16le out.mkv'
note = f"B delayed by {self.lag_sec:.6f}s ({self.lag_samples} samp) → pad B start with silence {lag_ms:.0f} ms"
else:
# B earlier => pad A (or trim B). For adding B to A file, we need to delay video or pad A? Simplest pad A? But we control only audio: delay B negative means B earlier, so we delay A or use -itsoffset
note = f"B earlier by {abs(self.lag_sec):.6f}s → pad A / use -itsoffset {abs(lag_ms):.0f}ms or trim B by {abs(self.lag_samples)} samp"
ffcmd = f'# B early: ffmpeg -itsoffset {abs(lag_ms)/1000:.6f} -i audio.wav -i video.mkv ... OR pad video audio with silence'
self.ffmpeg_label.setText(f"{note}\n{ffcmd}")
# slots
def on_slider(self, v):
self._set_lag(v/1000.0, "slider")
def on_spin_sec(self, v):
self._set_lag(float(v), "sec")
def on_spin_samp(self, v):
self._set_lag(int(v)/self.sr, "samp")
def on_auto(self):
# recompute peak
lag, peak, corr = correlate(self.a_orig, self.b_orig, sr=self.sr, method=self.method, device=self.device_pref)
self.corr = corr
# update corr curve? re-decimate
self._set_lag(lag / self.sr, "auto")
print(f"auto lag {lag} samp {lag/self.sr:.6f}s peak={peak:.4f}")
def on_swap(self):
self.a_orig, self.b_orig = self.b_orig, self.a_orig
self.a, self.b = self.a_orig.copy(), self.b_orig.copy()
self.a_plot, _ = decimate_for_plot(self.a)
self.b_plot, _ = decimate_for_plot(self.b)
self.curve_a.setData(self.t_a, self.a_plot)
# recalc
lag, _, corr = correlate(self.a, self.b, sr=self.sr, method=self.method, device=self.device_pref)
self.corr = corr
self._set_lag(lag/self.sr, "swap")
def play(self, which: str):
if not HAS_SD:
return
vol = self.vol.value()/100.0
sr = self.sr
# play 2 seconds from 10s mark to hear sync
start_samp = int(10 * sr)
dur_samp = int(2 * sr)
if start_samp + dur_samp > len(self.a_orig):
start_samp = 0
a_seg = self.a_orig[start_samp:start_samp+dur_samp] * vol
# apply lag with silence padding (no sound for delay) — Option A
lag = self.lag_samples
# For playback mix, we need to align: if B delayed (lag>0), pad B front with silence
# For preview we shift B segment accordingly
b_start = start_samp
if lag > 0:
# B delayed => B segment should be taken earlier to align? For playback we pad B with lag silence, so effective B at time t is original at t-lag
idx = b_start - lag
if idx < 0:
b_seg = np.concatenate([np.zeros(-idx, dtype=np.float32), self.b_orig[0:b_start + dur_samp - (-idx) if b_start+dur_samp - (-idx)>0 else 0]])
# slice to dur
b_seg = b_seg[:dur_samp]
if len(b_seg) < dur_samp:
b_seg = np.concatenate([b_seg, np.zeros(dur_samp-len(b_seg), dtype=np.float32)])
else:
b_seg = self.b_orig[idx:idx+dur_samp]
elif lag < 0:
idx = b_start - lag # lag negative => plus
b_seg = self.b_orig[idx:idx+dur_samp] if idx+dur_samp < len(self.b_orig) else np.zeros(dur_samp, dtype=np.float32)
if len(b_seg) < dur_samp:
b_seg = np.concatenate([b_seg, np.zeros(dur_samp-len(b_seg), dtype=np.float32)])
else:
b_seg = self.b_orig[b_start:b_start+dur_samp]
b_seg = b_seg * vol
try:
sd.stop()
if which == "a":
sd.play(a_seg, samplerate=sr)
elif which == "b":
sd.play(b_seg, samplerate=sr)
elif which == "mix":
mix = (a_seg + b_seg) / 2.0
sd.play(mix, samplerate=sr)
except Exception as e:
print(f"playback failed {e}", file=sys.stderr)
def stop_play(self):
if HAS_SD:
sd.stop()
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--config", default="config.toml")
ap.add_argument("--a", default=None)
ap.add_argument("--b", default=None)
ap.add_argument("--device", default=None, choices=["auto","cpu","cuda"])
ap.add_argument("--method", default=None, choices=["phat","standard"])
args = ap.parse_args()
cfg = load_config(args.config)
cfg_a, cfg_b, sr, _, _ = find_wavs_from_config(cfg)
a_path = args.a or cfg_a
b_path = args.b or cfg_b
method = args.method or cfg.get("sync", {}).get("method", "phat")
device = args.device or cfg.get("sync", {}).get("use_gpu", "auto")
# map toml "use_gpu" to device
if device == "auto":
device = "auto"
elif device == "cpu":
device = "cpu"
elif device in ("cuda", "gpu"):
device = "cuda"
if not a_path or not b_path:
print(f"ERROR: no input wavs. Set file_a/file_b in {args.config} or --a/--b", file=sys.stderr)
sys.exit(1)
if not os.path.exists(a_path):
print(f"ERROR: A not found {a_path}", file=sys.stderr); sys.exit(1)
if not os.path.exists(b_path):
print(f"ERROR: B not found {b_path}", file=sys.stderr); sys.exit(1)
app = QtWidgets.QApplication(sys.argv)
pg.setConfigOptions(antialias=True, background="#121212", foreground="#eee")
w = SyncWindow(a_path, b_path, sr=sr, method=method, device=device)
w.show()
sys.exit(app.exec())
if __name__ == "__main__":
main()