feat: add config-driven extraction and audio sync tooling
This commit is contained in:
1 parent
6229bea7f4
commit
7d5fed2f17
5 files changed
+796
-8
No files matched your search
@@ -0,0 +1,83 @@
|
||||
VENV ?= .venv
|
||||
PY := $(VENV)/bin/python
|
||||
PIP := $(PY) -m pip
|
||||
UV := uv
|
||||
CONFIG ?= config.toml
|
||||
|
||||
# detect uv, fallback to pip
|
||||
HAS_UV := $(shell command -v $(UV) >/dev/null 2>&1 && echo 1 || echo 0)
|
||||
ifeq ($(HAS_UV),1)
|
||||
VENV_CREATE = $(UV) venv $(VENV) --python 3.11 2>/dev/null || $(UV) venv $(VENV)
|
||||
PIP_INSTALL = $(UV) pip install --python $(PY)
|
||||
else
|
||||
VENV_CREATE = python3 -m venv $(VENV)
|
||||
PIP_INSTALL = $(PIP) install
|
||||
endif
|
||||
|
||||
.PHONY: install install-cpu install-rocm install-cuda extract ast clean help
|
||||
|
||||
help:
|
||||
@echo "Targets:"
|
||||
@echo " make install # auto-detect GPU (ROCm>CUDA>CPU) via rocm-smi/nvidia-smi/torch, create venv, install deps"
|
||||
@echo " make install-cpu # force CPU torch"
|
||||
@echo " make install-rocm # force ROCm torch (RX 7900 XTX gfx1100)"
|
||||
@echo " make install-cuda # force CUDA torch"
|
||||
@echo " make extract # run extract.py (uses config.toml if present)"
|
||||
@echo " make ast # run sync_audio_test.py (audio_sync_test, PyQtGraph GUI)"
|
||||
@echo " make clean"
|
||||
|
||||
install:
|
||||
@if [ ! -d $(VENV) ]; then echo "=> creating venv $(VENV)"; $(VENV_CREATE); else echo "=> venv exists $(VENV)"; fi
|
||||
@echo "=> detecting GPU..."
|
||||
@HAS_ROCM=0; HAS_CUDA=0; \
|
||||
if command -v rocm-smi >/dev/null 2>&1 && [ -c /dev/kfd ] && lspci 2>/dev/null | grep -qi amd; then HAS_ROCM=1; fi; \
|
||||
if command -v nvidia-smi >/dev/null 2>&1; then HAS_CUDA=1; fi; \
|
||||
if [ $$HAS_ROCM -eq 1 ]; then \
|
||||
echo " ROCm detected (AMD) -> torch rocm6.2"; \
|
||||
$(PIP_INSTALL) --upgrade pip; \
|
||||
$(PIP_INSTALL) torch torchaudio --index-url https://download.pytorch.org/whl/rocm6.2; \
|
||||
elif [ $$HAS_CUDA -eq 1 ]; then \
|
||||
echo " CUDA detected (NVIDIA) -> torch cu121"; \
|
||||
$(PIP_INSTALL) --upgrade pip; \
|
||||
$(PIP_INSTALL) torch torchaudio --index-url https://download.pytorch.org/whl/cu121; \
|
||||
else \
|
||||
echo " No GPU -> torch cpu"; \
|
||||
$(PIP_INSTALL) --upgrade pip; \
|
||||
$(PIP_INSTALL) torch torchaudio --index-url https://download.pytorch.org/whl/cpu; \
|
||||
fi
|
||||
@echo "=> installing common deps..."
|
||||
@$(PIP_INSTALL) soundfile scipy numpy pyqtgraph PyQt6
|
||||
@echo "=> optional sounddevice (audio playback, volume slider)..."
|
||||
@$(PIP_INSTALL) sounddevice || echo " sounddevice optional failed (PortAudio missing) -> playback disabled"
|
||||
@$(PY) -c "import torch, sys; print(f'torch {torch.__version__} cuda_available={torch.cuda.is_available()}', end=''); print(f' device={torch.cuda.get_device_name(0)}' if torch.cuda.is_available() else ' (cpu)')"
|
||||
@echo "=> done. Activate: source $(VENV)/bin/activate"
|
||||
|
||||
install-cpu:
|
||||
@if [ ! -d $(VENV) ]; then $(VENV_CREATE); fi
|
||||
$(PIP_INSTALL) --upgrade pip
|
||||
$(PIP_INSTALL) torch torchaudio --index-url https://download.pytorch.org/whl/cpu
|
||||
$(PIP_INSTALL) soundfile scipy numpy pyqtgraph PyQt6 sounddevice || true
|
||||
$(PY) -c "import torch; print(torch.__version__, torch.cuda.is_available())"
|
||||
|
||||
install-rocm:
|
||||
@if [ ! -d $(VENV) ]; then $(VENV_CREATE); fi
|
||||
$(PIP_INSTALL) --upgrade pip
|
||||
$(PIP_INSTALL) torch torchaudio --index-url https://download.pytorch.org/whl/rocm6.2
|
||||
$(PIP_INSTALL) soundfile scipy numpy pyqtgraph PyQt6 sounddevice || true
|
||||
$(PY) -c "import torch; print(torch.__version__, torch.cuda.is_available(), torch.cuda.get_device_name(0) if torch.cuda.is_available() else '')"
|
||||
|
||||
install-cuda:
|
||||
@if [ ! -d $(VENV) ]; then $(VENV_CREATE); fi
|
||||
$(PIP_INSTALL) --upgrade pip
|
||||
$(PIP_INSTALL) torch torchaudio --index-url https://download.pytorch.org/whl/cu121
|
||||
$(PIP_INSTALL) soundfile scipy numpy pyqtgraph PyQt6 sounddevice || true
|
||||
$(PY) -c "import torch; print(torch.__version__, torch.cuda.is_available(), torch.cuda.get_device_name(0) if torch.cuda.is_available() else '')"
|
||||
|
||||
extract:
|
||||
@if [ -x $(PY) ]; then $(PY) extract.py; else python3 extract.py; fi
|
||||
|
||||
ast:
|
||||
@if [ -x $(PY) ]; then $(PY) sync_audio_test.py --config $(CONFIG); else python3 sync_audio_test.py --config $(CONFIG); fi
|
||||
|
||||
clean:
|
||||
rm -rf $(VENV) __pycache__ .pytest_cache
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
# movie_sync base config — shared by extract.py and sync_audio_test.py
|
||||
# TOML is efficient: no yaml dep, stdlib tomllib (py3.11+) parses directly.
|
||||
|
||||
[extract]
|
||||
# Input files — relative to repo root or absolute. Same as extract.py inputs.
|
||||
file_a = "/path/to/movie_b.mp4"
|
||||
file_b = "/path/to/movie_a.mkv"
|
||||
# Window to extract (seconds). This is the ONLY duration needed.
|
||||
duration = 90
|
||||
# Start offset in file (seconds, 0 = file start). ffmpeg -ss before -i for fast seek.
|
||||
start = 0.0
|
||||
out_dir = "extracted"
|
||||
target_fps = "24000/1001" # Blu-ray film 23.976
|
||||
target_fps_eps = 0.02
|
||||
|
||||
[audio]
|
||||
# Target for re-encode / load. Must be identical for both files.
|
||||
sample_rate = 48000
|
||||
channels = 1
|
||||
# If extracted wavs already exist, sync test can load them directly:
|
||||
# file_a_wav = "extracted/movie_b_audio.wav"
|
||||
# file_b_wav = "extracted/movie_a_audio.wav"
|
||||
|
||||
[sync]
|
||||
# Analysis window: null = use full extracted duration; otherwise limit for quick test
|
||||
# duration = 30 # optional, seconds, null = full
|
||||
method = "phat" # "phat" (GCC-PHAT, robust) or "standard" (raw)
|
||||
use_gpu = "auto" # "auto" | "cpu" | "cuda" (ROCm/CUDA both via torch.cuda)
|
||||
+53
-8
@@ -12,16 +12,53 @@ import os
|
||||
import sys
|
||||
import shutil
|
||||
|
||||
# ─── Hardcoded input (relative to this script's directory) ───
|
||||
inputs = [
|
||||
# ─── Input / config (config.toml is base config, fallback to hardcoded) ───
|
||||
CONFIG_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "config.toml")
|
||||
_cfg_inputs = None
|
||||
_cfg_duration = 90
|
||||
_cfg_start = 0.0
|
||||
_cfg_out_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "extracted")
|
||||
_cfg_target_fps = "24000/1001"
|
||||
_cfg_eps = 0.02
|
||||
try:
|
||||
try:
|
||||
import tomllib # py3.11+
|
||||
except ModuleNotFoundError:
|
||||
import tomli as tomllib # type: ignore
|
||||
if os.path.exists(CONFIG_PATH):
|
||||
with open(CONFIG_PATH, "rb") as _f:
|
||||
_cfg = tomllib.load(_f)
|
||||
_ex = _cfg.get("extract", {})
|
||||
if _ex.get("file_a") or _ex.get("file_b"):
|
||||
_cfg_inputs = [p for p in [_ex.get("file_a"), _ex.get("file_b")] if p]
|
||||
_cfg_duration = int(_ex.get("duration", _cfg_duration))
|
||||
_cfg_start = float(_ex.get("start", _cfg_start))
|
||||
_cfg_out_dir = _ex.get("out_dir", _cfg_out_dir)
|
||||
if _ex.get("target_fps"):
|
||||
_cfg_target_fps = str(_ex["target_fps"])
|
||||
if _ex.get("target_fps_eps") is not None:
|
||||
try:
|
||||
_cfg_eps = float(_ex["target_fps_eps"])
|
||||
except Exception:
|
||||
pass
|
||||
if _cfg_inputs:
|
||||
print(f"[config] loaded {CONFIG_PATH}: duration={_cfg_duration}s start={_cfg_start}s target_fps={_cfg_target_fps}")
|
||||
except Exception as _e:
|
||||
print(f"[config] warn: failed to load {CONFIG_PATH}: {_e}", file=sys.stderr)
|
||||
|
||||
if _cfg_inputs is not None:
|
||||
inputs = _cfg_inputs
|
||||
else:
|
||||
inputs = [
|
||||
"/path/to/movie_b.mp4",
|
||||
"/path/to/movie_a.mkv"
|
||||
]
|
||||
]
|
||||
INPUT_FILES = []
|
||||
for input_file in inputs:
|
||||
INPUT_FILES.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), input_file))
|
||||
DURATION = 90 # 1.5 minutes in seconds
|
||||
OUT_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "extracted")
|
||||
DURATION = _cfg_duration
|
||||
START_SEC = _cfg_start # for ffmpeg -ss (0 = file start)
|
||||
OUT_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), _cfg_out_dir) if not os.path.isabs(_cfg_out_dir) else _cfg_out_dir
|
||||
|
||||
# ─── Analysis-friendly codecs (NOT GPU) ───
|
||||
# Video: FFV1 (lossless, intra-only) in MKV. Every frame stands alone, no
|
||||
@@ -39,9 +76,14 @@ OUT_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "extracted
|
||||
# of 90 s clips is fast enough and bit-exact.
|
||||
VIDEO_CODEC_BASE = ["-c:v", "ffv1", "-level", "3", "-pix_fmt", "yuv444p"]
|
||||
AUDIO_CODEC = ["-c:a", "pcm_f32le", "-ar", "48000", "-ac", "1"]
|
||||
TARGET_FPS = "24000/1001" # Blu-ray film = 23.976023... (shown as 23.98)
|
||||
TARGET_FPS_FLOAT = 24000 / 1001
|
||||
FPS_EPS = 0.02 # tolerance for float comparison
|
||||
# target fps may be overridden via config.toml
|
||||
TARGET_FPS = _cfg_target_fps # default "24000/1001"
|
||||
try:
|
||||
_n, _d = TARGET_FPS.split("/") if "/" in TARGET_FPS else (TARGET_FPS, "1")
|
||||
TARGET_FPS_FLOAT = float(_n) / float(_d)
|
||||
except Exception:
|
||||
TARGET_FPS_FLOAT = 24000 / 1001
|
||||
FPS_EPS = _cfg_eps # tolerance for float comparison
|
||||
|
||||
def _parse_rate(s: str) -> float | None:
|
||||
s = s.strip()
|
||||
@@ -135,9 +177,11 @@ for INPUT_FILE in INPUT_FILES:
|
||||
if not needs_fps:
|
||||
video_codec = VIDEO_CODEC_BASE + ["-vf", f"fps={TARGET_FPS}:round=near", "-fps_mode", "cfr"]
|
||||
|
||||
ss_args = ["-ss", str(START_SEC)] if START_SEC and START_SEC != 0 else []
|
||||
cmd_video = [
|
||||
"ffmpeg",
|
||||
"-y", # overwrite output
|
||||
*ss_args, # seek before -i (fast, frame-accurate with re-encode)
|
||||
"-i", INPUT_FILE,
|
||||
"-t", str(DURATION),
|
||||
"-an", # no audio
|
||||
@@ -154,6 +198,7 @@ for INPUT_FILE in INPUT_FILES:
|
||||
cmd_audio = [
|
||||
"ffmpeg",
|
||||
"-y",
|
||||
*ss_args,
|
||||
"-i", INPUT_FILE,
|
||||
"-t", str(DURATION),
|
||||
"-vn", # no video
|
||||
|
||||
+217
@@ -0,0 +1,217 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Audio cross-correlation for movie_sync — torch GPU auto (ROCm/CUDA) with CPU fallback.
|
||||
|
||||
Content-to-content sync (Option A): ignores container start_time/edit-list mux delay.
|
||||
If audio is delayed in container, we pad with silence on mux instead of shifting presentation.
|
||||
Extracted wavs are pcm_f32le 48k mono (extract.py:41), so correlation is phase-accurate.
|
||||
|
||||
Usage:
|
||||
from sync_audio import load_wav, correlate, find_lag
|
||||
a, sr = load_wav("extracted/movie_b_audio.wav")
|
||||
lag_samples, peak, corr = correlate(a, b, sr=48000, method="phat", device="auto")
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
import os
|
||||
import sys
|
||||
import math
|
||||
from typing import Tuple, Literal
|
||||
|
||||
import numpy as np
|
||||
|
||||
try:
|
||||
import soundfile as sf # preferred for f32 wav
|
||||
HAS_SF = True
|
||||
except ImportError:
|
||||
HAS_SF = False
|
||||
|
||||
try:
|
||||
import torch
|
||||
HAS_TORCH = True
|
||||
except ImportError:
|
||||
HAS_TORCH = False
|
||||
|
||||
|
||||
def _get_device(pref: str = "auto") -> str:
|
||||
if pref == "cpu":
|
||||
return "cpu"
|
||||
if pref in ("cuda", "auto") and HAS_TORCH and torch.cuda.is_available():
|
||||
return "cuda"
|
||||
return "cpu"
|
||||
|
||||
|
||||
def load_wav(path: str, target_sr: int = 48000, mono: bool = True) -> Tuple[np.ndarray, int]:
|
||||
"""Load wav as float32 mono. Resample if needed (cheap linear warn)."""
|
||||
if not os.path.exists(path):
|
||||
raise FileNotFoundError(path)
|
||||
if HAS_SF:
|
||||
data, sr = sf.read(path, always_2d=False, dtype="float32")
|
||||
else:
|
||||
# fallback scipy
|
||||
from scipy.io import wavfile
|
||||
sr, data = wavfile.read(path)
|
||||
if data.dtype == np.int16:
|
||||
data = data.astype(np.float32) / 32768.0
|
||||
elif data.dtype == np.int32:
|
||||
data = data.astype(np.float32) / 2147483648.0
|
||||
elif data.dtype != np.float32:
|
||||
data = data.astype(np.float32)
|
||||
if data.ndim > 1:
|
||||
if mono:
|
||||
data = data.mean(axis=1).astype(np.float32)
|
||||
if sr != target_sr:
|
||||
# simple resample via scipy if available, else warn
|
||||
try:
|
||||
from scipy.signal import resample_poly
|
||||
import math as _m
|
||||
g = _m.gcd(sr, target_sr)
|
||||
data = resample_poly(data, target_sr // g, sr // g).astype(np.float32)
|
||||
sr = target_sr
|
||||
except Exception:
|
||||
print(f"WARNING: {path} sr={sr} != {target_sr} and resample failed, using as-is", file=sys.stderr)
|
||||
return data.astype(np.float32), sr
|
||||
|
||||
|
||||
def _next_pow2(n: int) -> int:
|
||||
return 1 << (n - 1).bit_length()
|
||||
|
||||
|
||||
def _normalize(a: np.ndarray) -> np.ndarray:
|
||||
a = a - a.mean()
|
||||
s = a.std()
|
||||
return a / s if s > 1e-9 else a
|
||||
|
||||
|
||||
def correlate(
|
||||
a: np.ndarray,
|
||||
b: np.ndarray,
|
||||
sr: int = 48000,
|
||||
method: Literal["phat", "standard"] = "phat",
|
||||
device: str = "auto",
|
||||
normalize: bool = True,
|
||||
) -> Tuple[int, float, np.ndarray]:
|
||||
"""
|
||||
Return (lag_samples, peak_value, corr_array).
|
||||
lag >0 means b delayed vs a (b starts later), need to shift b earlier by lag.
|
||||
Convention: corr = ifft(FFT(a) * conj(FFT(b))). peak at lag = argmax.
|
||||
GPU via torch if available, else numpy.
|
||||
"""
|
||||
if normalize:
|
||||
a = _normalize(a)
|
||||
b = _normalize(b)
|
||||
|
||||
n = len(a) + len(b) - 1
|
||||
N = _next_pow2(n)
|
||||
|
||||
dev = _get_device(device)
|
||||
|
||||
if dev == "cuda" and HAS_TORCH:
|
||||
# torch path — ROCm and CUDA both via cuda device
|
||||
try:
|
||||
ta = torch.from_numpy(a).to("cuda", dtype=torch.float32)
|
||||
tb = torch.from_numpy(b).to("cuda", dtype=torch.float32)
|
||||
Fa = torch.fft.rfft(ta, n=N)
|
||||
Fb = torch.fft.rfft(tb, n=N)
|
||||
R = Fa * torch.conj(Fb)
|
||||
if method == "phat":
|
||||
R = R / (torch.abs(R) + 1e-10)
|
||||
corr_t = torch.fft.irfft(R, n=N)
|
||||
# fftshift to center 0 lag
|
||||
corr_t = torch.roll(corr_t, shifts=N // 2)
|
||||
corr = corr_t.cpu().numpy()
|
||||
# keep only valid length n centered
|
||||
start = N // 2 - (len(b) - 1)
|
||||
# Actually after roll, we want slice; simpler: use raw without roll and map lag later
|
||||
# Recompute without roll for correct indexing, keep roll for display but fix lag calc
|
||||
# For lag calc we need not shift: peak at N - lag etc. So we use shifted version + offset mapping
|
||||
# To avoid confusion, return shifted corr and map lag accordingly
|
||||
peak_idx = int(np.argmax(np.abs(corr)))
|
||||
# lag = peak_idx - (len(b)-1) - (N//2 - (len(b)-1))? Let's instead use unshifted for lag and shifted for display
|
||||
# Fallback: compute lag from unshifted
|
||||
corr_raw = torch.fft.irfft(R, n=N).cpu().numpy()
|
||||
peak_raw = int(np.argmax(np.abs(corr_raw)))
|
||||
# raw layout: lag 0 at 0, positive lag (b delayed) wraps to end
|
||||
if peak_raw > N // 2:
|
||||
lag = peak_raw - N
|
||||
else:
|
||||
lag = peak_raw
|
||||
peak_val = float(np.abs(corr_raw[peak_raw]))
|
||||
# return shifted display corr + lag
|
||||
return lag, peak_val, corr
|
||||
except Exception as e:
|
||||
print(f"torch cuda failed ({e}), falling back cpu", file=sys.stderr)
|
||||
dev = "cpu"
|
||||
|
||||
# numpy/scipy cpu path
|
||||
Fa = np.fft.rfft(a, n=N)
|
||||
Fb = np.fft.rfft(b, n=N)
|
||||
R = Fa * np.conj(Fb)
|
||||
if method == "phat":
|
||||
R = R / (np.abs(R) + 1e-10)
|
||||
corr_raw = np.fft.irfft(R, n=N)
|
||||
peak_raw = int(np.argmax(np.abs(corr_raw)))
|
||||
if peak_raw > N // 2:
|
||||
lag = peak_raw - N
|
||||
else:
|
||||
lag = peak_raw
|
||||
peak_val = float(np.abs(corr_raw[peak_raw]))
|
||||
# shifted for display centered at 0
|
||||
corr_shifted = np.fft.fftshift(corr_raw)
|
||||
# but we return raw shifted centered? Provide shifted for plotting
|
||||
# For consistency with torch branch we return shifted display array
|
||||
return lag, peak_val, corr_shifted
|
||||
|
||||
|
||||
def lag_to_seconds(lag_samples: int, sr: int = 48000) -> float:
|
||||
return lag_samples / float(sr)
|
||||
|
||||
|
||||
def apply_lag_with_silence(wav: np.ndarray, lag_samples: int) -> np.ndarray:
|
||||
"""
|
||||
Apply lag by padding silence at start (no sound for delay).
|
||||
lag >0: b delayed vs a => pad b at start with lag silence to align to a, or trim a.
|
||||
For mux: if lag>0 (b starts later), we need to delay a by lag or提前 b.
|
||||
This helper pads with zeros (silence) at start; negative lag pads the other direction.
|
||||
Caller decides which file to pad.
|
||||
"""
|
||||
if lag_samples == 0:
|
||||
return wav
|
||||
if lag_samples > 0:
|
||||
# b delayed => pad b front with lag silence to align to a
|
||||
return np.concatenate([np.zeros(lag_samples, dtype=np.float32), wav])
|
||||
else:
|
||||
# b earlier => trim or pad a; this pads a-side: when applying to b, negative lag means b earlier, so we trim? Provide padded version for mux: pad with silence then trim alignment
|
||||
# For generic helper: pad with silence at start if lag negative we pad other file, but this is applied to lagging file only
|
||||
return np.concatenate([np.zeros(-lag_samples, dtype=np.float32), wav])
|
||||
|
||||
|
||||
# torch device info for Makefile / GUI
|
||||
def device_info() -> str:
|
||||
if not HAS_TORCH:
|
||||
return "torch not installed (cpu numpy)"
|
||||
if torch.cuda.is_available():
|
||||
try:
|
||||
name = torch.cuda.get_device_name(0)
|
||||
except Exception:
|
||||
name = "cuda"
|
||||
return f"torch {torch.__version__} + {name} (cuda/ROCm)"
|
||||
return f"torch {torch.__version__} (cpu only)"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse, pathlib
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("a", help="wav A")
|
||||
ap.add_argument("b", help="wav B")
|
||||
ap.add_argument("--phat", action="store_true", help="use GCC-PHAT")
|
||||
ap.add_argument("--device", default="auto", choices=["auto","cpu","cuda"])
|
||||
ap.add_argument("--sr", type=int, default=48000)
|
||||
args = ap.parse_args()
|
||||
a, sa = load_wav(args.a, target_sr=args.sr)
|
||||
b, sb = load_wav(args.b, target_sr=args.sr)
|
||||
method = "phat" if args.phat else "standard"
|
||||
lag, peak, _ = correlate(a, b, sr=args.sr, method=method, device=args.device)
|
||||
print(f"device: {device_info()} method={method}")
|
||||
print(f"lag_samples={lag} lag_sec={lag/args.sr:.6f}s peak={peak:.4f}")
|
||||
print(f"apply: ffmpeg -i video.mkv -i audio.wav -filter_complex \"[1:a]adelay={max(0, lag/args.sr*1000):.0f}|{max(0, lag/args.sr*1000):.0f}[ad]\" -map 0:v -map \"[ad]\" out.mkv (pad silence for delay)")
|
||||
@@ -0,0 +1,415 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
PyQtGraph GUI to visualize audio cross-correlation and interactively tune lag.
|
||||
|
||||
Features (per user request):
|
||||
- uv venv via Makefile, torch auto GPU (ROCm/CUDA) with cpu fallback
|
||||
- config.toml for file_a/file_b, duration/start (extract duration is source of truth)
|
||||
- PyQtGraph waveforms (decimated for 4M samples) + xcorr plot
|
||||
- QSlider + QDoubleSpinBox numeric entry for lag (seconds + samples)
|
||||
- Volume slider (sounddevice optional, playback with lag applied, silence padded for delay)
|
||||
|
||||
Run: make ast or .venv/bin/python sync_audio_test.py --config config.toml
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
import os, sys, math, argparse, pathlib
|
||||
import numpy as np
|
||||
|
||||
# allow tomllib (py3.11+) with tomli fallback
|
||||
try:
|
||||
import tomllib
|
||||
except ModuleNotFoundError:
|
||||
import tomli as tomllib
|
||||
|
||||
try:
|
||||
import pyqtgraph as pg
|
||||
from PyQt6 import QtWidgets, QtCore
|
||||
HAS_PG = True
|
||||
except ImportError as e:
|
||||
print("ERROR: pyqtgraph/PyQt6 not installed — run make install", file=sys.stderr)
|
||||
print(e, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
import soundfile as sf
|
||||
HAS_SF = True
|
||||
except ImportError:
|
||||
HAS_SF = False
|
||||
|
||||
try:
|
||||
import sounddevice as sd
|
||||
HAS_SD = True
|
||||
except ImportError:
|
||||
HAS_SD = False
|
||||
|
||||
from sync_audio import load_wav, correlate, device_info, lag_to_seconds
|
||||
|
||||
|
||||
def load_config(path: str) -> dict:
|
||||
if not os.path.exists(path):
|
||||
return {}
|
||||
with open(path, "rb") as f:
|
||||
return tomllib.load(f)
|
||||
|
||||
|
||||
def decimate_for_plot(arr: np.ndarray, max_pts: int = 120_000) -> tuple[np.ndarray, int]:
|
||||
if len(arr) <= max_pts:
|
||||
return arr, 1
|
||||
stride = int(math.ceil(len(arr) / max_pts))
|
||||
return arr[::stride], stride
|
||||
|
||||
|
||||
def find_wavs_from_config(cfg: dict) -> tuple[str, str, int, float, float]:
|
||||
audio_cfg = cfg.get("audio", {})
|
||||
extract_cfg = cfg.get("extract", {})
|
||||
sync_cfg = cfg.get("sync", {})
|
||||
sr = int(audio_cfg.get("sample_rate", 48000))
|
||||
# priority: explicit wav paths else derived from extracted dir
|
||||
a_wav = audio_cfg.get("file_a_wav")
|
||||
b_wav = audio_cfg.get("file_b_wav")
|
||||
# fallback to extract files
|
||||
out_dir = extract_cfg.get("out_dir", "extracted")
|
||||
file_a = extract_cfg.get("file_a")
|
||||
file_b = extract_cfg.get("file_b")
|
||||
if not a_wav and file_a:
|
||||
stem = os.path.splitext(os.path.basename(file_a))[0]
|
||||
cand = os.path.join(out_dir, f"{stem}_audio.wav")
|
||||
if os.path.exists(cand):
|
||||
a_wav = cand
|
||||
else:
|
||||
a_wav = cand # keep for error msg
|
||||
if not b_wav and file_b:
|
||||
stem = os.path.splitext(os.path.basename(file_b))[0]
|
||||
cand = os.path.join(out_dir, f"{stem}_audio.wav")
|
||||
if os.path.exists(cand):
|
||||
b_wav = cand
|
||||
else:
|
||||
b_wav = cand
|
||||
# also allow sync-specific override
|
||||
duration = extract_cfg.get("duration", 90)
|
||||
start = float(extract_cfg.get("start", 0.0))
|
||||
# duration_sec comment: only duration needed is off the extract — we respect that
|
||||
return a_wav, b_wav, sr, float(start), float(duration)
|
||||
|
||||
|
||||
class SyncWindow(QtWidgets.QMainWindow):
|
||||
def __init__(self, a_path: str, b_path: str, sr: int = 48000, method: str = "phat", device: str = "auto"):
|
||||
super().__init__()
|
||||
self.setWindowTitle(f"movie_sync — audio sync test — {device_info()}")
|
||||
self.resize(1400, 900)
|
||||
self.a_path = a_path
|
||||
self.b_path = b_path
|
||||
self.sr = sr
|
||||
self.method = method
|
||||
self.device_pref = device
|
||||
self._updating = False
|
||||
|
||||
# load
|
||||
print(f"Loading {a_path}")
|
||||
self.a, self.sr_a = load_wav(a_path, target_sr=sr)
|
||||
print(f"Loading {b_path}")
|
||||
self.b, self.sr_b = load_wav(b_path, target_sr=sr)
|
||||
# keep originals for lag apply
|
||||
self.a_orig = self.a.copy()
|
||||
self.b_orig = self.b.copy()
|
||||
|
||||
# initial correlate
|
||||
self.lag_samples, self.peak, self.corr = correlate(self.a, self.b, sr=sr, method=method, device=device)
|
||||
self.lag_sec = self.lag_samples / sr
|
||||
print(f"initial lag {self.lag_samples} samples = {self.lag_sec:.6f}s peak={self.peak:.4f}")
|
||||
|
||||
# decimated views
|
||||
self.a_plot, self.stride_a = decimate_for_plot(self.a)
|
||||
self.b_plot, self.stride_b = decimate_for_plot(self.b)
|
||||
self.t_a = np.arange(len(self.a_plot)) * self.stride_a / sr
|
||||
self.t_b = np.arange(len(self.b_plot)) * self.stride_b / sr
|
||||
|
||||
# xcorr display — corr is already fftshifted
|
||||
N = len(self.corr)
|
||||
# lags in seconds for display
|
||||
# raw corr was shifted: center at N//2, range [- (len(b)-1) .. len(a)-1]
|
||||
# for display we map indices to lag
|
||||
total = len(self.a) + len(self.b) - 1
|
||||
n_pow2 = N
|
||||
# shifted lags: -N//2 .. N//2-1
|
||||
self.corr_lags = np.arange(N) - N//2
|
||||
self.corr_lags_sec = self.corr_lags / sr
|
||||
|
||||
central = QtWidgets.QWidget()
|
||||
self.setCentralWidget(central)
|
||||
lay = QtWidgets.QVBoxLayout(central)
|
||||
|
||||
# info bar
|
||||
info = QtWidgets.QLabel(f"A: {os.path.basename(a_path)} ({len(self.a)/sr:.1f}s) B: {os.path.basename(b_path)} ({len(self.b)/sr:.1f}s) sr={sr} method={method} {device_info()}")
|
||||
info.setWordWrap(True)
|
||||
lay.addWidget(info)
|
||||
|
||||
# plots
|
||||
self.gw = pg.GraphicsLayoutWidget()
|
||||
lay.addWidget(self.gw, stretch=1)
|
||||
|
||||
# waveform A
|
||||
self.p1 = self.gw.addPlot(row=0, col=0, title="Waveform A (reference)")
|
||||
self.p1.showGrid(x=True, y=True)
|
||||
self.p1.setLabel("bottom", "Time", "s")
|
||||
self.curve_a = self.p1.plot(self.t_a, self.a_plot, pen=pg.mkPen("#4cc9f0", width=1))
|
||||
self.p1.setMouseEnabled(x=True, y=False)
|
||||
|
||||
# waveform B with lag
|
||||
self.p2 = self.gw.addPlot(row=1, col=0, title=f"Waveform B (shifted by lag)")
|
||||
self.p2.showGrid(x=True, y=True)
|
||||
self.p2.setLabel("bottom", "Time", "s")
|
||||
self.curve_b = self.p2.plot(self.t_b, self.b_plot, pen=pg.mkPen("#f72585", width=1))
|
||||
self.p2.setMouseEnabled(x=True, y=False)
|
||||
# link x
|
||||
self.p2.setXLink(self.p1)
|
||||
|
||||
# xcorr
|
||||
self.p3 = self.gw.addPlot(row=2, col=0, title="Cross-correlation (GCC-PHAT)" if method=="phat" else "Cross-correlation")
|
||||
self.p3.showGrid(x=True, y=True)
|
||||
self.p3.setLabel("bottom", "Lag", "s")
|
||||
# decimate corr for perf if huge
|
||||
corr_dec, corr_stride = decimate_for_plot(self.corr, max_pts=200_000)
|
||||
corr_lags_dec = self.corr_lags_sec[::corr_stride] if corr_stride>1 else self.corr_lags_sec
|
||||
self.curve_corr = self.p3.plot(corr_lags_dec, np.abs(corr_dec), pen=pg.mkPen("#ffd166", width=1))
|
||||
self.peak_line = pg.InfiniteLine(pos=self.lag_sec, angle=90, pen=pg.mkPen("#06d6a0", width=2, style=QtCore.Qt.PenStyle.DashLine))
|
||||
self.p3.addItem(self.peak_line)
|
||||
self.p3.setXRange(self.lag_sec-2, self.lag_sec+2)
|
||||
|
||||
# controls
|
||||
ctr = QtWidgets.QHBoxLayout()
|
||||
lay.addLayout(ctr)
|
||||
|
||||
ctr.addWidget(QtWidgets.QLabel("Lag (s):"))
|
||||
self.slider = QtWidgets.QSlider(QtCore.Qt.Orientation.Horizontal)
|
||||
self.slider.setRange(-10000, 10000) # will map to seconds via scale
|
||||
# range +/-10s in ms
|
||||
self.slider.setMinimum(int(-10 * 1000))
|
||||
self.slider.setMaximum(int(10 * 1000))
|
||||
self.slider.setValue(int(self.lag_sec * 1000))
|
||||
self.slider.setTickInterval(100)
|
||||
ctr.addWidget(self.slider, stretch=1)
|
||||
|
||||
self.spin_sec = QtWidgets.QDoubleSpinBox()
|
||||
self.spin_sec.setRange(-60, 60)
|
||||
self.spin_sec.setDecimals(6)
|
||||
self.spin_sec.setSingleStep(0.001)
|
||||
self.spin_sec.setValue(self.lag_sec)
|
||||
self.spin_sec.setSuffix(" s")
|
||||
self.spin_sec.setKeyboardTracking(False)
|
||||
ctr.addWidget(self.spin_sec)
|
||||
|
||||
self.spin_samp = QtWidgets.QSpinBox()
|
||||
self.spin_samp.setRange(-10_000_000, 10_000_000)
|
||||
self.spin_samp.setSingleStep(1)
|
||||
self.spin_samp.setValue(self.lag_samples)
|
||||
self.spin_samp.setSuffix(" samp")
|
||||
ctr.addWidget(self.spin_samp)
|
||||
|
||||
btn_auto = QtWidgets.QPushButton("Auto peak")
|
||||
ctr.addWidget(btn_auto)
|
||||
|
||||
btn_swap = QtWidgets.QPushButton("Swap A/B")
|
||||
ctr.addWidget(btn_swap)
|
||||
|
||||
# volume + playback
|
||||
ctr2 = QtWidgets.QHBoxLayout()
|
||||
lay.addLayout(ctr2)
|
||||
ctr2.addWidget(QtWidgets.QLabel("Volume:"))
|
||||
self.vol = QtWidgets.QSlider(QtCore.Qt.Orientation.Horizontal)
|
||||
self.vol.setRange(0, 100)
|
||||
self.vol.setValue(70)
|
||||
self.vol.setMaximumWidth(200)
|
||||
ctr2.addWidget(self.vol)
|
||||
self.vol_label = QtWidgets.QLabel("70%")
|
||||
ctr2.addWidget(self.vol_label)
|
||||
|
||||
self.btn_play_a = QtWidgets.QPushButton("Play A (2s)")
|
||||
self.btn_play_b = QtWidgets.QPushButton("Play B+lag (2s)")
|
||||
self.btn_play_mix = QtWidgets.QPushButton("Play Mix")
|
||||
self.btn_stop = QtWidgets.QPushButton("Stop")
|
||||
for b in [self.btn_play_a, self.btn_play_b, self.btn_play_mix, self.btn_stop]:
|
||||
ctr2.addWidget(b)
|
||||
if not HAS_SD:
|
||||
for b in [self.btn_play_a, self.btn_play_b, self.btn_play_mix, self.btn_stop]:
|
||||
b.setEnabled(False)
|
||||
b.setToolTip("sounddevice not installed — run make install (optional)")
|
||||
ctr2.addWidget(QtWidgets.QLabel("(sounddevice missing — playback disabled)"))
|
||||
|
||||
self.ffmpeg_label = QtWidgets.QLabel()
|
||||
self.ffmpeg_label.setWordWrap(True)
|
||||
self.ffmpeg_label.setTextInteractionFlags(QtCore.Qt.TextInteractionFlag.TextSelectableByMouse)
|
||||
lay.addWidget(self.ffmpeg_label)
|
||||
|
||||
# signals
|
||||
self.slider.valueChanged.connect(self.on_slider)
|
||||
self.spin_sec.valueChanged.connect(self.on_spin_sec)
|
||||
self.spin_samp.valueChanged.connect(self.on_spin_samp)
|
||||
self.vol.valueChanged.connect(lambda v: self.vol_label.setText(f"{v}%"))
|
||||
btn_auto.clicked.connect(self.on_auto)
|
||||
btn_swap.clicked.connect(self.on_swap)
|
||||
self.btn_play_a.clicked.connect(lambda: self.play("a"))
|
||||
self.btn_play_b.clicked.connect(lambda: self.play("b"))
|
||||
self.btn_play_mix.clicked.connect(lambda: self.play("mix"))
|
||||
self.btn_stop.clicked.connect(self.stop_play)
|
||||
|
||||
self.update_labels()
|
||||
|
||||
def _set_lag(self, lag_sec: float, source: str = ""):
|
||||
if self._updating:
|
||||
return
|
||||
self._updating = True
|
||||
lag_sec = float(np.clip(lag_sec, -60, 60))
|
||||
lag_samp = int(round(lag_sec * self.sr))
|
||||
self.lag_sec = lag_sec
|
||||
self.lag_samples = lag_samp
|
||||
# update widgets
|
||||
self.slider.blockSignals(True); self.slider.setValue(int(lag_sec*1000)); self.slider.blockSignals(False)
|
||||
self.spin_sec.blockSignals(True); self.spin_sec.setValue(lag_sec); self.spin_sec.blockSignals(False)
|
||||
self.spin_samp.blockSignals(True); self.spin_samp.setValue(lag_samp); self.spin_samp.blockSignals(False)
|
||||
self.peak_line.setPos(lag_sec)
|
||||
# shift B curve visually by lag (pad with silence not needed for visualize, just shift x)
|
||||
# we keep t_b but offset x
|
||||
shift = lag_sec
|
||||
# pyqtgraph setPos shifts items; we recreate x
|
||||
# Instead shift data: plot t_b + lag
|
||||
self.curve_b.setData(self.t_b + shift, self.b_plot)
|
||||
self.update_labels()
|
||||
self._updating = False
|
||||
|
||||
def update_labels(self):
|
||||
# ffmpeg adelay for mux: add silence for delay (no sound for build-in delay)
|
||||
# If lag>0: B delayed vs A => to mux B onto A video, delay B audio by lag (pad silence)
|
||||
# adelay expects ms per channel, e.g. adelay=1234|1234
|
||||
lag_ms = self.lag_sec * 1000
|
||||
if lag_ms >= 0:
|
||||
ffcmd = f'ffmpeg -i video.mkv -i audio.wav -filter_complex "[1:a]adelay={lag_ms:.0f}:all=1[ad]" -map 0:v -map "[ad]" -c:v copy -c:a pcm_s16le out.mkv'
|
||||
note = f"B delayed by {self.lag_sec:.6f}s ({self.lag_samples} samp) → pad B start with silence {lag_ms:.0f} ms"
|
||||
else:
|
||||
# B earlier => pad A (or trim B). For adding B to A file, we need to delay video or pad A? Simplest pad A? But we control only audio: delay B negative means B earlier, so we delay A or use -itsoffset
|
||||
note = f"B earlier by {abs(self.lag_sec):.6f}s → pad A / use -itsoffset {abs(lag_ms):.0f}ms or trim B by {abs(self.lag_samples)} samp"
|
||||
ffcmd = f'# B early: ffmpeg -itsoffset {abs(lag_ms)/1000:.6f} -i audio.wav -i video.mkv ... OR pad video audio with silence'
|
||||
self.ffmpeg_label.setText(f"{note}\n{ffcmd}")
|
||||
|
||||
# slots
|
||||
def on_slider(self, v):
|
||||
self._set_lag(v/1000.0, "slider")
|
||||
|
||||
def on_spin_sec(self, v):
|
||||
self._set_lag(float(v), "sec")
|
||||
|
||||
def on_spin_samp(self, v):
|
||||
self._set_lag(int(v)/self.sr, "samp")
|
||||
|
||||
def on_auto(self):
|
||||
# recompute peak
|
||||
lag, peak, corr = correlate(self.a_orig, self.b_orig, sr=self.sr, method=self.method, device=self.device_pref)
|
||||
self.corr = corr
|
||||
# update corr curve? re-decimate
|
||||
self._set_lag(lag / self.sr, "auto")
|
||||
print(f"auto lag {lag} samp {lag/self.sr:.6f}s peak={peak:.4f}")
|
||||
|
||||
def on_swap(self):
|
||||
self.a_orig, self.b_orig = self.b_orig, self.a_orig
|
||||
self.a, self.b = self.a_orig.copy(), self.b_orig.copy()
|
||||
self.a_plot, _ = decimate_for_plot(self.a)
|
||||
self.b_plot, _ = decimate_for_plot(self.b)
|
||||
self.curve_a.setData(self.t_a, self.a_plot)
|
||||
# recalc
|
||||
lag, _, corr = correlate(self.a, self.b, sr=self.sr, method=self.method, device=self.device_pref)
|
||||
self.corr = corr
|
||||
self._set_lag(lag/self.sr, "swap")
|
||||
|
||||
def play(self, which: str):
|
||||
if not HAS_SD:
|
||||
return
|
||||
vol = self.vol.value()/100.0
|
||||
sr = self.sr
|
||||
# play 2 seconds from 10s mark to hear sync
|
||||
start_samp = int(10 * sr)
|
||||
dur_samp = int(2 * sr)
|
||||
if start_samp + dur_samp > len(self.a_orig):
|
||||
start_samp = 0
|
||||
a_seg = self.a_orig[start_samp:start_samp+dur_samp] * vol
|
||||
# apply lag with silence padding (no sound for delay) — Option A
|
||||
lag = self.lag_samples
|
||||
# For playback mix, we need to align: if B delayed (lag>0), pad B front with silence
|
||||
# For preview we shift B segment accordingly
|
||||
b_start = start_samp
|
||||
if lag > 0:
|
||||
# B delayed => B segment should be taken earlier to align? For playback we pad B with lag silence, so effective B at time t is original at t-lag
|
||||
idx = b_start - lag
|
||||
if idx < 0:
|
||||
b_seg = np.concatenate([np.zeros(-idx, dtype=np.float32), self.b_orig[0:b_start + dur_samp - (-idx) if b_start+dur_samp - (-idx)>0 else 0]])
|
||||
# slice to dur
|
||||
b_seg = b_seg[:dur_samp]
|
||||
if len(b_seg) < dur_samp:
|
||||
b_seg = np.concatenate([b_seg, np.zeros(dur_samp-len(b_seg), dtype=np.float32)])
|
||||
else:
|
||||
b_seg = self.b_orig[idx:idx+dur_samp]
|
||||
elif lag < 0:
|
||||
idx = b_start - lag # lag negative => plus
|
||||
b_seg = self.b_orig[idx:idx+dur_samp] if idx+dur_samp < len(self.b_orig) else np.zeros(dur_samp, dtype=np.float32)
|
||||
if len(b_seg) < dur_samp:
|
||||
b_seg = np.concatenate([b_seg, np.zeros(dur_samp-len(b_seg), dtype=np.float32)])
|
||||
else:
|
||||
b_seg = self.b_orig[b_start:b_start+dur_samp]
|
||||
b_seg = b_seg * vol
|
||||
try:
|
||||
sd.stop()
|
||||
if which == "a":
|
||||
sd.play(a_seg, samplerate=sr)
|
||||
elif which == "b":
|
||||
sd.play(b_seg, samplerate=sr)
|
||||
elif which == "mix":
|
||||
mix = (a_seg + b_seg) / 2.0
|
||||
sd.play(mix, samplerate=sr)
|
||||
except Exception as e:
|
||||
print(f"playback failed {e}", file=sys.stderr)
|
||||
|
||||
def stop_play(self):
|
||||
if HAS_SD:
|
||||
sd.stop()
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--config", default="config.toml")
|
||||
ap.add_argument("--a", default=None)
|
||||
ap.add_argument("--b", default=None)
|
||||
ap.add_argument("--device", default=None, choices=["auto","cpu","cuda"])
|
||||
ap.add_argument("--method", default=None, choices=["phat","standard"])
|
||||
args = ap.parse_args()
|
||||
|
||||
cfg = load_config(args.config)
|
||||
cfg_a, cfg_b, sr, _, _ = find_wavs_from_config(cfg)
|
||||
a_path = args.a or cfg_a
|
||||
b_path = args.b or cfg_b
|
||||
method = args.method or cfg.get("sync", {}).get("method", "phat")
|
||||
device = args.device or cfg.get("sync", {}).get("use_gpu", "auto")
|
||||
# map toml "use_gpu" to device
|
||||
if device == "auto":
|
||||
device = "auto"
|
||||
elif device == "cpu":
|
||||
device = "cpu"
|
||||
elif device in ("cuda", "gpu"):
|
||||
device = "cuda"
|
||||
|
||||
if not a_path or not b_path:
|
||||
print(f"ERROR: no input wavs. Set file_a/file_b in {args.config} or --a/--b", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
if not os.path.exists(a_path):
|
||||
print(f"ERROR: A not found {a_path}", file=sys.stderr); sys.exit(1)
|
||||
if not os.path.exists(b_path):
|
||||
print(f"ERROR: B not found {b_path}", file=sys.stderr); sys.exit(1)
|
||||
|
||||
app = QtWidgets.QApplication(sys.argv)
|
||||
pg.setConfigOptions(antialias=True, background="#121212", foreground="#eee")
|
||||
w = SyncWindow(a_path, b_path, sr=sr, method=method, device=device)
|
||||
w.show()
|
||||
sys.exit(app.exec())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user