diff --git a/SETUP.md b/SETUP.md index f1d9957..d9aa286 100644 --- a/SETUP.md +++ b/SETUP.md @@ -94,9 +94,12 @@ All wrapper scripts (`run_optimisation.sh` on Linux, `run_optimisation.ps1` on W | `--preset ` | SVT-AV1 Preset (4=best, 6=balanced, 8=fast) | 6 | | `--workers ` | Concurrent files to process | 1 | | `--samples ` | Samples for CRF search | 4 | -| `--thorough` | Use thorough mode (slower, more accurate) | false | -| `--encoder ` | ab-av1 encoder | svt-av1 | -| `--hwaccel ` | Hardware acceleration | none (auto: auto-detect) | +| `--encoder ` | Video encoder: svt-av1, av1_amf, av1_nvenc, av1_qsv | svt-av1 | +| `--hwaccel ` | Hardware decode acceleration (auto, d3d11va, vaapi) | none | +| `--use-hardware-worker` | Use 1 HW encoder worker + rest CPU workers | false | +| `--plex-url ` | Plex server URL for library refresh | none | +| `--plex-token ` | Plex auth token | none | +| `--log-dir ` | Log directory | /opt/Optmiser/logs | ## Multi-Machine Setup @@ -140,6 +143,99 @@ All three can run simultaneously! ## Hardware Acceleration +### Hardware Decoding vs Hardware Encoding + +There are two types of hardware acceleration: + +1. **Hardware Decoding (`--hwaccel`)**: Uses GPU to decode source video faster. Doesn't affect quality. +2. **Hardware Encoding (`--encoder`)**: Uses GPU to encode output video. Much faster but requires quality compensation. + +### Hardware Encoders + +| Encoder | GPU | Platform | Notes | +|---------|-----|----------|-------| +| `svt-av1` | CPU | All | Default. Best quality, slowest. | +| `av1_amf` | AMD | Windows/Linux | RX 7000 series, ~3-10x faster than CPU | +| `av1_nvenc` | NVIDIA | Windows/Linux | RTX 40 series, very fast | +| `av1_qsv` | Intel | Windows/Linux | Arc GPUs, Intel iGPU with AV1 | + +### FFmpeg with Hardware Encoder Support + +**Windows (pre-built with HW encoders):** +```powershell +# Most Windows FFmpeg builds include hardware encoders +# Verify with: +ffmpeg -encoders 2>&1 | findstr av1 + +# Should show: av1_amf, av1_nvenc, av1_qsv (depending on your GPU) + +# If missing, download from: https://github.com/BtbN/FFmpeg-Builds/releases +# Choose: ffmpeg-master-latest-win64-gpl.zip (includes all encoders) +``` + +**Linux (may need custom build):** +```bash +# Check available encoders +ffmpeg -encoders 2>&1 | grep av1 + +# For AMD (av1_amf) - requires AMF SDK +# Install AMD GPU drivers with AMF support +sudo apt install amdgpu-pro + +# For NVIDIA (av1_nvenc) - requires NVIDIA drivers +sudo apt install nvidia-driver-535 # or newer + +# For Intel (av1_qsv) - requires Intel Media SDK +sudo apt install intel-media-va-driver-non-free + +# If your distro's ffmpeg lacks HW encoders, use static build: +wget https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz +``` + +### Using Hardware Workers (1 GPU + 3 CPU) + +For mixed encoding with your RX 7900 XT: + +```powershell +# Windows - 4 workers: 1 uses GPU (av1_amf), 3 use CPU (svt-av1) +.\run_optimisation.ps1 ` + -Directory "Z:\" ` + -Vmaf 94 ` + -Workers 4 ` + -Encoder av1_amf ` + -Hwaccel auto ` + -UseHardwareWorker ` + -LogDir "C:\Users\bnair\Documents\VMAFOptimiser\logs" +``` + +```bash +# Linux - same concept +./run_optimisation.sh \ + --directory /media \ + --vmaf 94 \ + --workers 4 \ + --encoder av1_amf \ + --hwaccel auto \ + --use-hardware-worker +``` + +**How it works:** +- First worker to start claims the GPU and uses `av1_amf` +- Remaining 3 workers use `svt-av1` (CPU) +- GPU worker applies +2 VMAF offset automatically to match CPU quality + +### Quality Compensation + +Hardware encoders produce lower quality at the same settings. The script automatically compensates: + +| Target VMAF | CPU (svt-av1) | GPU (av1_amf) | +|-------------|---------------|---------------| +| 94 | Searches for VMAF 94 | Searches for VMAF 96 | +| 93 | Searches for VMAF 93 | Searches for VMAF 95 | +| 92 | Searches for VMAF 92 | Searches for VMAF 94 | + +This offset (`HW_ENCODER_VMAF_OFFSET = 2.0`) can be adjusted in `optimize_library.py`. + ### Automatic Detection When `--hwaccel auto` is specified, the wrapper scripts automatically select the best available hardware acceleration: diff --git a/optimize_library.py b/optimize_library.py index 376a200..c9c0491 100644 --- a/optimize_library.py +++ b/optimize_library.py @@ -10,7 +10,7 @@ import signal from pathlib import Path from datetime import datetime from concurrent.futures import ThreadPoolExecutor, as_completed, ProcessPoolExecutor -from threading import Lock +from threading import Lock, get_ident # --- Configuration --- DEFAULT_VMAF = 95.0 @@ -23,11 +23,27 @@ TARGETS = [94.0, 93.0, 92.0, 90.0] MIN_SAVINGS_PERCENT = 12.0 TARGET_SAVINGS_FOR_ESTIMATE = 15.0 +# Hardware encoder quality compensation (GPU needs higher VMAF target to match CPU quality) +HW_ENCODER_VMAF_OFFSET = 2.0 +HW_ENCODERS = {"av1_amf", "av1_nvenc", "av1_qsv", "av1_vaapi"} + # Global state for resume capability _processed_files = set() _lock = Lock() _shutdown_requested = False _AB_AV1_HELP_CACHE = {} +_hw_worker_id = None # Thread ID of the designated hardware worker + + +def claim_hardware_worker(): + """Attempt to claim hardware worker status for this thread. Returns True if claimed.""" + global _hw_worker_id + with _lock: + thread_id = get_ident() + if _hw_worker_id is None: + _hw_worker_id = thread_id + return True + return _hw_worker_id == thread_id def signal_handler(signum, frame): @@ -202,15 +218,30 @@ def run_command_streaming(cmd, description=""): return process.returncode -def run_crf_search(filepath, target_vmaf, preset, temp_dir, use_hw=False, hwaccel=None): +def run_crf_search( + filepath, + target_vmaf, + preset, + temp_dir, + encoder="svt-av1", + use_hw=False, + hwaccel=None, +): """Run CRF search for a specific VMAF target""" + is_hw_encoder = encoder in HW_ENCODERS + + # Apply VMAF offset for hardware encoders to match software quality + effective_vmaf = ( + target_vmaf + HW_ENCODER_VMAF_OFFSET if is_hw_encoder else target_vmaf + ) + cmd = [ "ab-av1", "crf-search", "-i", str(filepath), "--min-vmaf", - str(target_vmaf), + str(effective_vmaf), "--preset", str(preset), "--max-encoded-percent", @@ -218,18 +249,24 @@ def run_crf_search(filepath, target_vmaf, preset, temp_dir, use_hw=False, hwacce "--temp-dir", temp_dir, "--samples", - "4", # Use 4 samples for speed/accuracy balance + "4", ] - # Hardware encoding support + # Add encoder if not default + if encoder != "svt-av1": + if ab_av1_supports("crf-search", "--encoder"): + cmd.extend(["--encoder", encoder]) + + # Hardware decode acceleration if use_hw and hwaccel: if ab_av1_supports("crf-search", "--enc-input"): cmd.extend(["--enc-input", f"hwaccel={hwaccel}"]) if hwaccel == "vaapi": cmd.extend(["--enc-input", "hwaccel_output_format=vaapi"]) - print(f" - Searching for CRF to hit VMAF {target_vmaf}...") - returncode = run_command_streaming(cmd, f"crf-search VMAF {target_vmaf}") + vmaf_label = f"VMAF {effective_vmaf}" if is_hw_encoder else f"VMAF {target_vmaf}" + print(f" - Searching for CRF to hit {vmaf_label}...") + returncode = run_command_streaming(cmd, f"crf-search {vmaf_label}") if returncode == 0: # Parse output to find CRF and predicted size @@ -274,7 +311,13 @@ def run_crf_search(filepath, target_vmaf, preset, temp_dir, use_hw=False, hwacce def find_target_savings_params( - filepath, start_vmaf, preset, temp_dir, use_hw=False, hwaccel=None + filepath, + start_vmaf, + preset, + temp_dir, + encoder="svt-av1", + use_hw=False, + hwaccel=None, ): """Find VMAF target that achieves minimum savings""" print(f"\n --- Finding VMAF for {TARGET_SAVINGS_FOR_ESTIMATE}% savings ---") @@ -288,7 +331,9 @@ def find_target_savings_params( print( f" Testing VMAF {target} for {TARGET_SAVINGS_FOR_ESTIMATE}% target... (test {i + 1}/{len(test_targets)})" ) - result = run_crf_search(filepath, target, preset, temp_dir, use_hw, hwaccel) + result = run_crf_search( + filepath, target, preset, temp_dir, encoder, use_hw, hwaccel + ) if result: predicted_savings = 100.0 - result["predicted_percent"] @@ -317,7 +362,9 @@ def find_target_savings_params( return None -def run_encode(filepath, output_path, crf, preset, use_hw=False, hwaccel=None): +def run_encode( + filepath, output_path, crf, preset, encoder="svt-av1", use_hw=False, hwaccel=None +): """Run full encoding with real-time output streaming""" cmd = [ "ab-av1", @@ -334,7 +381,10 @@ def run_encode(filepath, output_path, crf, preset, use_hw=False, hwaccel=None): "copy", ] - # Hardware encoding support + if encoder != "svt-av1": + if ab_av1_supports("encode", "--encoder"): + cmd.extend(["--encoder", encoder]) + if use_hw and hwaccel: if ab_av1_supports("encode", "--enc-input"): cmd.extend(["--enc-input", f"hwaccel={hwaccel}"]) @@ -401,10 +451,26 @@ def refresh_plex(plex_url, plex_token): print(f" ⚠️ Failed to refresh Plex: {e}") -def process_file(filepath, log_dir, log_name, preset, use_hw=False, hwaccel=None): +def process_file( + filepath, + log_dir, + log_name, + preset, + hw_encoder="svt-av1", + use_hw_mode=False, + hwaccel=None, +): """Process a single video file with intelligent VMAF targeting""" global _shutdown_requested + # Determine if THIS worker should use hardware encoder + use_hw = False + if use_hw_mode and hwaccel and hw_encoder in HW_ENCODERS: + use_hw = claim_hardware_worker() + + # HW worker uses hardware encoder; CPU workers use svt-av1 + encoder = hw_encoder if use_hw else "svt-av1" + filepath = Path(filepath) lock_file = Path(log_dir).parent / ".lock" / f"{filepath.name}.lock" @@ -449,21 +515,21 @@ def process_file(filepath, log_dir, log_name, preset, use_hw=False, hwaccel=None # Step 1: Try VMAF 94 print(f"\n [Step 1] Testing VMAF 94...") search_result_94 = run_crf_search( - filepath, 94.0, preset, str(temp_dir), use_hw, hwaccel + filepath, 94.0, preset, str(temp_dir), encoder, use_hw, hwaccel ) if not search_result_94: print(f" !! Could not hit VMAF 94") search_result_94 = run_crf_search( - filepath, 93.0, preset, str(temp_dir), use_hw, hwaccel + filepath, 93.0, preset, str(temp_dir), encoder, use_hw, hwaccel ) if not search_result_94: search_result_94 = run_crf_search( - filepath, 92.0, preset, str(temp_dir), use_hw, hwaccel + filepath, 92.0, preset, str(temp_dir), encoder, use_hw, hwaccel ) if not search_result_94: search_result_94 = run_crf_search( - filepath, 90.0, preset, str(temp_dir), use_hw, hwaccel + filepath, 90.0, preset, str(temp_dir), encoder, use_hw, hwaccel ) if not search_result_94: @@ -499,7 +565,7 @@ def process_file(filepath, log_dir, log_name, preset, use_hw=False, hwaccel=None ) search_result_93 = run_crf_search( - filepath, 93.0, preset, str(temp_dir), use_hw, hwaccel + filepath, 93.0, preset, str(temp_dir), encoder, use_hw, hwaccel ) if search_result_93: predicted_savings_93 = 100.0 - search_result_93["predicted_percent"] @@ -524,7 +590,7 @@ def process_file(filepath, log_dir, log_name, preset, use_hw=False, hwaccel=None f" → Finding VMAF for {TARGET_SAVINGS_FOR_ESTIMATE}% savings..." ) target_result = find_target_savings_params( - filepath, 93.0, preset, str(temp_dir), use_hw, hwaccel + filepath, 93.0, preset, str(temp_dir), encoder, use_hw, hwaccel ) provide_recommendations( @@ -559,7 +625,13 @@ def process_file(filepath, log_dir, log_name, preset, use_hw=False, hwaccel=None start_time = time.time() res = run_encode( - filepath, temp_output, encode_params["crf"], preset, use_hw, hwaccel + filepath, + temp_output, + encode_params["crf"], + preset, + encoder, + use_hw, + hwaccel, ) if res != 0: @@ -688,10 +760,18 @@ def main(): "Examples: auto, vaapi, d3d11va, videotoolbox. Use 'none' to disable." ), ) + parser.add_argument( + "--encoder", + default="svt-av1", + help=( + "Video encoder to use. Default: svt-av1 (CPU). " + "Hardware encoders: av1_amf (AMD), av1_nvenc (NVIDIA), av1_qsv (Intel)." + ), + ) parser.add_argument( "--use-hardware-worker", action="store_true", - help="Use 1 hardware encoding worker + rest CPU workers (requires --hwaccel)", + help="Use 1 hardware encoding worker + rest CPU workers (requires --encoder with HW encoder)", ) parser.add_argument( "--plex-url", @@ -763,7 +843,9 @@ def main(): fail_count = 0 # Hardware worker configuration - use_hw_primary = args.use_hardware_worker and hwaccel is not None + # HW worker uses specified encoder; CPU workers use svt-av1 + hw_encoder = args.encoder if args.encoder in HW_ENCODERS else None + use_hw_primary = args.use_hardware_worker and hw_encoder is not None if args.workers == 1: # Single thread - just process files @@ -772,7 +854,13 @@ def main(): break processed_count += 1 result = process_file( - file_path, args.log_dir, log_name, args.preset, use_hw_primary, hwaccel + file_path, + args.log_dir, + log_name, + args.preset, + args.encoder, + use_hw_primary, + hwaccel, ) if result: success_count += 1 @@ -786,15 +874,16 @@ def main(): if _shutdown_requested: break - # Use hardware for first file, CPU for rest - use_hw_for_this = use_hw_primary and len(futures) == 0 + # All workers try to claim HW; only the first thread succeeds + # and will use HW for ALL its tasks future = executor.submit( process_file, file_path, args.log_dir, log_name, args.preset, - use_hw_for_this, + args.encoder, + use_hw_primary, hwaccel, ) futures.append(future) diff --git a/run_optimisation.ps1 b/run_optimisation.ps1 index 2a8556a..d26f32e 100644 --- a/run_optimisation.ps1 +++ b/run_optimisation.ps1 @@ -6,6 +6,7 @@ param( [int]$Workers = 1, [int]$Samples = 4, [string]$Hwaccel = "", + [string]$Encoder = "svt-av1", [switch]$UseHardwareWorker, [string]$PlexUrl = "", [string]$PlexToken = "", @@ -27,7 +28,7 @@ function Invoke-OptimizeLibrary { exit 1 } - $pythonCmd = Get-Command python3, python, py | Select-Object -FirstProperty Path -ErrorAction SilentlyContinue + $pythonCmd = Get-Command python3, python, py -ErrorAction SilentlyContinue | Select-Object -First 1 if (-not $pythonCmd) { Write-ColorOutput -Message "ERROR: Python 3 not found. Please install Python 3." -Color "Red" exit 1 @@ -40,6 +41,7 @@ function Invoke-OptimizeLibrary { "--preset", $Preset.ToString(), "--workers", $Workers.ToString(), "--samples", $Samples.ToString(), + "--encoder", $Encoder, "--log-dir", $LogDir ) @@ -65,11 +67,12 @@ function Invoke-OptimizeLibrary { Write-ColorOutput -Message " Preset: $Preset" -Color "White" Write-ColorOutput -Message " Workers: $Workers" -Color "White" Write-ColorOutput -Message " Samples: $Samples" -Color "White" + Write-ColorOutput -Message " Encoder: $Encoder" -Color "White" if ($Hwaccel) { Write-ColorOutput -Message " HW Accel: $Hwaccel" -Color "White" } if ($UseHardwareWorker) { - Write-ColorOutput -Message " Hardware worker: Enabled" -Color "White" + Write-ColorOutput -Message " Hardware worker: Enabled (1 HW + $($Workers - 1) CPU)" -Color "White" } if ($PlexUrl -and $PlexToken) { Write-ColorOutput -Message " Plex refresh: Enabled" -Color "White"