perfect-postcode/video/render.sh
2026-07-14 19:28:09 +01:00

458 lines
20 KiB
Bash
Executable file
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env bash
#
# End-to-end re-render of the dashboard demo videos.
#
# All per-storyboard knobs (aspect, fps, bitrate, prompt text, localized
# narration, voice persona, poster timestamp, brand strings…) live in
# src/storyboard.ts. A single visual storyboard can expand into multiple
# language variants there; this script renders every emitted slug.
#
# Two targets:
# local (default): assumes the docker-compose stack on host.docker.internal,
# bootstraps a recorder admin user automatically.
# prod: points at https://perfect-postcode.co.uk and skips the
# bootstrap step; you supply real account credentials.
#
# Usage:
# ./render.sh # local stack, English homepage landscape + portrait
# ./render.sh --prod # prod, English homepage landscape + portrait
# ./render.sh --target prod # same as --prod
# VIDEO_STORYBOARD_SET=ads ./render.sh --prod # render social ads instead
# VIDEO_STORYBOARD_SET=demo ./render.sh --prod # render every homepage locale
# ./render.sh --fresh-auth # force re-auth even if cache is fresh
# ./render.sh --resume # preserve completed recordings and continue
# ./render.sh --no-encode # stop at WebM, skip MP4 encode
# ./render.sh --no-audio # skip narration synthesis; silent MP4
# FORCE_AUTH=1 ./render.sh # same as --fresh-auth
# APP_URL=http://localhost:3001 ./render.sh # override frontend URL
# CHATTERBOX_URL=http://box:8004 ./render.sh # override the TTS server
#
# Cred env vars (read for both targets, but prod has no fallback defaults):
# LOGIN_EMAIL, LOGIN_PASSWORD: the dashboard account to record as
# (same email/password you'd type into
# the login modal)
# PB_ADMIN_EMAIL, PB_ADMIN_PASSWORD: PocketBase superuser, only used by
# --target local to bootstrap the
# recorder user; ignored on --prod
set -euo pipefail
# -- target -------------------------------------------------------------------
TARGET="${TARGET:-local}"
parsed_args=()
while [ $# -gt 0 ]; do
case "$1" in
--prod) TARGET=prod; shift ;;
--local) TARGET=local; shift ;;
--target) TARGET="$2"; shift 2 ;;
--target=*) TARGET="${1#--target=}"; shift ;;
*) parsed_args+=("$1"); shift ;;
esac
done
set -- "${parsed_args[@]+"${parsed_args[@]}"}"
case "$TARGET" in
local|prod) ;;
*) echo "Unknown --target: $TARGET (expected: local, prod)" >&2; exit 2 ;;
esac
# -- environment (target-specific URLs and credentials) ----------------------
if [ "$TARGET" = "prod" ]; then
# Prod serves frontend, /api/*, and /pb/* off the same domain.
export APP_URL="${APP_URL:-https://perfect-postcode.co.uk}"
export PB_URL="${PB_URL:-https://perfect-postcode.co.uk/pb}"
export API_URL="${API_URL:-https://perfect-postcode.co.uk}"
# Prod has no recorder-bootstrap path: the user supplies a real account.
PB_BOOTSTRAP_ADMIN="${PB_BOOTSTRAP_ADMIN:-0}"
if [ -z "${LOGIN_EMAIL:-}" ] || [ -z "${LOGIN_PASSWORD:-}" ]; then
echo "FAIL: --prod requires LOGIN_EMAIL and LOGIN_PASSWORD (your dashboard login)." >&2
echo " Example: LOGIN_EMAIL=you@example.com LOGIN_PASSWORD='...' $0 --prod" >&2
exit 2
fi
else
export APP_URL="${APP_URL:-http://host.docker.internal:3001}"
export PB_URL="${PB_URL:-http://host.docker.internal:8090}"
export API_URL="${API_URL:-http://host.docker.internal:8001}"
PB_ADMIN_EMAIL="${PB_ADMIN_EMAIL:-admin@propertymap.local}"
PB_ADMIN_PASSWORD="${PB_ADMIN_PASSWORD:-propertymap-dev-2024}"
LOGIN_EMAIL="${LOGIN_EMAIL:-demo-video@local.test}"
LOGIN_PASSWORD="${LOGIN_PASSWORD:-DemoVideoPass123!}"
PB_BOOTSTRAP_ADMIN="${PB_BOOTSTRAP_ADMIN:-1}"
fi
export PB_BOOTSTRAP_ADMIN
export LOGIN_EMAIL LOGIN_PASSWORD
# Narration TTS. Not target-specific: Chatterbox is a LAN service, so prod
# renders and local renders both narrate from the same box.
export CHATTERBOX_URL="${CHATTERBOX_URL:-http://host.docker.internal:8004}"
# Per-target storage state: switching targets must not reuse a stale token.
# config.ts reads AUTH_STATE_FILE for AUTH_STATE_PATH.
export AUTH_STATE_FILE="${AUTH_STATE_FILE:-auth.${TARGET}.json}"
AUTH_TTL_HOURS="${AUTH_TTL_HOURS:-24}" # re-auth if cache older than this
# Where the homepage <video> source lives. Vite copies frontend/public/* into
# the built bundle, so updating this path is what makes the new clip appear
# on the homepage. Override if the dashboard ever moves.
PUBLISH_DIR="${PUBLISH_DIR:-../frontend/public/video}"
FRESH_AUTH="${FORCE_AUTH:-0}"
RESUME_RECORDINGS="${RESUME_RECORDINGS:-0}"
DO_ENCODE=1
DO_AUDIO=1
for arg in "${@:-}"; do
[ -z "$arg" ] && continue
case "$arg" in
--fresh-auth) FRESH_AUTH=1 ;;
--resume) RESUME_RECORDINGS=1 ;;
--no-encode) DO_ENCODE=0 ;;
--no-audio) DO_AUDIO=0 ;;
-h|--help)
# Print the header comment block, stopping at the first non-comment
# line. A hardcoded end line silently truncates --help mid-sentence the
# next time someone documents a flag.
sed -n '3,/^[^#]/p' "$0" | sed '$d'
exit 0 ;;
*) echo "Unknown arg: $arg" >&2; exit 2 ;;
esac
done
cd "$(dirname "$0")"
# -- scratch space ------------------------------------------------------------
# Host /tmp is a small tmpfs (often <512MB) and /dev/shm is even smaller, but
# Playwright spawns a fresh Chromium user-data dir per launch (GPU shader
# cache, blob storage, …) and ffmpeg/uv also fall back to TMPDIR for their
# intermediates. Steer everything to a local scratch dir under video/ so the
# render survives on hosts with a tight /tmp budget. Override SCRATCH_DIR to
# point elsewhere if disk under video/ is also constrained.
SCRATCH_DIR="${SCRATCH_DIR:-$PWD/.tmp}"
mkdir -p "$SCRATCH_DIR"
export TMPDIR="$SCRATCH_DIR"
# -- helpers ------------------------------------------------------------------
say() { printf '\n[render] %s\n' "$*"; }
fail() { printf '\n[render] FAIL: %s\n' "$*" >&2; exit 1; }
http_code() {
curl -s -o /dev/null -w "%{http_code}" --connect-timeout 3 --max-time 5 "$1" || echo "000"
}
# Order storyboards so all English cuts synth together and all non-English
# cuts synth together. The Chatterbox server holds one checkpoint at a time
# and English lives on a different one from every other language, so an
# interleaved order (recording, recording-de, recording-zh, recording-hi)
# would pay a ~20s swap three times instead of once. Reads the language each
# storyboard declared in the narration script preflight just emitted; any
# storyboard whose script is unreadable sorts last and still gets rendered.
synth_order() {
local sb lang
for sb in "$@"; do
lang="$(node -e '
const fs = require("node:fs");
try {
process.stdout.write(
JSON.parse(fs.readFileSync(process.argv[1], "utf-8")).voice?.language ?? "zz"
);
} catch { process.stdout.write("zz"); }
' "output/$sb/narration-script.json" 2>/dev/null || echo "zz")"
# en sorts before everything else; the rest group by language code.
printf '%s\t%s\n' "$([ "$lang" = "en" ] && echo "0-en" || echo "1-$lang")" "$sb"
done | sort -s -k1,1 | cut -f2
}
wait_for() {
local url="$1" desc="$2" timeout="${3:-90}"
say "Waiting for $desc ($url)"
for i in $(seq 1 "$timeout"); do
if [ "$(http_code "$url")" = "200" ]; then
say " ready after ${i}s"
return 0
fi
sleep 1
done
fail "$desc not reachable after ${timeout}s"
}
# -- stack health -------------------------------------------------------------
say "Checking stack health"
fe_code="$(http_code "$APP_URL/")"
api_code="$(http_code "$API_URL/api/features")"
pb_code="$(http_code "$PB_URL/api/health")"
echo " frontend=$fe_code api=$api_code pocketbase=$pb_code"
if [ "$fe_code" != "200" ] || [ "$pb_code" != "200" ]; then
if [ "$TARGET" = "prod" ]; then
fail "Cannot reach prod ($APP_URL / $PB_URL). Check the URLs and your network."
else
fail "Stack down. From the repo root run: docker compose up -d"
fi
fi
if [ "$api_code" != "200" ]; then
wait_for "$API_URL/api/features" "Rust API" 120
fi
# -- node deps ----------------------------------------------------------------
if [ ! -d node_modules ]; then
say "Installing npm deps"
npm install --no-audit --no-fund
fi
# Chromium binaries live in Playwright's cache; install if missing.
# Playwright launches via chrome-headless-shell by default, so we must verify
# BOTH the chromium and chrome-headless-shell executables exist. Checking the
# actual binary (not just the parent dir) is intentional: a previously failed
# install (e.g. ENOSPC mid-download) leaves the chromium-* / chromium_headless_shell-*
# dir in place but without the binary inside, and a dir-only check would
# treat that as "installed".
chromium_bin="$(find "$HOME/.cache/ms-playwright" -path '*/chromium-*/chrome-linux/chrome' -print -quit 2>/dev/null || true)"
headless_shell_bin="$(find "$HOME/.cache/ms-playwright" -path '*/chromium_headless_shell-*/chrome-headless-shell-linux64/chrome-headless-shell' -print -quit 2>/dev/null || true)"
if ! npx --no-install playwright --version >/dev/null 2>&1 \
|| [ ! -x "${chromium_bin:-/nonexistent}" ] \
|| [ ! -x "${headless_shell_bin:-/nonexistent}" ]; then
say "Installing Playwright Chromium"
# Playwright stages the ~170MB browser zip in $TMPDIR before extracting into
# ~/.cache/ms-playwright. /tmp is often a small tmpfs (we've hit ENOSPC at
# 92%), so steer the staging dir into the cache dir itself, which lives on
# the same (larger) filesystem we're extracting into anyway.
pw_tmpdir="$HOME/.cache/ms-playwright/.tmp"
mkdir -p "$pw_tmpdir"
TMPDIR="$pw_tmpdir" npx playwright install chromium
rm -rf "$pw_tmpdir"
fi
# System libs Chromium dlopens (libnspr4, libnss3, libasound2…). Detect via
# the canonical missing one and shell out to playwright's apt wrapper. Needs
# sudo; gated on ldconfig so we don't prompt unnecessarily on every render.
if ! ldconfig -p 2>/dev/null | grep -q 'libnspr4\.so'; then
say "Installing Chromium system deps (sudo)"
sudo npx playwright install-deps chromium
fi
# -- build --------------------------------------------------------------------
say "Compiling TypeScript"
./node_modules/.bin/tsc
# -- auth ---------------------------------------------------------------------
need_auth=0
if [ "$FRESH_AUTH" = "1" ] || [ ! -f "$AUTH_STATE_FILE" ]; then
need_auth=1
else
# File mtime check, portable: if older than TTL, refresh.
if [ "$(find "$AUTH_STATE_FILE" -mmin "+$((AUTH_TTL_HOURS * 60))" -print 2>/dev/null)" ]; then
say "$AUTH_STATE_FILE is older than ${AUTH_TTL_HOURS}h, will refresh"
need_auth=1
fi
fi
if [ "$need_auth" = "1" ]; then
say "Minting fresh $AUTH_STATE_FILE (target: $TARGET, user: $LOGIN_EMAIL)"
PB_URL="$PB_URL" \
LOGIN_EMAIL="$LOGIN_EMAIL" LOGIN_PASSWORD="$LOGIN_PASSWORD" \
PB_ADMIN_EMAIL="${PB_ADMIN_EMAIL:-}" PB_ADMIN_PASSWORD="${PB_ADMIN_PASSWORD:-}" \
PB_BOOTSTRAP_ADMIN="$PB_BOOTSTRAP_ADMIN" \
AUTH_STATE_FILE="$AUTH_STATE_FILE" \
APP_URL="$APP_URL" \
node dist/auth.js
else
say "Reusing existing $AUTH_STATE_FILE"
fi
# -- preflight ---------------------------------------------------------------
# preflight emits per-storyboard narration scripts AND output/storyboards.json
# (the index this script loops over below). Run it BEFORE wiping per-storyboard
# files so we know what slugs to target.
mkdir -p output
say "Preflight: emitting narration scripts and storyboard index"
node dist/preflight.js
if [ ! -s output/storyboards.json ]; then
fail "preflight did not produce output/storyboards.json"
fi
# Pull the storyboard slugs out of the index. Use Node so we don't grow a jq
# dependency just for one read.
mapfile -t STORYBOARDS < <(node -e '
const idx = JSON.parse(require("fs").readFileSync("output/storyboards.json","utf8"));
for (const s of idx.storyboards) console.log(s.name);
')
if [ "${#STORYBOARDS[@]}" -eq 0 ]; then
fail "storyboards.json contains no storyboards"
fi
say "Storyboards to render: ${STORYBOARDS[*]}"
# Per-storyboard poster timestamp lookup (slug → seconds), set once so each
# loop body can read it without re-parsing the index.
poster_time_for() {
node -e '
const idx = JSON.parse(require("fs").readFileSync("output/storyboards.json","utf8"));
const sb = idx.storyboards.find(s => s.name === process.argv[1]);
if (!sb) { process.exit(1); }
process.stdout.write(String(sb.posterTimeS));
' "$1"
}
# Resolve the FINAL published video dimensions for a storyboard. The
# recording happens at the CSS viewport, but the encode pass upscales to
# `captureScale x viewport` via lanczos so the published mp4 is true
# 1080x1920 on mobile rather than a soft 540x960. Returns "WxH".
published_size_for() {
node -e '
const idx = JSON.parse(require("fs").readFileSync("output/storyboards.json","utf8"));
const sb = idx.storyboards.find(s => s.name === process.argv[1]);
if (!sb || !sb.publishedSize) { process.exit(1); }
process.stdout.write(`${sb.publishedSize.width}x${sb.publishedSize.height}`);
' "$1"
}
# -- per-storyboard wipe of leaking artefacts --------------------------------
# output/<sb>/audio/ is preserved; tts/synth.py decides whether the cached
# WAVs still match the script and skips generation when they do. In resume
# mode, keep successful recording.webm + narration.json pairs and let record.ts
# skip them, but still clear raw/intermediate files that may be left by a
# failed Playwright save.
for sb in "${STORYBOARDS[@]}"; do
if [ "$RESUME_RECORDINGS" = "1" ]; then
rm -f "output/$sb/page@"*.webm "output/$sb/page@"*.webm.untrimmed \
"output/$sb/recording.raw.webm" "output/$sb/recording.raw.webm.untrimmed" \
"output/$sb/recording.narrated.mp4"
else
rm -f "output/$sb/recording.webm" "output/$sb/recording.mp4" \
"output/$sb/page@"*.webm "output/$sb/page@"*.webm.untrimmed \
"output/$sb/recording.raw.webm" "output/$sb/recording.raw.webm.untrimmed" \
"output/$sb/recording.narrated.mp4" "output/$sb/poster.jpg" \
"output/$sb/narration.json"
fi
done
# -- synth (Chatterbox) ------------------------------------------------------
# Synth runs BEFORE recording: the recorder reads output/<sb>/audio/index.json
# for measured per-cue durations and sizes each cue's wall-clock to fit;
# --no-audio skips synth and the recorder falls back to a worst-case estimate.
#
# Narration is generated by the Chatterbox TTS server over HTTP, so there is
# nothing to install and no GPU to schedule here. Voice timbre needs no
# defending either: a predefined voice is a fixed asset on the server, so
# every cue of every storyboard clones from identical bytes by construction.
# (The Qwen3-TTS pipeline this replaced had to mint a reference WAV per render
# and hand-copy it between storyboards to stop two voices landing in one cut.)
#
# Storyboards are synthesised in language order so an en/de/zh set pays the
# ~20s English<->multilingual checkpoint swap once instead of once per cut.
# synth.py itself only swaps when the server is on the wrong checkpoint.
if [ "$DO_AUDIO" = "1" ]; then
if ! command -v uv >/dev/null 2>&1; then
fail "uv not on PATH (required for tts/synth.py). Install uv or rerun with --no-audio."
fi
say "Synchronising tts/ Python deps"
uv sync --project tts || fail "uv sync failed in video/tts"
cb_code="$(http_code "$CHATTERBOX_URL/api/model-info")"
if [ "$cb_code" != "200" ]; then
fail "Cannot reach the Chatterbox TTS server at $CHATTERBOX_URL (got $cb_code). Start it, set CHATTERBOX_URL, or rerun with --no-audio."
fi
for sb in $(synth_order "${STORYBOARDS[@]}"); do
say "Synthesising narration for [$sb]"
CHATTERBOX_URL="$CHATTERBOX_URL" uv run --project tts python tts/synth.py --storyboard "$sb" \
|| fail "tts/synth.py failed for $sb"
if [ ! -s "output/$sb/audio/index.json" ]; then
fail "synth did not produce output/$sb/audio/index.json"
fi
done
fi
# -- record ------------------------------------------------------------------
# record.ts iterates over storyboards in-process and writes per-storyboard
# recording.webm + narration.json. One Node invocation handles all of them
# so we don't spin up Playwright + GPU/WebGL + auth more than necessary.
say "Recording all storyboards"
RESUME_RECORDINGS="$RESUME_RECORDINGS" APP_URL="$APP_URL" node dist/record.js
for sb in "${STORYBOARDS[@]}"; do
if [ ! -s "output/$sb/recording.webm" ]; then
fail "[$sb] recording.webm missing or empty"
fi
node dist/verify.js "$sb" "output/$sb/recording.webm"
done
# -- encode + mux + publish (per storyboard) ---------------------------------
if [ "$DO_ENCODE" = "1" ] && ! command -v ffmpeg >/dev/null 2>&1; then
fail "ffmpeg not on PATH; rerun with --no-encode if you only need the WebM"
fi
for sb in "${STORYBOARDS[@]}"; do
if [ "$DO_ENCODE" = "1" ]; then
say "[$sb] Encoding to MP4"
# Lanczos upscale the recording to its published dimensions
# (captureScale × viewport). For captureScale=1 the filter is a
# no-op and ffmpeg copies the size through; for captureScale=2
# mobile cuts go 540x960 → 1080x1920 sharply because Chromium
# already rasterised internally at DPR=2.
pub_size="$(published_size_for "$sb")"
pub_w="${pub_size%x*}"
pub_h="${pub_size#*x}"
ffmpeg -y -loglevel warning -i "output/$sb/recording.webm" \
-vf "scale=${pub_w}:${pub_h}:flags=lanczos" \
-c:v libx264 -pix_fmt yuv420p -crf 14 -preset fast \
-movflags +faststart \
"output/$sb/recording.mp4"
# Poster: a single high-quality JPEG extracted from a representative
# moment in the output timeline. Used as the homepage <video poster=...>.
# - -ss AFTER -i = output-side seek, frame-accurate (input-side seek
# would land on the nearest keyframe, drifting back up to ~2s).
# - -update 1 tells ffmpeg the output is a single image, not a sequence.
# - -q:v 2 = high JPEG quality (~95%); poster file is ~120KB at 1080p.
poster_t="$(poster_time_for "$sb")"
say "[$sb] Extracting poster frame at ${poster_t}s"
ffmpeg -y -loglevel warning -i "output/$sb/recording.mp4" -ss "$poster_t" \
-frames:v 1 -update 1 -q:v 2 \
"output/$sb/poster.jpg"
node dist/verify.js "$sb" "output/$sb/recording.mp4" "output/$sb/poster.jpg"
fi
if [ "$DO_ENCODE" = "1" ] && [ "$DO_AUDIO" = "1" ]; then
if [ ! -s "output/$sb/narration.json" ]; then
fail "[$sb] narration.json missing: recorder did not log cues"
fi
say "[$sb] Muxing narration into output/$sb/recording.mp4"
uv run --project tts python tts/mux.py --storyboard "$sb" --replace \
|| fail "tts/mux.py failed for $sb"
node dist/verify.js "$sb" "output/$sb/recording.mp4"
fi
# Only publish when we did the encode (otherwise we'd be copying a stale
# mp4 next to a fresh webm). --no-encode skips publish.
if [ "$DO_ENCODE" = "1" ]; then
if [ ! -d "$PUBLISH_DIR" ]; then
say "Creating $PUBLISH_DIR"
mkdir -p "$PUBLISH_DIR"
fi
say "[$sb] Publishing to $PUBLISH_DIR/$sb.{mp4,jpg}"
cp "output/$sb/recording.mp4" "$PUBLISH_DIR/$sb.mp4"
cp "output/$sb/poster.jpg" "$PUBLISH_DIR/$sb.jpg"
[ -f "output/$sb/narration.vtt" ] && cp "output/$sb/narration.vtt" "$PUBLISH_DIR/$sb.vtt"
node dist/verify.js "$sb" "$PUBLISH_DIR/$sb.mp4" "$PUBLISH_DIR/$sb.jpg"
fi
done
# -- report ------------------------------------------------------------------
say "Done"
if command -v ffprobe >/dev/null 2>&1; then
for sb in "${STORYBOARDS[@]}"; do
for f in "output/$sb/recording.webm" "output/$sb/recording.mp4" \
"output/$sb/poster.jpg" \
"$PUBLISH_DIR/$sb.mp4" "$PUBLISH_DIR/$sb.jpg"; do
[ -f "$f" ] || continue
size=$(stat -c '%s' "$f" 2>/dev/null || stat -f '%z' "$f")
case "$f" in
*.mp4|*.webm)
dur=$(ffprobe -v error -show_entries format=duration -of default=noprint_wrappers=1:nokey=1 "$f")
printf ' %s %ss %s bytes\n' "$f" "$(printf '%.2f' "$dur")" "$size"
;;
*)
printf ' %s %s bytes\n' "$f" "$size"
;;
esac
done
done
fi