diff --git a/package.json b/package.json index 9cc72e2..d4c4138 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "license": "UNLICENSED", "dependencies": { "@remotion/cli": "4.0.522", + "@remotion/media-utils": "4.0.522", "@remotion/tailwind-v4": "4.0.522", "react": "19.2.3", "react-dom": "19.2.3", diff --git a/src/song-274/Transect.tsx b/src/song-274/Transect.tsx index bac24dd..66d4d32 100644 --- a/src/song-274/Transect.tsx +++ b/src/song-274/Transect.tsx @@ -19,7 +19,15 @@ import { } from "./camera"; import type { Cue, RosterItem, TransectData } from "./data"; import { extendBounds, GROUND, prefixBounds, projectGround, routePath } from "./layout"; -import { isAwake, nodeLookAt } from "./nodeAnimation"; +import { useWindowedAudioData } from "@remotion/media-utils"; +import { isAwake, mixLooks, nodeLookAt, spectrumLook } from "./nodeAnimation"; +import { + AUDIO_WINDOW_S, + audioWindowFrame, + bandLevels, + shuffledBandFrequencies, + spectrumWeight, +} from "./spectrum"; import type { TransectPalette } from "./palette"; import { coveredRoute, @@ -92,6 +100,17 @@ export const Transect: React.FC = ({ palette, audio, data }) => { ); const parts = useMemo(() => narrationParts(data ? data.lines : []), [data]); const cueMap = useMemo(() => cuesByObject(data ? data.cues : []), [data]); + // Each node gets one log-spaced band, dealt out in a seeded shuffle. + const frequencies = useMemo( + () => shuffledBandFrequencies(data ? data.items.length : 0), + [data], + ); + const { audioData, dataOffsetInSeconds } = useWindowedAudioData({ + src: staticFile(audio), + frame: audioWindowFrame(frame, fps), + fps, + windowInSeconds: AUDIO_WINDOW_S, + }); if (!data || !projection || !kmAt) { return ; @@ -129,10 +148,19 @@ export const Transect: React.FC = ({ palette, audio, data }) => { view, ); - const nodes = data.items.map((item: RosterItem) => ({ - item, - look: nodeLookAt(currentCue(cueMap.get(item.object), t), t), - })); + // In the outro the nodes hand over from their lyric looks to the spectrum. + const spectrum = spectrumWeight(t); + const levels = + spectrum > 0 && audioData + ? bandLevels({ audioData, dataOffsetInSeconds, frame, fps, frequencies }) + : null; + const nodes = data.items.map((item: RosterItem, i) => { + const lyricLook = nodeLookAt(currentCue(cueMap.get(item.object), t), t); + return { + item, + look: levels ? mixLooks(lyricLook, spectrumLook(levels[i]), spectrum) : lyricLook, + }; + }); const draw = ({ item, look }: (typeof nodes)[number]) => ( { return switchOff(t - offS, lightUp(offS - cue.startS)); }; +/** A node at full amplitude in the outro spectrum (see spectrum.ts). */ +export const SPECTRUM_PEAK: NodeLook = { radius: 9, opacity: 1 }; + +/** level scaling helper */ +export const levelExponential = (level: number, exponent: number = 2.0): number => + Math.pow(clamp01(level), exponent); + +/** + * A node in the outro spectrum. + * @param level its band's level, 0 (silence: resting) to 1 (SPECTRUM_PEAK) + */ +export const spectrumLook = (level: number): NodeLook => + mixLooks(RESTING, SPECTRUM_PEAK, clamp01(levelExponential(level))); + /** Whether a look is anything other than resting (drawn on top if so). */ export const isAwake = (look: NodeLook): boolean => look.radius !== RESTING.radius || look.opacity !== RESTING.opacity; diff --git a/src/song-274/spectrum.ts b/src/song-274/spectrum.ts new file mode 100644 index 0000000..bee856a --- /dev/null +++ b/src/song-274/spectrum.ts @@ -0,0 +1,152 @@ +import { visualizeAudio, type MediaUtilsAudioData } from "@remotion/media-utils"; +import { random } from "remotion"; + +/** + * The outro spectrum: from SPECTRUM_START_S the nodes stop following the lyric + * and act as a spectrum analyser. The log-spaced frequency bands are dealt out + * to the nodes in a seeded shuffle (see BAND_SEED), with no relation to where + * a node is on the road; each node's size follows its band's amplitude. + * + * Everything here is a pure function of the frame, so Studio previews and + * renders agree. Starting values were calibrated on the outro of + * 274_release2.wav (8:20 to the end, left channel). + */ + +/** When the spectrum takes over from the lyric-driven looks. */ +export const SPECTRUM_START_S = 504.4; +/** How long the hand-over from lyric looks to spectrum looks takes. */ +export const SPECTRUM_FADE_S = 1.5; + +/** Band centres are log-spaced across this range, one band per node. */ +export const FREQ_LOW_HZ = 40; +export const FREQ_HIGH_HZ = 12000; + +/** + * Seed for the node-to-band shuffle. Any string or number; change it to deal a + * different mapping. Seeded (Remotion's random()) rather than Math.random(), so + * every render tab and the Studio agree on which node gets which band. + */ +export const BAND_SEED = "274-spectrum"; + +/** + * Raw amplitude → level (0–1): (raw × GAIN × (f / FREQ_LOW_HZ)^TREBLE_TILT)^CURVE. + * High bands are far quieter than bass, so TREBLE_TILT lifts them; CURVE below 1 + * lifts quiet passages so more than the loudest bands move. + * + * On the release2 outro these give a median level of 0.35 (10–90% range + * 0.18–0.67), with nodes pinned at full size 0.6% of the time. GAIN 10 pinned + * them 11% of the time with the bass nodes nearly always full; CURVE 0.8 with + * GAIN 8 is livelier (0.15–0.85) but pins 6%. + */ +export const GAIN = 5; +export const TREBLE_TILT = 0.5; +export const CURVE = 0.6; + +/** + * Release: a node's level is the loudest of its last RELEASE_FRAMES frames, each + * frame back multiplied by RELEASE_DECAY once more. Hits jump up at once and + * fall back smoothly, without any state carried between frames. + */ +export const RELEASE_FRAMES = 6; +export const RELEASE_DECAY = 0.75; + +/** FFT size for visualizeAudio: bins are (sample rate / 2) / FFT_SAMPLES wide (23.4 Hz). */ +export const FFT_SAMPLES = 1024; +/** Seconds of audio useWindowedAudioData keeps loaded around the current frame. */ +export const AUDIO_WINDOW_S = 10; + +const clamp01 = (v: number) => Math.min(1, Math.max(0, v)); + +const easeInOutCubic = (p: number) => + p < 0.5 ? 4 * p * p * p : 1 - Math.pow(-2 * p + 2, 3) / 2; + +/** 0 before the spectrum, 1 once the hand-over has finished. */ +export const spectrumWeight = (t: number): number => + easeInOutCubic(clamp01((t - SPECTRUM_START_S) / SPECTRUM_FADE_S)); + +/** + * The frame to load audio around: the real frame once the spectrum is active, + * otherwise the spectrum's start, so nothing is fetched for the rest of the song. + */ +export const audioWindowFrame = (frame: number, fps: number): number => + Math.max(frame, Math.floor((SPECTRUM_START_S - SPECTRUM_FADE_S) * fps)); + +/** `count` log-spaced band centres from FREQ_LOW_HZ to FREQ_HIGH_HZ. */ +export const bandFrequencies = (count: number): number[] => + Array.from({ length: count }, (_, i) => + count === 1 + ? FREQ_LOW_HZ + : FREQ_LOW_HZ * Math.pow(FREQ_HIGH_HZ / FREQ_LOW_HZ, i / (count - 1)), + ); + +/** + * One band centre per node, in node order: bandFrequencies(count) dealt out in a + * Fisher–Yates shuffle seeded by BAND_SEED. + */ +export const shuffledBandFrequencies = (count: number): number[] => { + const frequencies = bandFrequencies(count); + for (let i = frequencies.length - 1; i > 0; i--) { + const j = Math.floor(random(`${BAND_SEED}-${i}`) * (i + 1)); + const swap = frequencies[i]; + frequencies[i] = frequencies[j]; + frequencies[j] = swap; + } + return frequencies; +}; + +/** + * A band's raw amplitude from one spectrum: the mean over the bins between its + * geometric edges, or, for bands narrower than a bin (the bass end), the value + * interpolated at its centre, so neighbouring bass nodes still differ. + */ +const bandValue = (spectrum: number[], centreHz: number, ratio: number, binHz: number) => { + const half = Math.sqrt(ratio); + const low = centreHz / half / binHz; + const high = centreHz * half / binHz; + const first = Math.max(1, Math.ceil(low)); + const last = Math.min(spectrum.length - 1, Math.floor(high)); + if (last >= first) { + let sum = 0; + for (let k = first; k <= last; k++) { + sum += spectrum[k]; + } + return sum / (last - first + 1); + } + const pos = Math.min(spectrum.length - 1.001, Math.max(0, centreHz / binHz)); + const k = Math.floor(pos); + return spectrum[k] + (spectrum[k + 1] - spectrum[k]) * (pos - k); +}; + +/** Every band's level (0–1) at `frame`, release included. */ +export const bandLevels = (args: { + audioData: MediaUtilsAudioData; + dataOffsetInSeconds: number; + frame: number; + fps: number; + frequencies: number[]; +}): number[] => { + const { audioData, dataOffsetInSeconds, frame, fps, frequencies } = args; + const binHz = audioData.sampleRate / 2 / FFT_SAMPLES; + const ratio = + frequencies.length > 1 ? Math.pow(FREQ_HIGH_HZ / FREQ_LOW_HZ, 1 / (frequencies.length - 1)) : 2; + const levels = frequencies.map(() => 0); + + for (let back = 0; back < RELEASE_FRAMES && frame - back >= 0; back++) { + const spectrum = visualizeAudio({ + audioData, + frame: frame - back, + fps, + numberOfSamples: FFT_SAMPLES, + dataOffsetInSeconds, + optimizeFor: "accuracy", + }); + const decay = Math.pow(RELEASE_DECAY, back); + frequencies.forEach((hz, i) => { + const raw = bandValue(spectrum, hz, ratio, binHz); + const level = + Math.pow(clamp01(raw * GAIN * Math.pow(hz / FREQ_LOW_HZ, TREBLE_TILT)), CURVE) * decay; + levels[i] = Math.max(levels[i], level); + }); + } + return levels; +};