diff --git a/packages/javascript/cypress/component/AudioFrequency.cy.ts b/packages/javascript/cypress/component/AudioFrequency.cy.ts new file mode 100644 index 0000000..ec6968b --- /dev/null +++ b/packages/javascript/cypress/component/AudioFrequency.cy.ts @@ -0,0 +1,150 @@ +import { MediaPreloader } from '../../src/state-based/MediaPreloader'; +import { SurfaceManager } from '../../src/state-based/SurfaceManager'; + +const constructAssetURL = (file: string) => `http://localhost:5173/__cypress/iframes/cypress/fixtures/${file}`; + +const AUDIO_OUTPUT = ''; // Default +const EXPECTED_HZ = 440; +const HZ_ε = 2; +const MIN_VOLUME_SILENCE = 0.1; +const VOLUME_ε = 0.01; + +async function analyzeAudio(audioContext: AudioContext, gainNode: GainNode): Promise<{ hz: number | undefined; volume: number }> { + const fftSize = 8192; + await audioContext.resume(); + + const analyser = audioContext.createAnalyser(); + analyser.fftSize = fftSize; + gainNode.connect(analyser); + + // Give the audio graph time to actually process samples before reading the analyser - + // right after connecting, its internal buffer is still empty/zeroed. + await new Promise((resolve) => setTimeout(resolve, 500)); + + const frequencyData = new Uint8Array(analyser.frequencyBinCount); + analyser.getByteFrequencyData(frequencyData); + + let peakBin = 0; + let peakMagnitude = 0; + for (let bin = 0; bin < frequencyData.length; bin++) { + if (frequencyData[bin] > peakMagnitude) { + peakMagnitude = frequencyData[bin]; + peakBin = bin; + } + } + + gainNode.disconnect(analyser); + const volume = peakMagnitude / 255; + const hz = volume > 0.1 ? (peakBin * audioContext.sampleRate) / fftSize : undefined; + return { hz, volume }; +} + +describe('Audio frequency verification tests', () => { + it('plays a real 440Hz tone', () => { + const now = Date.now(); + const preloader = new MediaPreloader(constructAssetURL); + const manager = new SurfaceManager( + constructAssetURL, + { + 'clip-id': { + type: 'audio', + file: 'sinwave@440hz.wav', + audioOutput: AUDIO_OUTPUT, + enablePlaybackRateAdjustment: true, + keyframes: [[now, { set: { t: 0, rate: 1 } }]], + }, + }, + preloader, + ); + cy.mount(manager); + + // wait to start playing + cy.get('audio') + .invoke('prop', 'currentTime') + .should(($time) => expect(parseFloat($time)).to.be.greaterThan(0.1)); + + cy.get('audio').then(async ($audio) => { + const gainNode = preloader.getGainNode($audio.get(0) as HTMLAudioElement)!; + const { hz } = await analyzeAudio(preloader.getAudioContext(AUDIO_OUTPUT), gainNode); + expect(hz).to.be.closeTo(EXPECTED_HZ, HZ_ε); + }); + }); + + it('changes volume', () => { + const now = Date.now(); + const preloader = new MediaPreloader(constructAssetURL); + const manager = new SurfaceManager( + constructAssetURL, + { + 'clip-id': { + type: 'audio', + file: 'sinwave@440hz.wav', + audioOutput: AUDIO_OUTPUT, + enablePlaybackRateAdjustment: true, + keyframes: [[now, { set: { t: 0, rate: 1, volume: 1 } }]], + }, + }, + preloader, + ); + cy.mount(manager); + + let fullVolumePeak = 0; + cy.get('audio') + .invoke('prop', 'currentTime') + .should(($time) => expect(parseFloat($time)).to.be.greaterThan(0.1)) + .then(async () => { + const audioElement = manager.element.querySelector('audio')!; + const gainNode = preloader.getGainNode(audioElement)!; + const { volume } = await analyzeAudio(preloader.getAudioContext(AUDIO_OUTPUT), gainNode); + expect(volume, 'full clip volume should be clearly audible').to.be.greaterThan(MIN_VOLUME_SILENCE); + fullVolumePeak = volume; + }) + .then(() => { + manager.setState({ + 'clip-id': { + type: 'audio', + file: 'sinwave@440hz.wav', + audioOutput: AUDIO_OUTPUT, + enablePlaybackRateAdjustment: true, + keyframes: [[now, { set: { t: 0, rate: 1, volume: 0.5 } }]], + }, + }); + }) + .wait(200) + .then(async () => { + const audioElement = manager.element.querySelector('audio')!; + const gainNode = preloader.getGainNode(audioElement)!; + const { volume } = await analyzeAudio(preloader.getAudioContext(AUDIO_OUTPUT), gainNode); + expect(volume, 'volume at 0.2 should be at least 50% quieter').to.be.lessThan(fullVolumePeak); + }); + }); + + it('is silent when playing at 0 volume', () => { + const now = Date.now(); + const preloader = new MediaPreloader(constructAssetURL); + const manager = new SurfaceManager( + constructAssetURL, + { + 'clip-id': { + type: 'audio', + file: 'sinwave@440hz.wav', + audioOutput: AUDIO_OUTPUT, + enablePlaybackRateAdjustment: true, + keyframes: [[now, { set: { t: 0, rate: 1, volume: 0 } }]], + }, + }, + preloader, + ); + cy.mount(manager); + + cy.get('audio') + .invoke('prop', 'currentTime') + .should(($time) => expect(parseFloat($time)).to.be.greaterThan(0.1)); + + cy.get('audio').then(async ($audio) => { + const gainNode = preloader.getGainNode($audio.get(0) as HTMLAudioElement)!; + const { volume } = await analyzeAudio(preloader.getAudioContext(AUDIO_OUTPUT), gainNode); + expect(volume).to.be.closeTo(0, VOLUME_ε); + }); + }); +}); diff --git a/packages/javascript/cypress/component/AudioStability.cy.ts b/packages/javascript/cypress/component/AudioStability.cy.ts index e7ace21..1fb8b0e 100644 --- a/packages/javascript/cypress/component/AudioStability.cy.ts +++ b/packages/javascript/cypress/component/AudioStability.cy.ts @@ -1,11 +1,10 @@ import { SurfaceManager } from '../../src/state-based/SurfaceManager'; const constructAssetURL = (file: string) => `http://localhost:5173/__cypress/iframes/cypress/fixtures/${file}`; -const getAudioOutput = () => ''; describe('Audio stability tests', () => { it('can wait without playing', () => { const now = Date.now(); - const manager = new SurfaceManager(constructAssetURL, getAudioOutput, { + const manager = new SurfaceManager(constructAssetURL, { 'clip-id': { file: 'sinwave@440hz.wav', type: 'audio', @@ -26,7 +25,7 @@ describe('Audio stability tests', () => { it('recovers from a pause', () => { const now = Date.now(); - const manager = new SurfaceManager(constructAssetURL, getAudioOutput, { + const manager = new SurfaceManager(constructAssetURL, { 'clip-id': { file: 'metronome@120bpm.wav', type: 'audio', @@ -51,7 +50,7 @@ describe('Audio stability tests', () => { it('recovers from a playbackRate change', () => { const now = Date.now(); - const manager = new SurfaceManager(constructAssetURL, getAudioOutput, { + const manager = new SurfaceManager(constructAssetURL, { 'clip-id': { file: 'metronome@120bpm.wav', type: 'audio', @@ -83,7 +82,7 @@ describe('Audio stability tests', () => { it('recovers from a seek', () => { const now = Date.now(); - const manager = new SurfaceManager(constructAssetURL, getAudioOutput, { + const manager = new SurfaceManager(constructAssetURL, { 'clip-id': { file: 'metronome@120bpm.wav', type: 'audio', @@ -104,31 +103,34 @@ describe('Audio stability tests', () => { }); it('recovers from volume change', () => { - const INITIAL_VOLUME = 0; - const CHANGED_VOLUME = 1; + /** + * Note this test checks for recovery of the volume property on the audioElement. + * The volume should always be 1, and the gain node will control the volume from there. + */ const now = Date.now(); - const manager = new SurfaceManager(constructAssetURL, getAudioOutput, { + const manager = new SurfaceManager(constructAssetURL, { 'clip-id': { type: 'audio', file: 'sinwave@440hz.wav', audioOutput: '', enablePlaybackRateAdjustment: true, - keyframes: [[now, { set: { t: 0, rate: 1, volume: INITIAL_VOLUME } }]], + keyframes: [[now, { set: { t: 0, rate: 1, volume: 0.5 } }]], }, }); cy.mount(manager); - cy.get('audio').invoke('prop', 'volume', CHANGED_VOLUME); - cy.get('audio').should('have.prop', 'volume', CHANGED_VOLUME); + cy.get('audio').should('have.prop', 'volume', 1); + cy.get('audio').invoke('prop', 'volume', 0.2); + cy.get('audio').should('have.prop', 'volume', 0.2); cy.wait(1000); - cy.get('audio').should('have.prop', 'volume', INITIAL_VOLUME); + cy.get('audio').should('have.prop', 'volume', 1); }); it('recovers from audio element deletion', () => { const now = Date.now(); - const manager = new SurfaceManager(constructAssetURL, getAudioOutput, { + const manager = new SurfaceManager(constructAssetURL, { 'clip-id': { type: 'audio', file: 'sinwave@440hz.wav', @@ -150,7 +152,7 @@ describe('Audio stability tests', () => { it('toggles looping', () => { const now = Date.now(); - const manager = new SurfaceManager(constructAssetURL, getAudioOutput, { + const manager = new SurfaceManager(constructAssetURL, { 'clip-id': { type: 'audio', file: 'sinwave@440hz.wav', diff --git a/packages/javascript/cypress/component/ImageStability.cy.ts b/packages/javascript/cypress/component/ImageStability.cy.ts index 5d95eca..5ca1b0a 100644 --- a/packages/javascript/cypress/component/ImageStability.cy.ts +++ b/packages/javascript/cypress/component/ImageStability.cy.ts @@ -1,11 +1,10 @@ import { SurfaceManager } from '../../src/state-based/SurfaceManager'; const constructAssetURL = (file: string) => `http://localhost:5173/__cypress/iframes/cypress/fixtures/${file}`; -const getAudioOutput = () => ''; describe('Image stability tests', () => { it('can show an image', () => { const now = Date.now(); - const manager = new SurfaceManager(constructAssetURL, getAudioOutput, { + const manager = new SurfaceManager(constructAssetURL, { 'clip-id': { file: 'indianred@2560x1440.png', type: 'image', @@ -19,7 +18,7 @@ describe('Image stability tests', () => { it("doesn't show a queued image", () => { const now = Date.now(); - const manager = new SurfaceManager(constructAssetURL, getAudioOutput, { + const manager = new SurfaceManager(constructAssetURL, { 'clip-id': { file: 'indianred@2560x1440.png', type: 'image', @@ -37,7 +36,7 @@ describe('Image stability tests', () => { const ORIGINAL_SRC = 'indianred@2560x1440.png'; const CHANGED_SRC = '404.png'; const now = Date.now(); - const manager = new SurfaceManager(constructAssetURL, getAudioOutput, { + const manager = new SurfaceManager(constructAssetURL, { 'clip-id': { file: ORIGINAL_SRC, type: 'image', @@ -59,7 +58,7 @@ describe('Image stability tests', () => { it('recovers from img element deletion', () => { const now = Date.now(); - const manager = new SurfaceManager(constructAssetURL, getAudioOutput, { + const manager = new SurfaceManager(constructAssetURL, { 'clip-id': { file: 'indianred@2560x1440.png', type: 'image', diff --git a/packages/javascript/cypress/component/SurfaceLayers.cy.ts b/packages/javascript/cypress/component/SurfaceLayers.cy.ts index ce9b342..3aa9253 100644 --- a/packages/javascript/cypress/component/SurfaceLayers.cy.ts +++ b/packages/javascript/cypress/component/SurfaceLayers.cy.ts @@ -3,11 +3,10 @@ const INDIAN_RED = { r: 191, g: 99, b: 96 }; const ROYAL_BLUE = { r: 75, g: 104, b: 218 }; const constructAssetURL = (file: string) => `http://localhost:5173/__cypress/iframes/cypress/fixtures/${file}`; -const getAudioOutput = () => ''; describe('Surface layer tests', () => { it('can take a known screenshot', () => { const now = Date.now(); - const manager = new SurfaceManager(constructAssetURL, getAudioOutput, { + const manager = new SurfaceManager(constructAssetURL, { red: { file: 'indianred@2560x1440.png', type: 'image', @@ -29,7 +28,7 @@ describe('Surface layer tests', () => { it('respects z-index', () => { const now = Date.now(); - const manager = new SurfaceManager(constructAssetURL, getAudioOutput, { + const manager = new SurfaceManager(constructAssetURL, { red: { file: 'indianred@2560x1440.png', type: 'image', diff --git a/packages/javascript/cypress/component/UpdatingSurfaceState.cy.ts b/packages/javascript/cypress/component/UpdatingSurfaceState.cy.ts index 7ac3d3b..5923ba4 100644 --- a/packages/javascript/cypress/component/UpdatingSurfaceState.cy.ts +++ b/packages/javascript/cypress/component/UpdatingSurfaceState.cy.ts @@ -1,10 +1,9 @@ import { DATA_CLIP_ID, SurfaceManager } from '../../src/state-based/SurfaceManager'; const constructAssetURL = (file: string) => `http://localhost:5173/__cypress/iframes/cypress/fixtures/${file}`; -const getAudioOutput = () => ''; describe('Updating surface state', () => { it('adds and removes a video clip', () => { - const manager = new SurfaceManager(constructAssetURL, getAudioOutput, {}); + const manager = new SurfaceManager(constructAssetURL, {}); cy.mount(manager); cy.get('video') @@ -38,7 +37,7 @@ describe('Updating surface state', () => { it('adds multiple media', () => { const now = Date.now(); - const manager = new SurfaceManager(constructAssetURL, getAudioOutput, {}); + const manager = new SurfaceManager(constructAssetURL, {}); cy.mount(manager); expect(manager.element.children.length).to.eq(0); diff --git a/packages/javascript/cypress/component/VideoStability.cy.ts b/packages/javascript/cypress/component/VideoStability.cy.ts index 6515bfa..3d653c4 100644 --- a/packages/javascript/cypress/component/VideoStability.cy.ts +++ b/packages/javascript/cypress/component/VideoStability.cy.ts @@ -1,11 +1,10 @@ import { SurfaceManager } from '../../src/state-based/SurfaceManager'; const constructAssetURL = (file: string) => `http://localhost:5173/__cypress/iframes/cypress/fixtures/${file}`; -const getAudioOutput = () => ''; describe('Video stability tests', () => { it('can wait without playing', () => { const now = Date.now(); - const manager = new SurfaceManager(constructAssetURL, getAudioOutput, { + const manager = new SurfaceManager(constructAssetURL, { 'clip-id': { file: 'libx264~yuv420p~60fps~10s@1280x720.mp4', type: 'video', @@ -27,7 +26,7 @@ describe('Video stability tests', () => { it('recovers from a pause', () => { const now = Date.now(); - const manager = new SurfaceManager(constructAssetURL, getAudioOutput, { + const manager = new SurfaceManager(constructAssetURL, { 'clip-id': { file: 'libx264~yuv420p~60fps~10s@1280x720.mp4', type: 'video', @@ -53,7 +52,7 @@ describe('Video stability tests', () => { it('recovers from a playbackRate change', () => { const now = Date.now(); - const manager = new SurfaceManager(constructAssetURL, getAudioOutput, { + const manager = new SurfaceManager(constructAssetURL, { 'clip-id': { file: 'libx264~yuv420p~60fps~10s@1280x720.mp4', type: 'video', @@ -86,7 +85,7 @@ describe('Video stability tests', () => { it('recovers from a seek', () => { const now = Date.now(); - const manager = new SurfaceManager(constructAssetURL, getAudioOutput, { + const manager = new SurfaceManager(constructAssetURL, { 'clip-id': { file: 'libx264~yuv420p~60fps~10s@1280x720.mp4', type: 'video', @@ -108,32 +107,35 @@ describe('Video stability tests', () => { }); it('recovers from volume change', () => { - const INITIAL_VOLUME = 0; - const CHANGED_VOLUME = 1; + /** + * Note this test checks for recovery of the volume property on the videoElement. + * The volume should always be 1, and the gain node will control the volume from there. + */ const now = Date.now(); - const manager = new SurfaceManager(constructAssetURL, getAudioOutput, { + const manager = new SurfaceManager(constructAssetURL, { 'clip-id': { type: 'video', file: 'libx264~yuv420p~60fps~10s@1280x720.mp4', audioOutput: '', fit: 'cover', enablePlaybackRateAdjustment: true, - keyframes: [[now, { set: { t: 0, rate: 1, volume: INITIAL_VOLUME } }]], + keyframes: [[now, { set: { t: 0, rate: 1, volume: 0.5 } }]], }, }); cy.mount(manager); - cy.get('video').invoke('prop', 'volume', CHANGED_VOLUME); - cy.get('video').should('have.prop', 'volume', CHANGED_VOLUME); + cy.get('video').should('have.prop', 'volume', 1); + cy.get('video').invoke('prop', 'volume', 0.2); + cy.get('video').should('have.prop', 'volume', 0.2); cy.wait(1000); - cy.get('video').should('have.prop', 'volume', INITIAL_VOLUME); + cy.get('video').should('have.prop', 'volume', 1); }); it('recovers from video element deletion', () => { const now = Date.now(); - const manager = new SurfaceManager(constructAssetURL, getAudioOutput, { + const manager = new SurfaceManager(constructAssetURL, { 'clip-id': { type: 'video', file: 'libx264~yuv420p~60fps~10s@1280x720.mp4', @@ -156,7 +158,7 @@ describe('Video stability tests', () => { it('toggles looping', () => { const now = Date.now(); - const manager = new SurfaceManager(constructAssetURL, getAudioOutput, { + const manager = new SurfaceManager(constructAssetURL, { 'clip-id': { type: 'video', file: 'libx264~yuv420p~60fps~10s@1280x720.mp4', diff --git a/packages/javascript/cypress/mount.ts b/packages/javascript/cypress/mount.ts index ad6400b..c029074 100644 --- a/packages/javascript/cypress/mount.ts +++ b/packages/javascript/cypress/mount.ts @@ -12,7 +12,7 @@ export function mount(surfaceManager: SurfaceManager): Cypress.Chainable { // clean up each time we mount a new component container.innerHTML = ''; const prevManager = (window as any).surfaceManager as SurfaceManager; - prevManager?.setState({}); + prevManager?.destroy(); // mount component (window as any).surfaceManager = surfaceManager; diff --git a/packages/javascript/src/state-based/MediaClipManager.ts b/packages/javascript/src/state-based/MediaClipManager.ts index 2d68e60..bc13fa0 100644 --- a/packages/javascript/src/state-based/MediaClipManager.ts +++ b/packages/javascript/src/state-based/MediaClipManager.ts @@ -34,7 +34,6 @@ export abstract class MediaClipManager { protected clipElement: HTMLElement, state: T, protected constructAssetURL: (file: string) => string, - protected getAudioOutput: (outputLabel: string) => string, protected mediaPreloader: MediaPreloader, ) { this._state = state; @@ -115,7 +114,7 @@ export function assertElement( } if (!element) { - element = preloader.getElement(clip.file, clip.type); + element = preloader.getElement(clip.file, clip.type, clip.audioOutput); } // Required for iOS @@ -162,7 +161,7 @@ export function assertVisualProperties( * Makes sure that the element sounds correct. * - It should have the right volume, and play out the correct speaker. */ -export function assertAudialProperties(mediaElement: HTMLMediaElement, properties: AudialProperties, sinkId: string, surfaceVolume: number) { +export function assertAudialProperties(mediaElement: HTMLMediaElement, gainNode: GainNode, properties: AudialProperties, surfaceVolume: number) { const clipVolume = properties.volume * surfaceVolume; if (IS_IOS) { // For iOS devices HTMLMediaElement.volume is readonly @@ -176,19 +175,10 @@ export function assertAudialProperties(mediaElement: HTMLMediaElement, propertie if (mediaElement.muted) { mediaElement.muted = false; } - if (mediaElement.volume !== clipVolume) { - mediaElement.volume = clipVolume; - } - if (mediaElement.sinkId !== sinkId) { - try { - mediaElement.setSinkId(sinkId).catch(() => { - /* Do nothing, will be tried in next loop */ - }); - // eslint-disable-next-line @typescript-eslint/no-unused-vars - } catch (_) { - /* Do nothing, will be tried in next loop */ - } + if (mediaElement.volume !== 1) { + mediaElement.volume = 1; } + gainNode.gain.value = properties.volume * surfaceVolume; } } @@ -448,8 +438,10 @@ export class AudioManager extends MediaClipManager { if (!currentState || !this.audioElement) return; - const sinkId = this.getAudioOutput(this._state.audioOutput); - assertAudialProperties(this.audioElement, currentState as AudialProperties, sinkId, this.volume); + const gainNode = this.mediaPreloader.getGainNode(this.audioElement); + if (gainNode) { + assertAudialProperties(this.audioElement, gainNode, currentState as AudialProperties, this.volume); + } const nextSyncState = assertTemporalProperties( this.audioElement, currentState as TemporalProperties, @@ -463,9 +455,12 @@ export class AudioManager extends MediaClipManager { public destroy(): void { if (this.audioElement) { + const gainNode = this.mediaPreloader.getGainNode(this.audioElement); + if (gainNode) { + gainNode.gain.value = 0; + } this.audioElement.pause(); this.audioElement.remove(); - this.audioElement.volume = 0; this.audioElement.currentTime = 0; this.mediaPreloader.releaseElement(this.audioElement); } @@ -495,9 +490,11 @@ export class VideoManager extends MediaClipManager { if (!currentState || !this.videoElement) return; - const sinkId = this.getAudioOutput(this._state.audioOutput); assertVisualProperties(this.videoElement, currentState as VisualProperties, this._state.fit); - assertAudialProperties(this.videoElement, currentState as AudialProperties, sinkId, this.volume); + const gainNode = this.mediaPreloader.getGainNode(this.videoElement); + if (gainNode) { + assertAudialProperties(this.videoElement, gainNode, currentState as AudialProperties, this.volume); + } const nextSyncState = assertTemporalProperties( this.videoElement, currentState as TemporalProperties, @@ -511,9 +508,12 @@ export class VideoManager extends MediaClipManager { public destroy(): void { if (this.videoElement) { + const gainNode = this.mediaPreloader.getGainNode(this.videoElement); + if (gainNode) { + gainNode.gain.value = 0; + } this.videoElement.pause(); this.videoElement.remove(); - this.videoElement.volume = 0; this.videoElement.currentTime = 0; this.mediaPreloader.releaseElement(this.videoElement); } diff --git a/packages/javascript/src/state-based/MediaPreloader.ts b/packages/javascript/src/state-based/MediaPreloader.ts index 611bfee..0c066ae 100644 --- a/packages/javascript/src/state-based/MediaPreloader.ts +++ b/packages/javascript/src/state-based/MediaPreloader.ts @@ -1,12 +1,37 @@ +import '../types/AudioContext'; import { MediaClientConfig } from '../types/CogsClientMessage'; +interface Media { + element: HTMLMediaElement; + type: 'audio' | 'video'; + inUse: boolean; + gainNode: GainNode | undefined; +} + +interface MediaPool { + [fileName: string]: { + spare: Media; + connected: { [audioOutput: string]: Media }; + }; +} + +const DEFAULT_AUDIO_OUTPUT = ''; + +/** + * Preloads audio and video to optimize time to playback. + * Lazily connects media elements to the required AudioContext, and keeps a spare one unconnected. + */ export class MediaPreloader { private _state: MediaClientConfig['files']; - private _elements: Record = {}; + private _mediaPool: MediaPool = {}; private _constructAssetURL: (file: string) => string; + private _audioOutputIds: Record = {}; + private _audioContext: AudioContext = new AudioContext(); + private _audioOutput: string = DEFAULT_AUDIO_OUTPUT; constructor(constructAssetURL: (file: string) => string, testState: MediaClientConfig['files'] = {}) { this._constructAssetURL = constructAssetURL; this._state = testState; + navigator?.mediaDevices?.addEventListener('devicechange', this._updateAudioOutputs); } get state() { @@ -17,80 +42,134 @@ export class MediaPreloader { this.update(); } + getAudioContext(audioOutput: string): AudioContext { + if (audioOutput === this._audioOutput) { + this._audioContext.resume(); + return this._audioContext; + } else { + this._audioContext.close(); + const ctx = new AudioContext(); + this._audioOutput = audioOutput; + this._audioContext = ctx; + this._audioContext.resume(); + const sinkId = this._audioOutputIds[audioOutput] ?? ''; + ctx.setSinkId?.(sinkId); + return ctx; + } + } + + getGainNode(element: HTMLMediaElement): GainNode | undefined { + for (const cache of Object.values(this._mediaPool)) { + for (const media of Object.values(cache.connected)) { + if (media.element === element) return media.gainNode; + } + } + } + + private getPreloadAttr(fileName: string): 'auto' | 'metadata' | 'none' { + switch (this._state[fileName]?.preload) { + case 'auto': + case true: + return 'auto'; + case 'metadata': + return 'metadata'; + default: + return 'none'; + } + } + private update() { - // Clean up previous elements - for (const [filename, media] of Object.entries(this._elements)) { + // Remove stale elements + for (const [filename, cache] of Object.entries(this._mediaPool)) { if (!(filename in this._state)) { - if (media.inUse) { - console.warn(`Failed to clean up element ${media.element.src}`); - continue; + cache.spare.element.src = ''; + cache.spare.element.load(); + for (const media of Object.values(cache.connected)) { + if (media.inUse) { + console.error(`Failed to clean up ${filename}`); + } else { + media.element.src = ''; + media.element.load(); + media.gainNode?.disconnect(); + } } - media.element.src = ''; - media.element.load(); - delete this._elements[filename]; + delete this._mediaPool[filename]; } - media.inUse = media.element.isConnected; } + // Create cache for new clips for (const [filename, fileConfig] of Object.entries(this._state)) { - if (filename in this._elements) { - continue; - } - // Create new elements - let preloadAttr: 'auto' | 'metadata' | 'none'; - if (fileConfig.preload === true) { - preloadAttr = 'auto'; - } else if (fileConfig.preload === false) { - preloadAttr = 'none'; - } else { - preloadAttr = fileConfig.preload; - } - switch (fileConfig.type) { - case 'audio': { - const element = document.createElement('audio'); - element.src = this._constructAssetURL(filename); - element.preload = preloadAttr; - this._elements[filename] = { element, inUse: false, type: 'audio' }; - break; - } - case 'video': { - const element = document.createElement('video'); - element.src = this._constructAssetURL(filename); - element.preload = preloadAttr; - this._elements[filename] = { element, inUse: false, type: 'video' }; - break; - } + const cache = this._mediaPool[filename]; + if (!cache || !cache.spare) { + cache.spare = this.createMedia(filename, fileConfig.type); } } } - getElement(file: string, type: 'audio' | 'video') { - const media = this._elements[file]; - if (media && media.inUse === false) { - media.inUse = true; - return media.element; - } else { - const element = document.createElement(type); - element.src = this._constructAssetURL(file); - if (type === 'video') { - this._elements[file] = { element, type, inUse: true }; - } - return element; + private createMedia(file: string, type: 'audio' | 'video'): Media { + const element = document.createElement(type); + element.src = this._constructAssetURL(file); + element.preload = this.getPreloadAttr(file); + return { element, type, inUse: false, gainNode: undefined }; + } + + // Connects an element into the Web Audio graph. Must only be called once per element. + private connectElement(media: Media, audioOutput: string) { + const ctx = this.getAudioContext(audioOutput); + const source = ctx.createMediaElementSource(media.element); + const gainNode = ctx.createGain(); + source.connect(gainNode); + gainNode.connect(ctx.destination); + media.gainNode = gainNode; + } + + getElement(file: string, type: 'audio' | 'video', audioOutput: string) { + const cache = this._mediaPool[file] ?? (this._mediaPool[file] = { connected: {}, spare: this.createMedia(file, type) }); + + // Reuse element if already connected to audio graph + const connectedMedia = cache.connected[audioOutput]; + if (connectedMedia && !connectedMedia.inUse) { + connectedMedia.inUse = true; + return connectedMedia.element; } + + // Use spare if available, connect to graph + const ready = cache.spare; + cache.spare = this.createMedia(file, type); + ready.inUse = true; + this.connectElement(ready, audioOutput); + cache.connected[audioOutput] ??= ready; + return ready.element; } - releaseElement(resource: string | HTMLElement) { - if (typeof resource === 'string') { - const media = this._elements[resource]; - if (media) { - media.inUse = false; + releaseElement(element: HTMLMediaElement) { + for (const cache of Object.values(this._mediaPool)) { + for (const media of Object.values(cache.connected)) { + if (media.element === element) media.inUse = false; } - } else { - Object.values(this._elements).forEach((media) => { - if (media.element === resource) { - media.inUse = false; - } - }); } } + + private _updateAudioOutputs = async () => { + const audioOutputIds: Record = {}; + + if (!navigator?.mediaDevices) { + // `navigator.mediaDevices` is undefined on COGS AV <= 4.5 because of secure origin permissions + return; + } + + const devices = await navigator.mediaDevices.enumerateDevices(); + const outputs = devices.filter((device) => device.kind === 'audiooutput'); + outputs.forEach((output) => { + audioOutputIds[output.label] = output.deviceId; + }); + + this._audioOutputIds = audioOutputIds; + }; + + destroy() { + this._audioContext.close(); + this._mediaPool = {}; + navigator?.mediaDevices?.removeEventListener('devicechange', this._updateAudioOutputs); + } } diff --git a/packages/javascript/src/state-based/SurfaceManager.ts b/packages/javascript/src/state-based/SurfaceManager.ts index 77d88da..5d200a4 100644 --- a/packages/javascript/src/state-based/SurfaceManager.ts +++ b/packages/javascript/src/state-based/SurfaceManager.ts @@ -39,9 +39,8 @@ export class SurfaceManager { constructor( private constructAssetUrl: (file: string) => string, - private getAudioOutput: (outputLabel: string) => string, testState?: MediaSurfaceState, - private mediaPreloader: MediaPreloader = new MediaPreloader(constructAssetUrl), + private _mediaPreloader: MediaPreloader = new MediaPreloader(constructAssetUrl), ) { this._element = document.createElement('div'); this._element.className = 'surface-manager'; @@ -91,39 +90,18 @@ export class SurfaceManager { if (!resource.manager) { switch (clip.type) { case 'image': - resource.manager = new ImageManager( - this._element, - resource.element, - clip, - this.constructAssetUrl, - this.getAudioOutput, - this.mediaPreloader, - ); + resource.manager = new ImageManager(this._element, resource.element, clip, this.constructAssetUrl, this._mediaPreloader); resource.manager.loop(); break; case 'audio': { - const audioManager = new AudioManager( - this._element, - resource.element, - clip, - this.constructAssetUrl, - this.getAudioOutput, - this.mediaPreloader, - ); + const audioManager = new AudioManager(this._element, resource.element, clip, this.constructAssetUrl, this._mediaPreloader); resource.manager = audioManager; audioManager.volume = this._volume; audioManager.loop(); break; } case 'video': { - const videoManager = new VideoManager( - this._element, - resource.element, - clip, - this.constructAssetUrl, - this.getAudioOutput, - this.mediaPreloader, - ); + const videoManager = new VideoManager(this._element, resource.element, clip, this.constructAssetUrl, this._mediaPreloader); resource.manager = videoManager; videoManager.volume = this._volume; videoManager.loop(); @@ -135,4 +113,9 @@ export class SurfaceManager { } }); } + + public destroy() { + this.setState({}); + this._mediaPreloader.destroy(); + } } diff --git a/packages/javascript/src/types/AudioContext.ts b/packages/javascript/src/types/AudioContext.ts new file mode 100644 index 0000000..19d8de1 --- /dev/null +++ b/packages/javascript/src/types/AudioContext.ts @@ -0,0 +1,11 @@ +// The Audio Output Devices API (https://webaudio.github.io/web-audio-api/#dom-audiocontext-setsinkid) +// is not yet part of TypeScript's lib.dom.d.ts. Chrome 110+ only; absent entirely on Safari/Firefox, +// so callers must feature-detect with `typeof audioContext.setSinkId === 'function'`. +export {}; + +declare global { + interface AudioContext { + setSinkId?(sinkId: string | { type: 'none' }): Promise; + readonly sinkId?: string | { type: 'none' }; + } +} diff --git a/packages/react/src/components/MediaSurface.tsx b/packages/react/src/components/MediaSurface.tsx index c1d61d8..caa6f09 100644 --- a/packages/react/src/components/MediaSurface.tsx +++ b/packages/react/src/components/MediaSurface.tsx @@ -11,28 +11,6 @@ export function MediaSurface({ cogsConnection }: MediaSurfaceProps) { const mediaPreloaderRef = useRef(null); const [surfaceElem, setSurfaceElem] = useState(null); - // Keep updated list of audio outputs - const audioOutputs = useRef>({}); - useEffect(() => { - async function updateAudioOutputs() { - audioOutputs.current = {}; - if (!navigator?.mediaDevices) { - // `navigator.mediaDevices` is undefined on COGS AV <= 4.5 because of secure origin permissions - return; - } - - const devices = await navigator.mediaDevices.enumerateDevices(); - const outputs = devices.filter((device) => device.kind === 'audiooutput'); - outputs.forEach((output) => { - audioOutputs.current[output.label] = output.deviceId; - }); - } - - updateAudioOutputs(); - navigator?.mediaDevices?.addEventListener('devicechange', updateAudioOutputs); - return () => navigator?.mediaDevices?.removeEventListener('devicechange', updateAudioOutputs); - }, []); - // Create and attach new surface manager useEffect(() => { const constructURL = (url: string) => cogsConnection.getAssetUrl(url); @@ -44,7 +22,7 @@ export function MediaSurface({ cogsConnection }: MediaSurfaceProps) { preloader.setState(files); } - const sm = new SurfaceManager(constructURL, (outputLabel: string) => audioOutputs.current[outputLabel] ?? '', {}, preloader); + const sm = new SurfaceManager(constructURL, {}, preloader); if (volumeRef.current !== undefined) { sm.volume = volumeRef.current; } @@ -52,7 +30,7 @@ export function MediaSurface({ cogsConnection }: MediaSurfaceProps) { surfaceElem?.replaceChildren(sm.element); return () => { - surfaceManagerRef.current?.setState({}); + surfaceManagerRef.current?.destroy(); surfaceManagerRef.current = undefined; surfaceElem?.replaceChildren(/* empty */); };