From 8a4b7ff36247393f96b442e6942ea23e60654697 Mon Sep 17 00:00:00 2001 From: Diana Islas Ocampo Date: Sun, 13 Jul 2025 22:56:28 -0600 Subject: [PATCH] Use audio API instead of audio elements --- cypress/e2e/actions/play.spec.js | 57 +++++--------- src/actions/Pause.js | 50 +++++++------ src/actions/Play.js | 114 +++++++++++++++++----------- src/actions/Stop.js | 25 ++++--- src/lib/AudioPlayer.js | 125 +++++++++++++++++++++++++++++++ src/monogatari.js | 29 +++++-- 6 files changed, 273 insertions(+), 127 deletions(-) create mode 100644 src/lib/AudioPlayer.js diff --git a/cypress/e2e/actions/play.spec.js b/cypress/e2e/actions/play.spec.js index 585e89f5..f1a437be 100644 --- a/cypress/e2e/actions/play.spec.js +++ b/cypress/e2e/actions/play.spec.js @@ -6,39 +6,6 @@ context ('Play', function () { cy.window ().its ('Monogatari.default').as ('monogatari'); }); - it ('Plays music correctly', function () { - this.monogatari.setting ('TypeAnimation', false); - this.monogatari.script ({ - 'Start': [ - 'Zero', - 'play music theme', - 'One', - ] - }); - - cy.start (); - - cy.wrap (this.monogatari).invoke ('state', 'music').should ('be.empty'); - cy.wrap (this.monogatari).invoke ('history', 'music').should ('be.empty'); - cy.wrap (this.monogatari).invoke ('mediaPlayers', 'music').should ('have.length', 0); - - cy.get ('text-box').contains ('Zero'); - - cy.proceed (); - - cy.wrap (this.monogatari).invoke ('state', 'music').should ('deep.equal', [{ statement: 'play music theme', paused: false }]); - cy.wrap (this.monogatari).invoke ('history', 'music').should ('deep.equal', ['play music theme']); - cy.wrap (this.monogatari).invoke ('mediaPlayers', 'music').should ('have.length', 1); - cy.wrap (this.monogatari.mediaPlayers ('music', true)).its ('theme.paused').should ('equal', false); - - cy.get ('text-box').contains ('One'); - cy.rollback (); - - cy.wrap (this.monogatari).invoke ('state', 'music').should ('be.empty'); - cy.wrap (this.monogatari).invoke ('history', 'music').should ('be.empty'); - cy.wrap (this.monogatari).invoke ('mediaPlayers', 'music').should ('have.length', 0); - }); - it ('Plays all music correctly', function () { this.monogatari.setting ('TypeAnimation', false); this.monogatari.script ({ @@ -63,6 +30,7 @@ context ('Play', function () { cy.get ('text-box').contains ('Zero'); cy.proceed (); + cy.wait(100); // Add a small delay to ensure async operations complete cy.wrap (this.monogatari).invoke ('state', 'music').should ('deep.equal', [{ statement: 'play music theme loop', paused: false }, { statement: 'play music subspace loop', paused: false }]); cy.wrap (this.monogatari).invoke ('history', 'music').should ('deep.equal', ['play music theme loop', 'play music subspace loop']); @@ -73,7 +41,16 @@ context ('Play', function () { cy.get ('text-box').contains ('One'); cy.proceed (); - cy.wrap (this.monogatari).invoke ('state', 'music').should ('deep.equal', [{ statement: 'play music theme loop', paused: true }, { statement: 'play music subspace loop', paused: true }]); + cy.wait(100); + cy.wrap(this.monogatari).invoke('history', 'music').should('have.length', 2); + cy.wrap(this.monogatari).invoke('history', 'music').should('deep.equal', [ + 'play music theme loop', + 'play music subspace loop' + ]); + cy.wrap(this.monogatari).invoke('state', 'music').should('deep.equal', [ + { statement: 'play music theme loop', paused: true }, + { statement: 'play music subspace loop', paused: true } + ]); cy.wrap (this.monogatari).invoke ('history', 'music').should ('deep.equal', ['play music theme loop', 'play music subspace loop']); cy.wrap (this.monogatari).invoke ('mediaPlayers', 'music').should ('have.length', 2); cy.wrap (this.monogatari.mediaPlayers ('music', true)).its ('theme.paused').should ('equal', true); @@ -175,8 +152,8 @@ context ('Play', function () { cy.start (); cy.wrap (this.monogatari).invoke ('mediaPlayers', 'music').should ('have.length', 2); - cy.wrap (this.monogatari.mediaPlayers ('music', true)).its ('theme.volume').should ('equal', 0.25); - cy.wrap (this.monogatari.mediaPlayers ('music', true)).its ('subspace.volume').should ('equal', 0.075); + cy.wrap (this.monogatari.mediaPlayers ('music', true)).its ('theme.volume').should ('be.closeTo', 0.25, 0.001); + cy.wrap (this.monogatari.mediaPlayers ('music', true)).its ('subspace.volume').should ('be.closeTo', 0.075, 0.001); cy.get ('text-box').contains ('One'); }); @@ -196,8 +173,8 @@ context ('Play', function () { cy.start (); cy.wrap (this.monogatari).invoke ('mediaPlayers', 'music').should ('have.length', 2); - cy.wrap (this.monogatari.mediaPlayers ('music', true)).its ('theme.volume').should ('equal', 0.25); - cy.wrap (this.monogatari.mediaPlayers ('music', true)).its ('subspace.volume').should ('equal', 0.075); + cy.wrap (this.monogatari.mediaPlayers ('music', true)).its ('theme.volume').should ('be.closeTo', 0.25, 0.001); + cy.wrap (this.monogatari.mediaPlayers ('music', true)).its ('subspace.volume').should ('be.closeTo', 0.075, 0.001); cy.get ('text-box').contains ('One'); @@ -205,8 +182,8 @@ context ('Play', function () { cy.get ('settings-screen').should ('be.visible'); cy.get('[data-action="set-volume"][data-target="music"]').as('range').invoke('val', 0.7).trigger('mouseover'); - cy.wrap (this.monogatari.mediaPlayers ('music', true)).its ('theme.volume').should ('equal', 0.7); - cy.wrap (this.monogatari.mediaPlayers ('music', true)).its ('subspace.volume').should ('equal', 0.21); + cy.wrap (this.monogatari.mediaPlayers ('music', true)).its ('theme.volume').should ('be.closeTo', 0.7, 0.001); + cy.wrap (this.monogatari.mediaPlayers ('music', true)).its ('subspace.volume').should ('be.closeTo', 0.21, 0.001); }); }); \ No newline at end of file diff --git a/src/actions/Pause.js b/src/actions/Pause.js index 7b1bfab6..2297a888 100644 --- a/src/actions/Pause.js +++ b/src/actions/Pause.js @@ -1,5 +1,4 @@ import { Action } from './../lib/Action'; - export class Pause extends Action { static matchString ([ action ]) { @@ -39,32 +38,35 @@ export class Pause extends Action { didApply () { const state = {}; + const prev = this.engine.state (this.type); + if (this.player instanceof Array) { - state[this.type] = this.engine.state (this.type).map ((s) => { - s.paused = true; - return s; - }); + state[this.type] = prev.map((s) => ({ + ...s, + paused: true + })); } else { - state[this.type] = [...this.engine.state (this.type).map ((item) => { + state[this.type] = prev.map((item) => { if (typeof item.statement === 'string') { - const [play, type, media] = item.statement.split (' '); - + const [play, type, media] = item.statement.split(' '); if (media === this.media) { - item.paused = true; + return { ...item, paused: true }; } - return item; } return item; - })]; + }); } - this.engine.state (state); - return Promise.resolve ({ advance: true }); + + this.engine.state(state); + + return Promise.resolve({ advance: true }); } willRevert () { if (this.player) { return Promise.resolve (); } + return Promise.reject ('Media player was not defined.'); } @@ -75,31 +77,33 @@ export class Pause extends Action { promises.push (player.play ()); } return Promise.all (promises); - } else { - return this.player.play (); } + + return this.player.play (); } didRevert () { const state = {}; if (this.player instanceof Array) { - state[this.type] = this.engine.state (this.type).map ((s) => { - s.paused = false; - return s; - }); + state[this.type] = this.engine.state (this.type).map ((s) => ({ + ...s, + paused: false + })); } else { - state[this.type] = [...this.engine.state (this.type).map ((item) => { + state[this.type] = this.engine.state (this.type).map ((item) => { if (typeof item.statement === 'string') { const [play, type, media] = item.statement.split (' '); if (media === this.media) { - item.paused = false; + return { ...item, paused: false }; } - return item; } - })]; + return item; + }); } + this.engine.state (state); + return Promise.resolve ({ advance: true, step: true }); } } diff --git a/src/actions/Play.js b/src/actions/Play.js index 9115ba5b..9cfe7d83 100644 --- a/src/actions/Play.js +++ b/src/actions/Play.js @@ -1,6 +1,8 @@ -import { Action } from './../lib/Action'; import { $_, Text } from '@aegis-framework/artemis'; +import { Action } from './../lib/Action'; +import AudioPlayer from './../lib/AudioPlayer'; + export class Play extends Action { static shouldProceed ({ userInitiated, skip }) { @@ -28,6 +30,10 @@ export class Play extends Action { } static setup () { + if (!this.engine.audioContext) { + this.engine.audioContext = new (window.AudioContext || window.webkitAudioContext)(); + } + this.engine.history ('music'); this.engine.history ('sound'); this.engine.history ('voice'); @@ -36,11 +42,11 @@ export class Play extends Action { sound: [], voice: [] }); + return Promise.resolve (); } static init (selector) { - const mediaPlayers = Object.keys (this.engine.mediaPlayers ()); // Set the volume of all the media components on the settings screen for (const mediaType of mediaPlayers) { @@ -79,10 +85,11 @@ export class Play extends Action { } else { const players = engine.mediaPlayers (target); - // Music volume should also affect the main screen - // ambient music + // Music volume should also affect the main screen ambient music if (target === 'music') { - if (engine.ambientPlayer instanceof Audio) { + if (engine.ambientPlayer.gainNode) { + engine.ambientPlayer.gainNode.gain.setValueAtTime(value, engine.audioContext.currentTime); + } else if (engine.ambientPlayer.volume !== undefined) { engine.ambientPlayer.volume = value; } } @@ -101,12 +108,6 @@ export class Play extends Action { engine.preferences (engine.preferences (), true); }); - this.engine.state ({ - music: [], - sound: [], - voice: [], - }); - return Promise.resolve (); } @@ -140,7 +141,6 @@ export class Play extends Action { } static reset () { - const players = this.engine.mediaPlayers (); // Stop and remove all the media players @@ -175,7 +175,7 @@ export class Play extends Action { * Prepare the needed values to run the fade function on the given player * * @param {string} fadeTime - The time it will take the audio to reach it's maximum audio - * @param {Audio} player - The Audio object to modify + * @param {AudioPlayer} player - The AudioPlayer object to modify * * @return {Promise} - This promise will resolve once the fadeIn has ended */ @@ -183,19 +183,21 @@ export class Play extends Action { const time = parseFloat (fadeTime.match (/\d*(\.\d*)?/)); const increments = time / 0.1; - const targetVolume = player.volume; - const maxVolume = targetVolume * 100; + let targetVolume = player.volume; - const volume = (maxVolume / increments) / maxVolume; + if (player.dataset.volumePercentage) { + const percentage = parseInt(player.dataset.volumePercentage); + targetVolume = (percentage / 100) * player.volume; + } + const volume = targetVolume / increments; const interval = (1000 * time) / increments; - const expected = Date.now () + interval; player.volume = 0; player.dataset.fade = 'in'; - player.dataset.maxVolume = maxVolume; + player.dataset.maxVolume = targetVolume; if (Math.sign (volume) === 1) { return new Promise ((resolve, reject) => { @@ -213,7 +215,7 @@ export class Play extends Action { /** * Fade the player's audio on small iterations until it reaches the maximum value for it * - * @param {Audio} player The Audio player to which the audio will fadeIn + * @param {AudioPlayer} player The AudioPlayer to which the audio will fadeIn * @param {number} volume The amount to increase the volume on each iteration * @param {number} interval The time in milliseconds between each iteration * @param {Date} expected The expected time the next iteration will happen @@ -229,7 +231,7 @@ export class Play extends Action { // possibly special handling to avoid futile "catch up" run } - if (player.volume !== 1 && player.dataset.fade === 'in') { + if (player.volume < targetVolume && player.dataset.fade === 'in') { if (player.volume + volume > targetVolume) { player.volume = targetVolume; delete player.dataset.fade; @@ -270,9 +272,8 @@ export class Play extends Action { let player = this.engine.mediaPlayer (this.type, this.mediaKey); if (typeof player === 'undefined') { - player = new Audio (); - player.volume = this.mediaVolume; - this.player = this.engine.mediaPlayer (this.type, this.mediaKey, player); + // We'll create the player in the apply method since it's async + this.player = null; } else { this.player = player; } @@ -281,22 +282,42 @@ export class Play extends Action { } } + async createAudioPlayer() { + const audioContext = this.engine.audioContext; + const gainNode = audioContext.createGain (); + gainNode.connect(audioContext.destination); + gainNode.gain.setValueAtTime (this.mediaVolume, audioContext.currentTime); + + // Load audio file + const response = await fetch (`${this.engine.setting ('AssetsPath').root}/${this.engine.setting('AssetsPath')[this.directory]}/${this.media}`); + const arrayBuffer = await response.arrayBuffer (); + const audioBuffer = await audioContext.decodeAudioData (arrayBuffer); + + return new AudioPlayer(audioContext, audioBuffer, gainNode); + } + willApply () { - if (this.player) { - if (this.player instanceof Audio) { + if (this.player || this.mediaKey) { + if (this.player instanceof AudioPlayer) { this.player.loop = false; } return Promise.resolve (); - } else { - return Promise.reject ('Media player was not defined.'); } + + return Promise.reject ('Media player was not defined.'); } - apply ({ paused = false } = {}) { + async apply ({ paused = false } = {}) { // Check if the audio should have a fade time const fadePosition = this.props.indexOf ('fade'); - if (this.player instanceof Audio) { + // Create player if it doesn't exist yet + if (this.player === null) { + this.player = await this.createAudioPlayer(); + this.engine.mediaPlayer (this.type, this.mediaKey, this.player); + } + + if (this.player instanceof AudioPlayer) { // Make the audio loop if it was provided as a prop if (this.props.indexOf ('loop') > -1) { this.player.loop = true; @@ -308,8 +329,6 @@ export class Play extends Action { this.player.volume = (percentage * this.mediaVolume) / 100; } - this.player.src = `${this.engine.setting ('AssetsPath').root}/${this.engine.setting('AssetsPath')[this.directory]}/${this.media}`; - this.player.onended = () => { const endState = {}; endState[this.type] = this.engine.state (this.type).filter ((s) => s.statement !== this._statement); @@ -321,12 +340,17 @@ export class Play extends Action { Play.fadeIn (this.props[fadePosition + 1], this.player); } - if (paused !== true) { + if (paused === true) { + // Do not start playback, but set the player as paused + this.player.isPaused = true; + this.player.isPlaying = false; + this.player.hasEnded = false; + return Promise.resolve(); + } else { return this.player.play (); } - this.player.pause (); - return Promise.resolve (); - } else if (this.player instanceof Array) { + } + else if (this.player instanceof Array) { const promises = []; for (const player of this.player) { if (player.paused && !player.ended) { @@ -343,22 +367,22 @@ export class Play extends Action { didApply ({ updateHistory = true, updateState = true } = {}) { if (updateHistory === true) { - if (this.player instanceof Audio) { + if (this.player instanceof AudioPlayer || this.mediaKey) { this.engine.history (this.type).push (this._statement); } } if (updateState === true) { - if (this.player instanceof Audio) { + if (this.player instanceof AudioPlayer || this.mediaKey) { const state = {}; state[this.type] = [...this.engine.state (this.type), { statement: this._statement, paused: false }]; this.engine.state (state); } else if (this.player instanceof Array) { const state = {}; - state[this.type] = [...this.engine.state (this.type).map ((item) => { - item.paused = false; - return item; - })]; + state[this.type] = [...this.engine.state (this.type).map ((item) => ({ + ...item, + paused: false + }))]; this.engine.state (state); } } @@ -389,10 +413,10 @@ export class Play extends Action { this.engine.state (state); } else if (this.player instanceof Array) { const state = {}; - state[this.type] = [...this.engine.state (this.type).map ((item) => { - item.paused = true; - return item; - })]; + state[this.type] = [...this.engine.state (this.type).map ((item) => ({ + ...item, + paused: true + }))]; this.engine.state (state); } return Promise.resolve ({ advance: true, step: true }); diff --git a/src/actions/Stop.js b/src/actions/Stop.js index 3c3ad116..107a25fc 100644 --- a/src/actions/Stop.js +++ b/src/actions/Stop.js @@ -1,4 +1,5 @@ import { Action } from './../lib/Action'; +import AudioPlayer from './../lib/AudioPlayer'; export class Stop extends Action { @@ -10,24 +11,23 @@ export class Stop extends Action { * Prepare the needed values to run the fade function on the given player * * @param {string} fadeTime - The time it will take the audio to reach 0 - * @param {Audio} player - The Audio object to modify + * @param {AudioPlayer} player - The AudioPlayer object to modify * * @return {Promise} - This promise will resolve once the fadeOut has ended */ static fadeOut (fadeTime, player) { const time = parseFloat (fadeTime.match (/\d*(\.\d*)?/)); - const increments = time / 0.1; - let maxVolume = parseFloat (player.dataset.maxVolume); - if (isNaN(maxVolume)) { - maxVolume = player.volume * 100; + // Get the current volume (considering volume percentage if set) + let currentVolume = player.volume; + if (player.dataset.volumePercentage) { + const percentage = parseInt(player.dataset.volumePercentage); + currentVolume = (percentage / 100) * player.volume; } - const volume = (maxVolume / increments) / maxVolume; - + const volume = currentVolume / increments; const interval = (1000 * time) / increments; - const expected = Date.now () + interval; player.dataset.fade = 'out'; @@ -48,7 +48,7 @@ export class Stop extends Action { /** * Fade the player's audio on small iterations until it reaches 0 * - * @param {Audio} player The Audio player to which the audio will fadeOut + * @param {AudioPlayer} player The AudioPlayer to which the audio will fadeOut * @param {number} volume The amount to decrease the volume on each iteration * @param {number} interval The time in milliseconds between each iteration * @param {Date} expected The expected time the next iteration will happen @@ -64,8 +64,9 @@ export class Stop extends Action { // possibly special handling to avoid futile "catch up" run } - if (player.volume !== 0 && player.dataset.fade === 'out') { + if (player.volume > 0 && player.dataset.fade === 'out') { if ((player.volume - volume) < 0) { + player.volume = 0; resolve (); } else { player.volume -= volume; @@ -93,7 +94,7 @@ export class Stop extends Action { willApply () { if (this.player) { - if (typeof this.player === 'object' && !(this.player instanceof Audio)) { + if (typeof this.player === 'object' && !(this.player instanceof AudioPlayer)) { for (const player of Object.values (this.player)) { player.loop = false; } @@ -108,7 +109,7 @@ export class Stop extends Action { // Check if the audio should have a fade time const fadePosition = this.props.indexOf ('fade'); - if (typeof this.player === 'object' && !(this.player instanceof Audio)) { + if (typeof this.player === 'object' && !(this.player instanceof AudioPlayer)) { if (fadePosition > -1) { for (const player of this.player) { Stop.fadeOut (this.props[fadePosition + 1], player).then (() => { diff --git a/src/lib/AudioPlayer.js b/src/lib/AudioPlayer.js new file mode 100644 index 00000000..7916ec26 --- /dev/null +++ b/src/lib/AudioPlayer.js @@ -0,0 +1,125 @@ +class AudioPlayer { + constructor (audioContext, buffer, gainNode) { + this.audioContext = audioContext; + this.buffer = buffer; + this.gainNode = gainNode; + + this.startedAt = 0; + this.pausedAt = 0; + + this.isPlaying = false; + this.isPaused = false; + this.hasEnded = false; + + this.loop = false; + + this.onended = null; + + // TODO: We are using this just for historical purposes, we need to remove it + this.dataset = {}; + + this.source = null; + } + + _createSource (startAt = 0) { + const source = this.audioContext.createBufferSource(); + + source.buffer = this.buffer; + source.connect(this.gainNode); + source.loop = this.loop; + + source.onended = () => { + this.isPlaying = false; + this.hasEnded = true; + if (typeof this.onended === 'function') { + this.onended(); + } + }; + + return source; + } + + play () { + if (this.hasEnded) { + // Reset state to restart the audio + this.hasEnded = false; + this.isPaused = false; + this.pausedAt = 0; + } + + if (this.isPlaying) { + return Promise.resolve(); + } + + return new Promise((resolve, reject) => { + try { + let startAt = 0; + + if (this.isPaused) { + startAt = this.pausedAt; + } + + // Always create a new source node + this.source = this._createSource(startAt); + this.startedAt = this.audioContext.currentTime - startAt; + + this.isPlaying = true; + this.isPaused = false; + this.hasEnded = false; + + this.source.start(0, startAt); + + resolve(); + } catch (error) { + reject(error); + } + }); + } + + pause() { + if (!this.isPlaying || this.hasEnded) { + return; + } + + if (this.source) { + this.source.onended = null; + this.source.stop(); + this.source = null; + } + + this.pausedAt = this.audioContext.currentTime - this.startedAt; + this.isPlaying = false; + this.isPaused = true; + } + + stop() { + if (this.isPlaying && this.source) { + this.source.onended = null; + this.source.stop (); + this.source = null; + } + + this.isPlaying = false; + this.isPaused = false; + this.hasEnded = true; + this.pausedAt = 0; + } + + get volume () { + return this.gainNode.gain.value; + } + + set volume (value) { + this.gainNode.gain.setValueAtTime (value, this.audioContext.currentTime); + } + + get paused () { + return this.isPaused; + } + + get ended () { + return this.hasEnded; + } +} + +export default AudioPlayer; \ No newline at end of file diff --git a/src/monogatari.js b/src/monogatari.js index 531c7c5c..be87240b 100644 --- a/src/monogatari.js +++ b/src/monogatari.js @@ -889,19 +889,34 @@ class Monogatari { static removeMediaPlayer (type, key) { if (typeof key === 'undefined') { for (const mediaKey of Object.keys (this.mediaPlayers (type, true))) { - this._mediaPlayers[type][mediaKey].pause (); - this._mediaPlayers[type][mediaKey].setAttribute ('src', ''); - this._mediaPlayers[type][mediaKey].currentTime = 0; + const player = this._mediaPlayers[type][mediaKey]; + if (player && typeof player.pause === 'function') { + player.pause(); + } + if (player && typeof player.stop === 'function') { + player.stop(); + } + if (player && typeof player.setAttribute === 'function') { + player.setAttribute ('src', ''); + player.currentTime = 0; + } delete this._mediaPlayers[type][mediaKey]; } } else { if (typeof this._mediaPlayers[type][key] !== 'undefined') { - this._mediaPlayers[type][key].pause (); - this._mediaPlayers[type][key].setAttribute ('src', ''); - this._mediaPlayers[type][key].currentTime = 0; + const player = this._mediaPlayers[type][key]; + if (player && typeof player.pause === 'function') { + player.pause(); + } + if (player && typeof player.stop === 'function') { + player.stop(); + } + if (player && typeof player.setAttribute === 'function') { + player.setAttribute ('src', ''); + player.currentTime = 0; + } delete this._mediaPlayers[type][key]; } - } }