Extracting audio from a video file

Question

Edit : This post is no duplicate of mine. I am trying to extract the audio data as binary, got no problems with playing the audio file as separate as I mentioned before.

I am trying to extract audio from a video file on client-side by using Web Audio Api .

var audioContext = new(window.AudioContext || window.webkitAudioContext)();
fileData = new Blob([input.files[0]]);
var videoFileAsBuffer = new Promise(getBuffer);
videoFileAsBuffer.then(function (data) {
    audioContext.decodeAudioData(data).then(function (decodedAudioData) {
        mySoundBuffer = decodedAudioData;
        soundSource = audioContext.createBufferSource();
        soundSource.buffer = mySoundBuffer;
        // soundSource.connect(audioContext.destination);
        // soundSource.start();
    });

When I uncomment the two lines at the end, I am hearing the sound of the uploaded video file. Though, when I create a link to download the file with the help of getChannelData method, it's almost the same size as video file.

I was expecting decodedAudioData to have only audio binary data, and send that to my webservice, which is the only one I need. However, that didn't work out as I expected. Anyone knows a way to extract audio of a video file on client-side? Thanks in advance.

Here is the getBuffer method in case anyone wants to know:

function getBuffer(resolve) {
    var reader = new FileReader();
    reader.onload = function () {
        var arrayBuffer = reader.result;
        resolve(arrayBuffer);
    }
    reader.readAsArrayBuffer(fileData);
}

Edit : I was able to decode the video file and get audiobuffer by using OfflineAudioContext inside decodeAudioData function.

var offlineAudioContext = new OfflineAudioContext(2, 44100 * 100, 44100);
var soundSource = offlineAudioContext.createBufferSource();
...
soundSource.connect(offlineAudioContext.destination);
soundSource.start();
offlineAudioContext.startRendering().then(function (renderedBuffer) {
    console.log(renderedBuffer); // outputs audiobuffer
    var song = audioContext.createBufferSource();
    song.buffer = renderedBuffer;
    song.connect(audioContext.destination);
    song.start();
}).catch(function (err) {
    console.log('Rendering failed: ' + err);
});

renderedBuffer is an audiobuffer, had no problem outputting the data, tested with Audacity 's import raw data option. But the problem is, the new file (filled with renderedBuffer.getChannelData(0) ) has higher size than the original video has. Isn't that supposed to have lower size since it only contains audio of the video file?

Answer 1

Ok, I actually had the answer already. Raw audio data is huge, that's why it's even greater than the video itself in size.

var offlineAudioContext = new OfflineAudioContext(numberOfChannels, sampleRate * duration, sampleRate);
var soundSource = offlineAudioContext.createBufferSource();
...
reader.readAsArrayBuffer(blob); // video file
reader.onload = function () {
  var videoFileAsBuffer = reader.result; // arraybuffer
  audioContext.decodeAudioData(videoFileAsBuffer).then(function (decodedAudioData) {
    myBuffer = decodedAudioData;
    soundSource.buffer = myBuffer;
    soundSource.connect(offlineAudioContext.destination);
    soundSource.start();

    offlineAudioContext.startRendering().then(function (renderedBuffer) {
      console.log(renderedBuffer); // outputs audiobuffer
    }).catch(function (err) {
      console.log('Rendering failed: ' + err);
    });
  });
};

After that, I was able to convert the audiobuffer (renderedbuffer) into a wav file by using audiobuffer-to-wav library. OfflineAudioContext is just needed to modify the cropped audio.

Edit: Here is the js fiddle example. decodedAudioData method will suffice if you don't want to override the audio data.

Answer 2

I think you should change your code from

var soundSource = offlineAudioContext.createBufferSource();
......
soundSource.connect(offlineAudioContext.destination);

to these

 const soundSource = audioContext.createBufferSource()
 soundSource.buffer = decodedAudioData
 soundSource.connect(audioContext.destination)

in order to play the audio

Answer 3

I'm using this code to extract audio:

// initialize the audioContext
var audioContext = new webkitAudioContext();
var video = document.getElementById("myVideo");
var mediaSource = audioContext.createMediaElementSource(video);
var analyser = audioContext.createAnalyser();
mediaSource.connect(analyser);
analyser.connect(audioContext.destination);

---

// this will give you the sound data

video.play();

function getSoundData() {
   var sample = new Float32Array(analyser.frequencyBinCount);
   return analyser.getFloatFrequencyData(sample);  
}

EDIT:

I've not run this code so far but I think this should work also:

Fetching an XMLHttpRequest to your videourl (working also with local files) instead of using WebAudio API.

var xhr = new XMLHttpRequest();
xhr.open('GET', 'https://test.com/myvideo.mp4', true);
xhr.responseType = 'blob';

xhr.onload = function(e) {
    var binaryData = this.response;
    console.log(binaryData)
};

xhr.send();

EDIT 2:

 var audioFileUrl = 'video.mp3'; fetch(audioFileUrl) .then(function(res) { res.blob().then(function(blob) { var size = blob.size; var type = blob.type; var reader = new FileReader(); reader.addEventListener("loadend", function() { var base64FileData = reader.result.toString(); var mediaFile = { fileUrl: audioFileUrl, size: blob.size, type: blob.type, src: base64FileData }; }); reader.readAsDataURL(blob); }); }); 

or try using some kind of code like this:

<audio id="audio" src="http://yourvideo.com/video.mp4" controls autoplay></audio>


function getAudio() {
  var elem = document.getElementById("audio")

  // get Base 64 string from elem.src
}

I hope it is helpful because for me these few lines are working quite fine so far :)