Can I use Javascript to stream audio sound from Pepper's microphone using ALAudioDevice (setClientPreferences and then subscribe)?
Question
I was successfully streaming audio sound from Pepper's microphone to my notebook using python:
class SoundReceiverModule(naoqi.ALModule):
.......
def start( self ):
audio = naoqi.ALProxy( "ALAudioDevice", self.strNaoIp, 9559 );
nNbrChannelFlag = 3;
nDeinterleave = 0;
nSampleRate = 48000;
audio.setClientPreferences( self.getName(), nSampleRate, nNbrChannelFlag, nDeinterleave );
audio.subscribe( self.getName() );
def processRemote( self, nbOfChannels, nbrOfSamplesByChannel, aTimeStamp, buffer ):
aSoundDataInterlaced = np.fromstring( str(buffer), dtype=np.int16 );
aSoundData = np.reshape( aSoundDataInterlaced, (nbOfChannels, nbrOfSamplesByChannel), 'F' );
.......
However, when I do it using Javascript, no signal was generated after subscribing the ALAudioDevice. Can anyone help me to solve this problem? Thanks!
4 answers
solution1
0 2020-06-14 10:58:14
I can't test this at the moment sorry, but I hope this serves as a starting point. Please let me know if this works, or if not and you get any errors.
class SoundReceiverModule {
constructor(){
}
processRemote(nbOfChannels, nbrOfSamplesByChannel, buffer, timeStamp){
console.log(buffer)
}
}
const channels = 3;
const deinterleaved = 0;
const sampleRate = 48000;
const moduleName = SoundReceiverModule.name;
// with /libs/qimessaging/2/qimessaging.js included
QiSession.connect(function (session) {
session.service("ALAudioDevice").then(function (ALAudioDevice) {
ALAudioDevice.setClientPreferences(moduleName, sampleRate, channels, deinterleaved);
ALAudioDevice.subscribe(moduleName);
});
});
solution2
0 2020-06-20 04:49:01
Here's a different approach I'll add as an alternate answer because it's more of a workaround. The idea is to use the SoundReceiverModule in Python like you have, then publish the sound data in an event which can be received by the javascript APIs. We will pass this data through the new ALMemory event, sound-data-processed .
You'll need to make a Choregraphe app that looks like this. Note that the web page must be stored as html/index.html .
In index.html put the following. Of course, you can move this code to a separate.js file and include it in the html.
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
<title>SoundReceiverModuleTest</title>
</head>
<body style="background: black;">
<script src="/libs/qimessaging/2/qimessaging.js"></script>
<script type="text/javascript">
QiSession.connect(function (session) {
session.service("ALMemory").then(function (ALMemory) {
ALMemory.subscriber("sound-data-processed").then(function (subscriber){
subscriber.signal.connect(function (data){
// here is the sound data from the Python event
// alert will show it on Pepper's tablet screen
alert(data);
});
});
});
});
</script>
</body>
</html>
I've slightly modified the example Python audio module from the docs here . You can run it from an external computer (if you have the python qi sdk installed) like so:
python audio_module.py --ip <nao-ip>
Where audio_module.py is how I've saved the below file, and <nao-ip> is the IP address of your Pepper on the same network as your computer.
#! /usr/bin/env python
# -*- encoding: UTF-8 -*-
"""Example: Get Signal from Front Microphone & Calculate its rms Power"""
import qi
import argparse
import sys
import time
import numpy as np
class SoundProcessingModule(object):
"""
A simple get signal from the front microphone of Nao & calculate its rms power.
It requires numpy.
"""
def __init__( self, app):
"""
Initialise services and variables.
"""
super(SoundProcessingModule, self).__init__()
app.start()
session = app.session
# Get the service ALAudioDevice.
self.audio_service = session.service("ALAudioDevice")
self.memory_service = session.service('ALMemory')
self.isProcessingDone = False
self.nbOfFramesToProcess = 20
self.framesCount=0
self.micFront = []
self.module_name = "SoundProcessingModule"
def startProcessing(self):
"""
Start processing
"""
# ask for the front microphone signal sampled at 16kHz
# if you want the 4 channels call setClientPreferences(self.module_name, 48000, 0, 0)
self.audio_service.setClientPreferences(self.module_name, 16000, 3, 0)
self.audio_service.subscribe(self.module_name)
while self.isProcessingDone == False:
time.sleep(1)
self.audio_service.unsubscribe(self.module_name)
def processRemote(self, nbOfChannels, nbOfSamplesByChannel, timeStamp, inputBuffer):
"""
Publish sound data to ALMemory event.
"""
self.framesCount = self.framesCount + 1
if (self.framesCount <= self.nbOfFramesToProcess):
# convert inputBuffer to signed integer as it is interpreted as a string by python
soundData = self.convertStr2SignedInt(inputBuffer)
# send the data to an ALMemory event that can be read in Javascript
self.memory_service.raise_event('sound-data-processed', soundData)
else :
self.isProcessingDone=True
def convertStr2SignedInt(self, data) :
"""
This function takes a string containing 16 bits little endian sound
samples as input and returns a vector containing the 16 bits sound
samples values converted between -1 and 1.
"""
signedData=[]
ind=0;
for i in range (0,len(data)/2) :
signedData.append(data[ind]+data[ind+1]*256)
ind=ind+2
for i in range (0,len(signedData)) :
if signedData[i]>=32768 :
signedData[i]=signedData[i]-65536
for i in range (0,len(signedData)) :
signedData[i]=signedData[i]/32768.0
return signedData
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--ip", type=str, default="127.0.0.1",
help="Robot IP address. On robot or Local Naoqi: use '127.0.0.1'.")
parser.add_argument("--port", type=int, default=9559,
help="Naoqi port number")
args = parser.parse_args()
try:
# Initialize qi framework.
connection_url = "tcp://" + args.ip + ":" + str(args.port)
app = qi.Application(["SoundProcessingModule", "--qi-url=" + connection_url])
except RuntimeError:
print ("Can't connect to Naoqi at ip \"" + args.ip + "\" on port " + str(args.port) +".\n"
"Please check your script arguments. Run with -h option for help.")
sys.exit(1)
MySoundProcessingModule = SoundProcessingModule(app)
app.session.registerService("SoundProcessingModule", MySoundProcessingModule)
MySoundProcessingModule.startProcessing()
solution3
0 2020-06-22 01:29:25
Error generated when the choregraphe was run:
[ERROR] behavior.box:_safeCallOfUserMethod:125 _Behavior__lastUploadedChoregrapheBehaviorbehavior_11662506592:/ALSoundDevice_1: Traceback (most recent call last): File "/opt/aldebaran/lib/python2.7/site-packages/albehavior.py", line 115, in _safeCallOfUserMethod func() File "<string>", line 31, in onUnload File "/opt/aldebaran/lib/python2.7/site-packages/ialbehavior.py", line 108, in <lambda> __getattr__ = lambda self, name: _swig_getattr(self, behavior, name) File "/opt/aldebaran/lib/python2.7/site-packages/ialbehavior.py", line 57, in _swig_getattr raise AttributeError(name) AttributeError: soundReceiver [ERROR] behavior.box:_safeCallOfUserMethod:125 _Behavior__lastUploadedChoregrapheBehaviorbehavior_11662506592:/ALSoundDevice_1: Traceback (most recent call last): File "/opt/aldebaran/lib/python2.7/site-packages/albehavior.py", line 115, in _safeCallOfUserMethod func() File "<string>", line 34, in onInput_onStart TypeError: __init__() takes exactly 2 arguments (1 given)
However, when I added an input name to self.soundReceiver = SoundReceiverModule( 'SoundReceiver' ), then pepper's shoulder leds turned yellow (then red) after running the choregraphe and no signal was received on browsing the webpage http://pepper-ip/apps/....
solution4
0 2020-06-26 03:33:30
The problem was partly solved:
import numpy as np
class SoundReceiverModule(ALModule):
def __init__( self, strModuleName, strNaoIp ):
try:
ALModule.__init__(self, strModuleName )
self.BIND_PYTHON( self.getName(), "callback" )
self.strNaoIp = strNaoIp
except BaseException, err:
print( "loading error: %s" % str(err) )
def __del__( self ):
print( "SoundReceiverModule.__del__: cleaning everything" );
self.stop();
def start(self):
audio = ALProxy("ALAudioDevice", self.strNaoIp, 9559)
nNbrChannelFlag = 3
nDeinterleave = 0
nSampleRate = 48000
audio.setClientPreferences(self.getName(), nSampleRate, nNbrChannelFlag, nDeinterleave)
audio.subscribe(self.getName())
def stop(self):
audio = ALProxy("ALAudioDevice", self.strNaoIp, 9559)
audio.unsubscribe(self.getName())
def processRemote(self, nbOfChannels, nbrOfSamplesByChannel, aTimeStamp, buffer):
self.memory = ALProxy("ALMemory")
aSoundDataInterlaced = np.fromstring(str(buffer), dtype=np.int16)
aSoundData = np.reshape(aSoundDataInterlaced, (nbOfChannels, nbrOfSamplesByChannel), 'F')
self.memory.raiseEvent('sound-data-processed', aSoundData)
class MyClass(GeneratedClass):
def __init__(self):
GeneratedClass.__init__(self)
self.myBroker = ALBroker("myBroker", "0.0.0.0", 0, "192.168.24.201", 9559)
def onLoad(self):
#put initialization code here
pass
def onUnload(self):
#put clean-up code here
SoundReceiver.stop()
def onInput_onStart(self):
global SoundReceiver
SoundReceiver = SoundReceiverModule("SoundReceiver", "192.168.24.201")
SoundReceiver.start()
def onInput_onStop(self):
self.myBroker.shutdown()
self.onUnload()
def onStopped():
self.onStopped()
However, the unsolved problem is 'no sound data streamed from subscribing the SoundReceiver'.
The technical post webpages of this site follow the CC BY-SA 4.0 protocol. If you need to reprint, please indicate the site URL or the original address.Any question please contact:yoyou2525@163.com.