I’m trying to record audio and play it in real time using OpenSL ES, I can send the audio packet to the server, and the server is sending the packet to users correctly. However, there is a problem that I don’t know why. The user who is recording can hear himself (the audio goes through the server and goes back to him), but when another user is speaking, the audio packages arrive and when I will queue the application closes.
Here is the part where I play the audio packets that arrive from the server.
setFila(int playerid, uint8_t* buffer, size_t bufferSize)
{
if (buffer != NULL && playerBufferQueueItf != NULL && playerPlay != NULL && engineEngine != NULL)
{
(*playerBufferQueueItf)->Enqueue(playerBufferQueueItf, buffer, bufferSize); // the application closes here when it receives audio from another Local that is not Localplayer, when the application that is sending is listening works normally ...
}
}
I took this link as a base,
//
// Created by haohao on 2018/1/12.
//
#include <jni.h>
#include <string>
#include <assert.h>
#include <SLES/OpenSLES.h>
#include <SLES/OpenSLES_Android.h>
#include <android/log.h>
#define AUDIO_SRC_PATH "/sdcard/audio.pcm"
#define LOGI(FORMAT, ...) __android_log_print(ANDROID_LOG_INFO,"haohao",FORMAT,##__VA_ARGS__);
#define LOGE(FORMAT, ...) __android_log_print(ANDROID_LOG_ERROR,"haohao",FORMAT,##__VA_ARGS__);
#define NUM_RECORDER_EXPLICIT_INTERFACES 2
#define NUM_BUFFER_QUEUE 1
#define SAMPLE_RATE 44100
#define PERIOD_TIME 20 // 20ms
#define FRAME_SIZE SAMPLE_RATE * PERIOD_TIME / 1000
#define CHANNELS 2
#define BUFFER_SIZE (FRAME_SIZE * CHANNELS)
// engine interfaces
static SLObjectItf engineObject = NULL;
static SLEngineItf engineEngine = NULL;
// audio recorder interfaces
static SLObjectItf recorderObject = NULL;
static SLRecordItf recorderRecord = NULL;
static SLAndroidSimpleBufferQueueItf recorderBuffQueueItf = NULL;
static SLAndroidConfigurationItf configItf = NULL;
// pcm audio player interfaces
static SLObjectItf playerObject = NULL;
static SLPlayItf playerPlay = NULL;
static SLObjectItf outputMixObjext = NULL; // mixer
static SLAndroidSimpleBufferQueueItf playerBufferQueueItf = NULL;
void createEngine(){
SLEngineOption EngineOption[] = {
{(SLuint32) SL_ENGINEOPTION_THREADSAFE, (SLuint32) SL_BOOLEAN_TRUE}
};
SLresult result;
result = slCreateEngine(&engineObject, 1, EngineOption, 0, NULL, NULL);
assert(SL_RESULT_SUCCESS == result);
/* Realizing the SL Engine in synchronous mode. */
result = (*engineObject)->Realize(engineObject, SL_BOOLEAN_FALSE);
assert(SL_RESULT_SUCCESS == result);
// get the engine interface, which is needed in order to create other objects
result = (*engineObject)->GetInterface(engineObject, SL_IID_ENGINE, &engineEngine);
assert(SL_RESULT_SUCCESS == result);
}
class AudioContext {
public:
FILE *pfile;
uint8_t *buffer;
size_t bufferSize;
AudioContext(FILE *pfile, uint8_t *buffer, size_t bufferSize){
this->pfile = pfile;
this->buffer = buffer;
this->bufferSize = bufferSize;
}
};
static AudioContext *recorderContext = NULL;
// Callback when recording audio
void AudioRecorderCallback(SLAndroidSimpleBufferQueueItf bufferQueueItf, void *context){
AudioContext *recorderContext = (AudioContext*)context;
assert(recorderContext != NULL);
if (recorderContext->buffer != NULL) {
fwrite(recorderContext->buffer, recorderContext->bufferSize, 1, recorderContext->pfile);
LOGI("save a frame audio data.");
SLresult result;
SLuint32 state;
result = (*recorderRecord)->GetRecordState(recorderRecord, &state);
assert(SL_RESULT_SUCCESS == result);
(void) result;
if (state == SL_RECORDSTATE_RECORDING) {
result = (*bufferQueueItf)->Enqueue(bufferQueueItf, recorderContext->buffer, recorderContext->bufferSize);
assert(SL_RESULT_SUCCESS == result);
(void) result;
}
}
}
// callback when playing audio
void AudioPlayerCallback(SLAndroidSimpleBufferQueueItf bufferQueueItf, void *context){
AudioContext *playerContext = (AudioContext*)context;
if (!feof(playerContext->pfile)) {
fread(playerContext->buffer, playerContext->bufferSize, 1, playerContext->pfile);
LOGI("read a frame audio data.");
(*bufferQueueItf)->Enqueue(bufferQueueItf, playerContext->buffer, playerContext->bufferSize);
} else {
fclose(playerContext->pfile);
delete playerContext->buffer;
}
}
// Create an audio player
void createAudioPlayer(SLEngineItf engineEngine, SLObjectItf outputMixObject, SLObjectItf &audioPlayerObject){
SLDataLocator_AndroidSimpleBufferQueue dataSourceLocator = {
SL_DATALOCATOR_ANDROIDSIMPLEBUFFERQUEUE,
1
};
// PCM data source format
SLDataFormat_PCM dataSourceFormat = {
SL_DATAFORMAT_PCM,
2,
SL_SAMPLINGRATE_44_1,
SL_PCMSAMPLEFORMAT_FIXED_16,
16,
SL_SPEAKER_FRONT_LEFT| SL_SPEAKER_FRONT_RIGHT,
SL_BYTEORDER_LITTLEENDIAN
};
SLDataSource dataSource = {
&dataSourceLocator,
&dataSourceFormat
};
SLDataLocator_OutputMix dataSinkLocator = {
SL_DATALOCATOR_OUTPUTMIX, // Locator type
outputMixObject // output mix
};
SLDataSink dataSink = {
&dataSinkLocator, // Locator
0,
};
// required interface
SLInterfaceID interfaceIDs[] = {
SL_IID_BUFFERQUEUE
};
SLboolean requiredInterfaces[] = {
SL_BOOLEAN_TRUE
};
// Create an audio playback object
SLresult result = (*engineEngine)->CreateAudioPlayer(
engineEngine,
&audioPlayerObject,
&dataSource,
&dataSink,
1,
interfaceIDs,
requiredInterfaces
);
assert(SL_RESULT_SUCCESS == result);
(void) result;
}
extern "C" {
// Start playing audio
JNIEXPORT void JNICALL
Java_com_haohao_opensl_1es_AudioRecorder_startPlay(JNIEnv *env, jobject instance) {
// Create the engine
if (engineEngine == NULL) {
createEngine();
}
// Create a mixer
SLresult result;
result = (*engineEngine)->CreateOutputMix(engineEngine, &outputMixObjext, 0, 0, 0);
assert(SL_RESULT_SUCCESS == result);
(void) result;
result = (*outputMixObjext)->Realize(outputMixObjext, SL_BOOLEAN_FALSE);
assert(SL_RESULT_SUCCESS == result);
(void) result;
FILE *p_file = fopen(AUDIO_SRC_PATH, "r");
// Create a player
createAudioPlayer(engineEngine, outputMixObjext, playerObject);
result = (*playerObject)->Realize(playerObject, SL_BOOLEAN_FALSE);
assert(SL_RESULT_SUCCESS == result);
(void) result;
result = (*playerObject)->GetInterface(playerObject, SL_IID_BUFFERQUEUE,
&playerBufferQueueItf);
assert(SL_RESULT_SUCCESS == result);
(void) result;
uint8_t *buffer = new uint8_t[BUFFER_SIZE];
AudioContext *playerContext = new AudioContext(p_file, buffer, BUFFER_SIZE);
result = (*playerBufferQueueItf)->RegisterCallback(playerBufferQueueItf, AudioPlayerCallback,
playerContext);
assert(SL_RESULT_SUCCESS == result);
(void) result;
result = (*playerObject)->GetInterface(playerObject, SL_IID_PLAY, &playerPlay);
assert(SL_RESULT_SUCCESS == result);
(void) result;
result = (*playerPlay)->SetPlayState(playerPlay, SL_PLAYSTATE_PLAYING);
assert(SL_RESULT_SUCCESS == result);
AudioPlayerCallback(playerBufferQueueItf, playerContext);
}
// stop playing audio
JNIEXPORT void JNICALL
Java_com_haohao_opensl_1es_AudioRecorder_stopPlay(JNIEnv *env, jobject instance) {
if (playerPlay != NULL) {
SLresult result;
result = (*playerPlay)->SetPlayState(playerPlay, SL_PLAYSTATE_STOPPED);
assert(SL_RESULT_SUCCESS == result);
}
}
// Start collecting audio data and save it locally
JNIEXPORT void JNICALL
Java_com_haohao_opensl_1es_AudioRecorder_startRecord(JNIEnv *env, jobject instance) {
if (engineEngine == NULL) {
createEngine();
}
if (recorderObject != NULL) {
LOGI("Audio recorder already has been created.");
return ;
}
FILE *p_file = fopen(AUDIO_SRC_PATH, "w");
if (p_file == NULL) {
LOGI("Fail to open file.");
return ;
}
SLresult result;
/* setup the data source*/
SLDataLocator_IODevice ioDevice = {
SL_DATALOCATOR_IODEVICE,
SL_IODEVICE_AUDIOINPUT,
SL_DEFAULTDEVICEID_AUDIOINPUT,
NULL
};
SLDataSource recSource = {&ioDevice, NULL};
SLDataLocator_AndroidSimpleBufferQueue recBufferQueue = {
SL_DATALOCATOR_ANDROIDSIMPLEBUFFERQUEUE,
NUM_BUFFER_QUEUE
};
SLDataFormat_PCM pcm = {
SL_DATAFORMAT_PCM, // data in pcm format
2, // 2 channels (stereo)
SL_SAMPLINGRATE_44_1, // sampling frequency of 44100hz
SL_PCMSAMPLEFORMAT_FIXED_16,
SL_PCMSAMPLEFORMAT_FIXED_16,
SL_SPEAKER_FRONT_LEFT| SL_SPEAKER_FRONT_RIGHT,
SL_BYTEORDER_LITTLEENDIAN
};
SLDataSink dataSink = { &recBufferQueue, &pcm };
SLInterfaceID iids[NUM_RECORDER_EXPLICIT_INTERFACES] = {SL_IID_ANDROIDSIMPLEBUFFERQUEUE, SL_IID_ANDROIDCONFIGURATION};
SLboolean required[NUM_RECORDER_EXPLICIT_INTERFACES] = {SL_BOOLEAN_TRUE, SL_BOOLEAN_TRUE};
/* Create the audio recorder */
result = (*engineEngine)->CreateAudioRecorder(engineEngine, &recorderObject , &recSource, &dataSink,
NUM_RECORDER_EXPLICIT_INTERFACES, iids, required);
assert(SL_RESULT_SUCCESS == result);
/* get the android configuration interface*/
result = (*recorderObject)->GetInterface(recorderObject, SL_IID_ANDROIDCONFIGURATION, &configItf);
assert(SL_RESULT_SUCCESS == result);
/* Realize the recorder in synchronous mode. */
result = (*recorderObject)->Realize(recorderObject, SL_BOOLEAN_FALSE);
assert(SL_RESULT_SUCCESS == result);
/* Get the buffer queue interface which was explicitly requested */
result = (*recorderObject)->GetInterface(recorderObject, SL_IID_ANDROIDSIMPLEBUFFERQUEUE, (void*) &recorderBuffQueueItf);
assert(SL_RESULT_SUCCESS == result);
/* get the record interface */
result = (*recorderObject)->GetInterface(recorderObject, SL_IID_RECORD, &recorderRecord);
assert(SL_RESULT_SUCCESS == result);
uint8_t *buffer = new uint8_t[BUFFER_SIZE];
recorderContext = new AudioContext(p_file, buffer, BUFFER_SIZE);
result = (*recorderBuffQueueItf)->RegisterCallback(recorderBuffQueueItf, AudioRecorderCallback, recorderContext);
assert(SL_RESULT_SUCCESS == result);
/* Enqueue buffers to map the region of memory allocated to store the recorded data */
result = (*recorderBuffQueueItf)->Enqueue(recorderBuffQueueItf, recorderContext->buffer, BUFFER_SIZE);
assert(SL_RESULT_SUCCESS == result);
/* Start recording */
// Start recording audio
result = (*recorderRecord)->SetRecordState(recorderRecord, SL_RECORDSTATE_RECORDING);
assert(SL_RESULT_SUCCESS == result);
LOGI("Starting recording");
}
// stop audio collection
JNIEXPORT void JNICALL
Java_com_haohao_opensl_1es_AudioRecorder_stopRecord(JNIEnv *env, jobject instance) {
if (recorderRecord != NULL) {
SLresult result = (*recorderRecord)->SetRecordState(recorderRecord, SL_RECORDSTATE_STOPPED);
assert(SL_RESULT_SUCCESS == result);
if (recorderContext != NULL) {
fclose(recorderContext->pfile);
delete recorderContext->buffer;
}
}
}
// free resources
JNIEXPORT void JNICALL
Java_com_haohao_opensl_1es_AudioRecorder_release(JNIEnv *env, jobject instance) {
if (recorderObject != NULL) {
(*recorderObject)->Destroy(recorderObject);
recorderObject = NULL;
recorderRecord = NULL;
recorderBuffQueueItf = NULL;
configItf = NULL;
recorderContext = NULL;
}
if (playerObject != NULL) {
(*playerObject)->Destroy(playerObject);
playerObject = NULL;
playerPlay = NULL;
playerBufferQueueItf = NULL;
outputMixObjext = NULL;
}
// destroy engine object, and invalidate all associated interfaces
if (engineObject != NULL) {
(*engineObject)->Destroy(engineObject);
engineObject = NULL;
engineEngine = NULL;
}
}
};
it writes to a file and reproduces, my record sends to the server and reproduces in real time. (does not store anything in the file)
My init listen
// Create the engine
if (engineEngine == NULL)
createEngine();
// Create a mixer
SLresult result;
result = (*engineEngine)->CreateOutputMix(engineEngine, &outputMixObjext, 0, 0, 0);
assert(SL_RESULT_SUCCESS == result);
(void)result;
result = (*outputMixObjext)->Realize(outputMixObjext, SL_BOOLEAN_FALSE);
assert(SL_RESULT_SUCCESS == result);
(void)result;
// Create a player
createAudioPlayer(engineEngine, outputMixObjext, playerObject);
result = (*playerObject)->Realize(playerObject, SL_BOOLEAN_FALSE);
assert(SL_RESULT_SUCCESS == result);
(void)result;
result = (*playerObject)->GetInterface(playerObject, SL_IID_BUFFERQUEUE, &playerBufferQueueItf);
assert(SL_RESULT_SUCCESS == result);
(void)result;
result = (*playerObject)->GetInterface(playerObject, SL_IID_PLAY, &playerPlay);
assert(SL_RESULT_SUCCESS == result);
(void)result;
result = (*playerPlay)->SetPlayState(playerPlay, SL_PLAYSTATE_PLAYING);
assert(SL_RESULT_SUCCESS == result);