WebRTC录音(1)-实现通话双向录音

最近公司的iPad项目中一个功能点涉及到了VOIP通讯中的录音,需要在已有的WebRTC引擎中增加录音功能,录制通话双方的声音
参考了往上一位兄弟的博文(链接在此 http://blog.csdn.net/darkinger/article/details/13627479),代码实现基本问题不大,就是由于WebRTC本身版本更新导致部分代码要改动下结构;
但是那位兄台的代码存在两个问题
1 在一处地方没有描述清楚,导致混音不可用.后面仔细跟踪了下,调整下顺序即可.
2 录制出来的文件默认是PCM16K的裸数据,而不是通话使用的编码方式,在这里走了一天弯路(还得怪自己懒,其实去看下代码就知道了)

OK,以下是我做的全盘修改:

//////////////voe_file.h///////////////
基类增加两个虚函数接口,用于启停录音调用

    //DECWANG_4RECORD 20140814
    virtual int StartRecordingPlayoutAndMic(const char* fileNameUTF8,
                                            CodecInst* compression = NULL,
                                            int maxSizeBytes = -1) = 0;
    virtual int StopRecordingPlayoutAndMic() = 0;
    /**/

//////////////voe_file_impl.h///////////////

子类定义增加两个虚函数接口,用于启停录音调用

    virtual int StartRecordingPlayoutAndMic(const char* fileNameUTF8,
                                            CodecInst* compression = NULL,
                                            int maxSizeBytes = -1);
    virtual int StopRecordingPlayoutAndMic();

 //////////////voe_file_impl.cc///////////////

int VoEFileImpl::StartRecordingPlayoutAndMic(const char* fileNameUTF8, CodecInst* compression,int maxSizeBytes)
{
    //DECWANG_4RECORD,用于启动录音进程
    WEBRTC_TRACE(kTraceApiCall, kTraceVoice, VoEId(_shared->instance_id(),-1),
                 "StartRecordingPlayoutAndMic(fileNameUTF8=%s, "
                 "compression, maxSizeBytes=%d)",
                 fileNameUTF8, maxSizeBytes);
    assert(1024 == FileWrapper::kMaxFileNameSize);
    
    if (!_shared->statics()->Initialized())
    {
        _shared->statics()->SetLastError(VE_NOT_INITED, kTraceError);
        return -1;
    }
     _shared->outputall_mixer()->StartRecordingPlayout(fileNameUTF8, compression);
    return 0;
}
    
int VoEFileImpl::StopRecordingPlayoutAndMic()
{
    //DECWANG_4RECORD ,用于停止录音进程   
    WEBRTC_TRACE(kTraceApiCall, kTraceVoice, VoEId(_shared->instance_id(),-1),
                     "StopRecordingPlayoutAndMic");
    if (!_shared->statics()->Initialized())
    {
        _shared->statics()->SetLastError(VE_NOT_INITED, kTraceError);
        return -1;
    }
    return _shared->outputall_mixer()->StopRecordingPlayout();
}

 //////////////share_data.h///////////////
增加一个处理双向数据的混音对象,并增加对应的对象访问接口

public:
    //DECWANG_4RECORD
    OutputMixer* outputall_mixer() { return _outputAllMixerPtr; }
    Statistics* statics() {return &_engineStatistics;}
protected:    
    //DECWANG_4RECORD
    OutputMixer* _outputAllMixerPtr;    //for Record speaker+mic

 //////////////share_data.cc///////////////
初始化

SharedData::SharedData() :
    _instanceId(++_gInstanceCounter),
    _apiCritPtr(CriticalSectionWrapper::CreateCriticalSection()),
    _channelManager(_gInstanceCounter),
    _engineStatistics(_gInstanceCounter),
    _audioDevicePtr(NULL),
    audioproc_(NULL),
    _moduleProcessThreadPtr(ProcessThread::CreateProcessThread()),
    _externalRecording(false),
    _externalPlayout(false)
{
    Trace::CreateTrace();
    if (OutputMixer::Create(_outputMixerPtr, _gInstanceCounter) == 0)
    {
        _outputMixerPtr->SetEngineInformation(_engineStatistics);
    }
    if (TransmitMixer::Create(_transmitMixerPtr, _gInstanceCounter) == 0)
    {
        _transmitMixerPtr->SetEngineInformation(*_moduleProcessThreadPtr,
                                                _engineStatistics,
                                                _channelManager);
    }
    //DECWANG_4RECORD
    if (OutputMixer::Create(_outputAllMixerPtr, _gInstanceCounter) == 0)
    {
        _outputAllMixerPtr->SetEngineInformation(_engineStatistics);
    }
    /**/

    _audioDeviceLayer = AudioDeviceModule::kPlatformDefaultAudio;
}

 //////////////audio_conference_mixer_defines.h///////////////
增加一个处理混音的数据类

//DECWANG_4RECORD
//for Record speaker+mic
//record mic or playout signal from OutputMixer output
class AudioFrameMixerPart:public MixerParticipant
{
public:
    AudioFrameMixerPart();
    void SetAudioFrame(AudioFrame &audioFrame);
    uint16_t GetPayloadDataLengthInSamples();
public:
    // From MixerParticipant
    int32_t GetAudioFrame(const int32_t id,AudioFrame& audioFrame);
    int32_t NeededFrequency(const int32_t id);
private:
        AudioFrame _audioFrame;
};

 //////////////audio_conference_mixer_impl.cc///////////////
实现混音数据类的必备接口,原有代码是在.h中实现,为了干净,干脆直接全部移到cc中

AudioFrameMixerPart::AudioFrameMixerPart()
{
}
void AudioFrameMixerPart::SetAudioFrame(AudioFrame &audioFrame)
{
    _audioFrame.CopyFrom(audioFrame);
}
uint16_t AudioFrameMixerPart::GetPayloadDataLengthInSamples()
{
    return _audioFrame.samples_per_channel_;
}
int32_t AudioFrameMixerPart::GetAudioFrame(const int32_t id,AudioFrame& audioFrame)
{
    if (_audioFrame.samples_per_channel_ <= 0)
        return -1;
            
    audioFrame.CopyFrom(_audioFrame);
    return 0;
};
int32_t AudioFrameMixerPart::NeededFrequency(const int32_t id)
{
    return _audioFrame.sample_rate_hz_;
}

 //////////////output_mixer.h///////////////
增加一个共有成员函数,用于返回数据帧

public:
    AudioFrame* GetAudioFrame();
//////////////output_mixer.cc///////////////
//DECWANG_4RECORD
AudioFrame* OutputMixer::GetAudioFrame()
{
    return &_audioFrame;
}

 //////////////transmit_mixer.h///////////////
增加成员函数

public:
    AudioFrame* GetAudioFrame();
//////////////transmit_mixer.cc///////////////
AudioFrame* TransmitMixer::GetAudioFrame()
{
    return &_audioFrame;
}

 //////////////voe_base_impl.h///////////////
头文件引用,增加
#include "webrtc/modules/audio_conference_mixer/interface/audio_conference_mixer_defines.h"
私有成员

private:
    AudioFrameMixerPart _afmTransmitMixer;
    AudioFrameMixerPart _afmOutputMixer;     
//////////////voe_base_impl.cc///////////////
int32_t VoEBaseImpl::RecordedDataIsAvailable(
        const void* audioSamples,
        uint32_t nSamples,
        uint8_t nBytesPerSample,
        uint8_t nChannels,
        uint32_t samplesPerSec,
        uint32_t totalDelayMS,
        int32_t clockDrift,
        uint32_t currentMicLevel,
        bool keyPressed,
        uint32_t& newMicLevel)
{
    WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_shared->instance_id(), -1),
                 "VoEBaseImpl::RecordedDataIsAvailable(nSamples=%u, "
                     "nBytesPerSample=%u, nChannels=%u, samplesPerSec=%u, "
                     "totalDelayMS=%u, clockDrift=%d, currentMicLevel=%u)",
                 nSamples, nBytesPerSample, nChannels, samplesPerSec,
                 totalDelayMS, clockDrift, currentMicLevel);
    newMicLevel = static_cast<uint32_t>(ProcessRecordedDataWithAPM(
        NULL, 0, audioSamples, samplesPerSec, nChannels, nSamples,
        totalDelayMS, clockDrift, currentMicLevel, keyPressed));
    //for Record speaker+mic
    //DECWANG_4RECORD,用于拷贝已有的音频帧,用于下一步的混音
    _afmTransmitMixer.SetAudioFrame(*(_shared->transmit_mixer()->GetAudioFrame()));
    

    return 0;
}

int32_t VoEBaseImpl::NeedMorePlayData(
        uint32_t nSamples,
        uint8_t nBytesPerSample,
        uint8_t nChannels,
        uint32_t samplesPerSec,
        void* audioSamples,
        uint32_t& nSamplesOut)
{
    WEBRTC_TRACE(kTraceStream, kTraceVoice, VoEId(_shared->instance_id(), -1),
                 "VoEBaseImpl::NeedMorePlayData(nSamples=%u, "
                     "nBytesPerSample=%d, nChannels=%d, samplesPerSec=%u)",
                 nSamples, nBytesPerSample, nChannels, samplesPerSec);

    assert(_shared->output_mixer() != NULL);
    
    //DECWANG_4RECORD,获取音频帧,与资料出处的区别就在于此.
    //for Record speaker+mic
    _afmOutputMixer.SetAudioFrame(*(_shared->output_mixer()->GetAudioFrame()));
    /**/
    
    // TODO(andrew): if the device is running in mono, we should tell the mixer
    // here so that it will only request mono from AudioCodingModule.
    // Perform mixing of all active participants (channel-based mixing)
    _shared->output_mixer()->MixActiveChannels();

    // Additional operations on the combined signal
    _shared->output_mixer()->DoOperationsOnCombinedSignal();

    // Retrieve the final output mix (resampled to match the ADM)
    _shared->output_mixer()->GetMixedAudio(samplesPerSec, nChannels,
        &_audioFrame);

    assert(static_cast<int>(nSamples) == _audioFrame.samples_per_channel_);
    assert(samplesPerSec ==
        static_cast<uint32_t>(_audioFrame.sample_rate_hz_));

    // Deliver audio (PCM) samples to the ADM
    memcpy(
           (int16_t*) audioSamples,
           (const int16_t*) _audioFrame.data_,
           sizeof(int16_t) * (_audioFrame.samples_per_channel_
                   * _audioFrame.num_channels_));

    nSamplesOut = _audioFrame.samples_per_channel_;
    //DECWANG_4RECORD,用于实际混音动作
    //for Record speaker+mic
    if (_afmOutputMixer.GetPayloadDataLengthInSamples() == _afmTransmitMixer.GetPayloadDataLengthInSamples())
    {
        AudioFrame audioFrameX;
        _shared->outputall_mixer()->MixActiveChannels();
        _shared->outputall_mixer()->DoOperationsOnCombinedSignal();
        _shared->outputall_mixer()->GetMixedAudio(samplesPerSec, nChannels, &audioFrameX);
    }
    /**/
    return 0;
}
int VoEBaseImpl::Init(AudioDeviceModule* external_adm,
                      AudioProcessing* audioproc)
{
    WEBRTC_TRACE(kTraceApiCall, kTraceVoice, VoEId(_shared->instance_id(), -1),
        "Init(external_adm=0x%p)", external_adm);
    CriticalSectionScoped cs(_shared->crit_sec());

    WebRtcSpl_Init();

    if (_shared->statistics().Initialized())
    {
        return 0;
    }

    if (_shared->process_thread())
    {
        if (_shared->process_thread()->Start() != 0)
        {
            _shared->SetLastError(VE_THREAD_ERROR, kTraceError,
                "Init() failed to start module process thread");
            return -1;
        }
    }

    // Create an internal ADM if the user has not added an external
    // ADM implementation as input to Init().
    if (external_adm == NULL)
    {
        // Create the internal ADM implementation.
        _shared->set_audio_device(AudioDeviceModuleImpl::Create(
            VoEId(_shared->instance_id(), -1), _shared->audio_device_layer()));

        if (_shared->audio_device() == NULL)
        {
            _shared->SetLastError(VE_NO_MEMORY, kTraceCritical,
                "Init() failed to create the ADM");
            return -1;
        }
    }
    else
    {
        // Use the already existing external ADM implementation.
        _shared->set_audio_device(external_adm);
        WEBRTC_TRACE(kTraceInfo, kTraceVoice, VoEId(_shared->instance_id(), -1),
            "An external ADM implementation will be used in VoiceEngine");
    }

    // Register the ADM to the process thread, which will drive the error
    // callback mechanism
    if (_shared->process_thread() &&
        _shared->process_thread()->RegisterModule(_shared->audio_device()) != 0)
    {
        _shared->SetLastError(VE_AUDIO_DEVICE_MODULE_ERROR, kTraceError,
            "Init() failed to register the ADM");
        return -1;
    }
    bool available(false);

    // --------------------
    // Reinitialize the ADM

    // Register the AudioObserver implementation
    if (_shared->audio_device()->RegisterEventObserver(this) != 0) {
      _shared->SetLastError(VE_AUDIO_DEVICE_MODULE_ERROR, kTraceWarning,
          "Init() failed to register event observer for the ADM");
    }

    // Register the AudioTransport implementation
    if (_shared->audio_device()->RegisterAudioCallback(this) != 0) {
      _shared->SetLastError(VE_AUDIO_DEVICE_MODULE_ERROR, kTraceWarning,
          "Init() failed to register audio callback for the ADM");
    }

    // ADM initialization
    if (_shared->audio_device()->Init() != 0)
    {
        _shared->SetLastError(VE_AUDIO_DEVICE_MODULE_ERROR, kTraceError,
            "Init() failed to initialize the ADM");
        return -1;
    }

    // Initialize the default speaker
    if (_shared->audio_device()->SetPlayoutDevice(
            WEBRTC_VOICE_ENGINE_DEFAULT_DEVICE) != 0)
    {
        _shared->SetLastError(VE_AUDIO_DEVICE_MODULE_ERROR, kTraceInfo,
            "Init() failed to set the default output device");
    }
    if (_shared->audio_device()->SpeakerIsAvailable(&available) != 0)
    {
        _shared->SetLastError(VE_CANNOT_ACCESS_SPEAKER_VOL, kTraceInfo,
            "Init() failed to check speaker availability, trying to "
            "initialize speaker anyway");
    }
    else if (!available)
    {
        _shared->SetLastError(VE_CANNOT_ACCESS_SPEAKER_VOL, kTraceInfo,
            "Init() speaker not available, trying to initialize speaker "
            "anyway");
    }
    if (_shared->audio_device()->InitSpeaker() != 0)
    {
        _shared->SetLastError(VE_CANNOT_ACCESS_SPEAKER_VOL, kTraceInfo,
            "Init() failed to initialize the speaker");
    }

    // Initialize the default microphone
    if (_shared->audio_device()->SetRecordingDevice(
            WEBRTC_VOICE_ENGINE_DEFAULT_DEVICE) != 0)
    {
        _shared->SetLastError(VE_SOUNDCARD_ERROR, kTraceInfo,
            "Init() failed to set the default input device");
    }
    if (_shared->audio_device()->MicrophoneIsAvailable(&available) != 0)
    {
        _shared->SetLastError(VE_CANNOT_ACCESS_MIC_VOL, kTraceInfo,
            "Init() failed to check microphone availability, trying to "
            "initialize microphone anyway");
    }
    else if (!available)
    {
        _shared->SetLastError(VE_CANNOT_ACCESS_MIC_VOL, kTraceInfo,
            "Init() microphone not available, trying to initialize "
            "microphone anyway");
    }
    if (_shared->audio_device()->InitMicrophone() != 0)
    {
        _shared->SetLastError(VE_CANNOT_ACCESS_MIC_VOL, kTraceInfo,
            "Init() failed to initialize the microphone");
    }

    // Set number of channels
    if (_shared->audio_device()->StereoPlayoutIsAvailable(&available) != 0) {
      _shared->SetLastError(VE_SOUNDCARD_ERROR, kTraceWarning,
          "Init() failed to query stereo playout mode");
    }
    if (_shared->audio_device()->SetStereoPlayout(available) != 0)
    {
        _shared->SetLastError(VE_SOUNDCARD_ERROR, kTraceWarning,
            "Init() failed to set mono/stereo playout mode");
    }

    // TODO(andrew): These functions don't tell us whether stereo recording
    // is truly available. We simply set the AudioProcessing input to stereo
    // here, because we have to wait until receiving the first frame to
    // determine the actual number of channels anyway.
    //
    // These functions may be changed; tracked here:
    // http://code.google.com/p/webrtc/issues/detail?id=204
    _shared->audio_device()->StereoRecordingIsAvailable(&available);
    if (_shared->audio_device()->SetStereoRecording(available) != 0)
    {
        _shared->SetLastError(VE_SOUNDCARD_ERROR, kTraceWarning,
            "Init() failed to set mono/stereo recording mode");
    }

    if (!audioproc) {
      audioproc = AudioProcessing::Create(VoEId(_shared->instance_id(), -1));
      if (!audioproc) {
        LOG(LS_ERROR) << "Failed to create AudioProcessing.";
        _shared->SetLastError(VE_NO_MEMORY);
        return -1;
      }
    }
    _shared->set_audio_processing(audioproc);
    /*DECWANG_4RECORD设置混音模块调用指针
     */
    _shared->outputall_mixer()->SetAudioProcessingModule(_shared->audio_processing());
    _shared->outputall_mixer()->SetMixabilityStatus(_afmTransmitMixer, true);
    _shared->outputall_mixer()->SetMixabilityStatus(_afmOutputMixer, true);
    /**/

    // Set the error state for any failures in this block.
    _shared->SetLastError(VE_APM_ERROR);
    if (audioproc->echo_cancellation()->set_device_sample_rate_hz(48000)) {
      LOG_FERR1(LS_ERROR, set_device_sample_rate_hz, 48000);
      return -1;
    }
    //DECWANG_ADD 20110620 FOR RECORD SYNC ,

    // Assume 16 kHz mono until the audio frames are received from the capture
    // device, at which point this can be updated.
    if (audioproc->set_sample_rate_hz(16000)) {
      LOG_FERR1(LS_ERROR, set_sample_rate_hz, 16000);
      return -1;
    }
    if (audioproc->set_num_channels(1, 1) != 0) {
      LOG_FERR2(LS_ERROR, set_num_channels, 1, 1);
      return -1;
    }
    if (audioproc->set_num_reverse_channels(1) != 0) {
      LOG_FERR1(LS_ERROR, set_num_reverse_channels, 1);
      return -1;
    }
    
    // Configure AudioProcessing components. All are disabled by default.
    if (audioproc->high_pass_filter()->Enable(true) != 0) {
      LOG_FERR1(LS_ERROR, high_pass_filter()->Enable, true);
      return -1;
    }
    if (audioproc->echo_cancellation()->enable_drift_compensation(false) != 0) {
      LOG_FERR1(LS_ERROR, enable_drift_compensation, false);
      return -1;
    }
    if (audioproc->noise_suppression()->set_level(kDefaultNsMode) != 0) {
      LOG_FERR1(LS_ERROR, noise_suppression()->set_level, kDefaultNsMode);
      return -1;
    }
    GainControl* agc = audioproc->gain_control();
    if (agc->set_analog_level_limits(kMinVolumeLevel, kMaxVolumeLevel) != 0) {
      LOG_FERR2(LS_ERROR, agc->set_analog_level_limits, kMinVolumeLevel,
                kMaxVolumeLevel);
      return -1;
    }
    if (agc->set_mode(kDefaultAgcMode) != 0) {
      LOG_FERR1(LS_ERROR, agc->set_mode, kDefaultAgcMode);
      return -1;
    }
    if (agc->Enable(kDefaultAgcState) != 0) {
      LOG_FERR1(LS_ERROR, agc->Enable, kDefaultAgcState);
      return -1;
    }
    _shared->SetLastError(0);  // Clear error state.

#ifdef WEBRTC_VOICE_ENGINE_AGC
    bool agc_enabled = agc->mode() == GainControl::kAdaptiveAnalog &&
                       agc->is_enabled();
    if (_shared->audio_device()->SetAGC(agc_enabled) != 0) {
      LOG_FERR1(LS_ERROR, audio_device()->SetAGC, agc_enabled);
      _shared->SetLastError(VE_AUDIO_DEVICE_MODULE_ERROR);
      // TODO(ajm): No error return here due to
      // https://code.google.com/p/webrtc/issues/detail?id=1464
    }
#endif

    return _shared->statistics().SetInitialized();
}

 //////////////.h///////////////
//////////////.h///////////////
至此,结束,进行通话时就可以进行录音了.下一篇将介绍录制的文件如何提高语音质量和格式转换的问题了.See you Next.
转载请注明出处 decwang@qq.com