WebRTC实现背景声音的混流

背景

在Windows上使用WebRTC做视频采集，然后使用RTMP进行直播推流。默认情况下WebRTC只会采集麦克风的声音，而不会采集机器的背景声音。需要编码实现背景声音的采集和混音功能。

思路

Windows提供的API中有音频采集的相关方法，官方也给出了简单的说明和示例，虽然不能运行：）。所以可以通过Windows的API来采集PCM格式的音频，然后通过WebRTC的群聊混音机制来进行音频合成

核心代码

 音频采集部分

DWORD AudioCaptureCore::DoCaptureThread()
{
    keepRecording_ = true;
    HANDLE waitArray[2] = { _hShutdownCaptureEvent, _hCaptureSamplesReadyEvent };
    HRESULT hr = S_OK;

    LARGE_INTEGER t1;
    LARGE_INTEGER t2;
    int32_t time(0);

    BYTE* syncBuffer = NULL;
    UINT32 syncBufIndex = 0;

    _readSamples = 0;

    // Initialize COM as MTA in this thread.
    ScopedCOMInitializer comInit(ScopedCOMInitializer::kMTA);
    if (!comInit.succeeded()) {
        WEBRTC_TRACE(kTraceError, kTraceAudioDevice, _id,
            "failed to initialize COM in capture thread");
        return 1;
    }

    hr = InitCaptureThreadPriority();
    if (FAILED(hr))
    {
        return hr;
    }

    _Lock();


    REFERENCE_TIME hnsRequestedDuration = REFTIMES_PER_SEC;
    REFERENCE_TIME hnsActualDuration;
    UINT32 bufferLength;
    UINT32 numFramesAvailable;
    IMMDeviceEnumerator *pEnumerator = NULL;
    IMMDevice *pDevice = NULL;
    WAVEFORMATEX *pwfx = NULL;
    UINT32 packetLength = 0;
    BOOL bDone = FALSE;
    BYTE *pData;
    DWORD flags;


    hr = CoCreateInstance(CLSID_MMDeviceEnumerator, NULL, CLSCTX_ALL,
        IID_IMMDeviceEnumerator, (void**)&pEnumerator);
    EXIT_ON_ERROR(hr);

        hr = pEnumerator->GetDefaultAudioEndpoint(eRender, eConsole, &pDevice);
    EXIT_ON_ERROR(hr);

        hr = pDevice->Activate(IID_IAudioClient, CLSCTX_ALL, NULL, (void**)&_ptrAudioClientIn);
    EXIT_ON_ERROR(hr);

    // 
    hr = _ptrAudioClientIn->GetMixFormat(&pwfx);
    EXIT_ON_ERROR(hr);

    WAVEFORMATEX waveFormat;
    waveFormat.wFormatTag = WAVE_FORMAT_PCM;
    waveFormat.nChannels = 2;
    waveFormat.nSamplesPerSec = pwfx->nSamplesPerSec;
    waveFormat.nAvgBytesPerSec = pwfx->nSamplesPerSec * 4;
    waveFormat.wBitsPerSample = 16;
    waveFormat.nBlockAlign = 4;
    waveFormat.cbSize = 0;

    *pwfx = waveFormat;

    hr = _ptrAudioClientIn->Initialize(AUDCLNT_SHAREMODE_SHARED,
        AUDCLNT_STREAMFLAGS_LOOPBACK,
        hnsRequestedDuration,
        0,
        pwfx,
        NULL);
    EXIT_ON_ERROR(hr);

            // Set the VoE format equal to the AEC output format.
            _recAudioFrameSize = pwfx->nBlockAlign;
            _recSampleRate = pwfx->nSamplesPerSec;
            _recBlockSize = pwfx->nSamplesPerSec / 100;
            _recChannels = pwfx->nChannels;

            if (_ptrAudioBuffer)
            {
                // Update the audio buffer with the selected parameters
                _ptrAudioBuffer->SetRecordingSampleRate(_recSampleRate);
                _ptrAudioBuffer->SetRecordingChannels((uint8_t)_recChannels);
            }
            else
            {
                // We can enter this state during CoreAudioIsSupported() when no AudioDeviceImplementation
                // has been created, hence the AudioDeviceBuffer does not exist.
                // It is OK to end up here since we don't initiate any media in CoreAudioIsSupported().
                WEBRTC_TRACE(kTraceInfo, kTraceAudioDevice, _id, "AudioDeviceBuffer must be attached before streaming can start");
            }


            // Get the size of the allocated buffer.
            hr = _ptrAudioClientIn->GetBufferSize(&bufferLength);
            EXIT_ON_ERROR(hr);

            hr = _ptrAudioClientIn->GetService(__uuidof(IAudioCaptureClient), (void**)&_ptrCaptureClient);
            EXIT_ON_ERROR(hr);

            // Notify the audio sink which format to use.
                // 如上一行注释，以下的代码是将获取到的音频格式传给另外的类（自己定义的），同样的，因为
                // 手动制定了音频格式，所以就不需要通知了
            // hr = pMySink->SetFormat(pwfx);
            // EXIT_ON_ERROR(hr)

            // Calculate the actual duration of the allocated buffer.
            hnsActualDuration = (double)REFTIMES_PER_SEC * bufferLength / pwfx->nSamplesPerSec;

            //hr = _ptrAudioClientIn->Start();  // Start recording.
            //EXIT_ON_ERROR(hr);



    // Get size of capturing buffer (length is expressed as the number of audio frames the buffer can hold).
    // This value is fixed during the capturing session.
    //
            if (_ptrAudioClientIn == NULL)
    {
        WEBRTC_TRACE(kTraceError, kTraceAudioDevice, _id,
            "input state has been modified before capture loop starts.");
        return 1;
    }
            hr = _ptrAudioClientIn->GetBufferSize(&bufferLength);
    EXIT_ON_ERROR(hr);
    WEBRTC_TRACE(kTraceInfo, kTraceAudioDevice, _id, "[CAPT] size of buffer       : %u", bufferLength);

    // Allocate memory for sync buffer.
    // It is used for compensation between native 44.1 and internal 44.0 and
    // for cases when the capture buffer is larger than 10ms.
    //
    const UINT32 syncBufferSize = 2 * (bufferLength * _recAudioFrameSize);
    syncBuffer = new BYTE[syncBufferSize];
    if (syncBuffer == NULL)
    {
        return (DWORD)E_POINTER;
    }
    WEBRTC_TRACE(kTraceInfo, kTraceAudioDevice, _id, "[CAPT] size of sync buffer  : %u [bytes]", syncBufferSize);

    // Get maximum latency for the current stream (will not change for the lifetime of the IAudioClient object).
    //
    REFERENCE_TIME latency;
            _ptrAudioClientIn->GetStreamLatency(&latency);
    WEBRTC_TRACE(kTraceInfo, kTraceAudioDevice, _id, "[CAPT] max stream latency   : %u (%3.2f ms)",
        (DWORD)latency, (double)(latency / 10000.0));

    // Get the length of the periodic interval separating successive processing passes by
    // the audio engine on the data in the endpoint buffer.
    //
    REFERENCE_TIME devPeriod = 0;
    REFERENCE_TIME devPeriodMin = 0;
            _ptrAudioClientIn->GetDevicePeriod(&devPeriod, &devPeriodMin);
    WEBRTC_TRACE(kTraceInfo, kTraceAudioDevice, _id, "[CAPT] device period        : %u (%3.2f ms)",
        (DWORD)devPeriod, (double)(devPeriod / 10000.0));

    double extraDelayMS = (double)((latency + devPeriod) / 10000.0);
    WEBRTC_TRACE(kTraceInfo, kTraceAudioDevice, _id, "[CAPT] extraDelayMS         : %3.2f", extraDelayMS);

    double endpointBufferSizeMS = 10.0 * ((double)bufferLength / (double)_recBlockSize);
    WEBRTC_TRACE(kTraceInfo, kTraceAudioDevice, _id, "[CAPT] endpointBufferSizeMS : %3.2f", endpointBufferSizeMS);

    // Start up the capturing stream.
    //
            hr = _ptrAudioClientIn->Start();
    EXIT_ON_ERROR(hr);

    _UnLock();

    // Set event which will ensure that the calling thread modifies the recording state to true.
    //
    SetEvent(_hCaptureStartedEvent);

    // >> ---------------------------- THREAD LOOP ----------------------------


        while (keepRecording_)
        {
            BYTE *pData = 0;
            UINT32 framesAvailable = 0;
            DWORD flags = 0;
            UINT64 recTime = 0;
            UINT64 recPos = 0;

            std::cout << "bgm audio capturing" << std::endl;

            _Lock();

            // Sanity check to ensure that essential states are not modified
            // during the unlocked period.
                            if (_ptrCaptureClient == NULL || _ptrAudioClientIn == NULL)
            {
                _UnLock();
                WEBRTC_TRACE(kTraceCritical, kTraceAudioDevice, _id,
                    "input state has been modified during unlocked period");
                goto Exit;
            }

            //  Find out how much capture data is available
            //
            hr = _ptrCaptureClient->GetBuffer(&pData,           // packet which is ready to be read by used
                &framesAvailable, // #frames in the captured packet (can be zero)
                &flags,           // support flags (check)
                &recPos,          // device position of first audio frame in data packet
                &recTime);        // value of performance counter at the time of recording the first audio frame

            if (SUCCEEDED(hr))
            {
                if (AUDCLNT_S_BUFFER_EMPTY == hr)
                {
                    // Buffer was empty => start waiting for a new capture notification event
                    _UnLock();
                    continue;
                }

                if (flags & AUDCLNT_BUFFERFLAGS_SILENT)
                {
                    // Treat all of the data in the packet as silence and ignore the actual data values.
                    WEBRTC_TRACE(kTraceWarning, kTraceAudioDevice, _id, "AUDCLNT_BUFFERFLAGS_SILENT");
                    pData = NULL;
                }

                assert(framesAvailable != 0);

                if (pData)
                {
                    CopyMemory(&syncBuffer[syncBufIndex*_recAudioFrameSize], pData, framesAvailable*_recAudioFrameSize);
                }
                else
                {
                    ZeroMemory(&syncBuffer[syncBufIndex*_recAudioFrameSize], framesAvailable*_recAudioFrameSize);
                }
                assert(syncBufferSize >= (syncBufIndex*_recAudioFrameSize) + framesAvailable*_recAudioFrameSize);

                // Release the capture buffer
                //
                hr = _ptrCaptureClient->ReleaseBuffer(framesAvailable);
                EXIT_ON_ERROR(hr);

                _readSamples += framesAvailable;
                syncBufIndex += framesAvailable;

                QueryPerformanceCounter(&t1);

                // Get the current recording and playout delay.
                uint32_t sndCardRecDelay = (uint32_t)
                    (((((UINT64)t1.QuadPart * _perfCounterFactor) - recTime)
                        / 10000) + (10 * syncBufIndex) / _recBlockSize - 10);
                uint32_t sndCardPlayDelay =
                    static_cast<uint32_t>(_sndCardPlayDelay);

                _sndCardRecDelay = sndCardRecDelay;

                while (syncBufIndex >= _recBlockSize)
                {
                    if (_ptrAudioBuffer)
                    {
                        _ptrAudioBuffer->SetRecordedBuffer((const int8_t*)syncBuffer, _recBlockSize);
                        _ptrAudioBuffer->SetVQEData(sndCardPlayDelay,
                            sndCardRecDelay,
                            0);

                        _ptrAudioBuffer->SetTypingStatus(KeyPressed());

                        QueryPerformanceCounter(&t1);    // measure time: START

                        _UnLock();  // release lock while making the callback
                        _ptrAudioBuffer->DeliverRecordedData();
                        _Lock();    // restore the lock

                        QueryPerformanceCounter(&t2);    // measure time: STOP

                                                         // Measure "average CPU load".
                                                         // Basically what we do here is to measure how many percent of our 10ms period
                                                         // is used for encoding and decoding. This value shuld be used as a warning indicator
                                                         // only and not seen as an absolute value. Running at ~100% will lead to bad QoS.
                        time = (int)(t2.QuadPart - t1.QuadPart);
                        _avgCPULoad = (float)(_avgCPULoad*.99 + (time + _playAcc) / (double)(_perfCounterFreq.QuadPart));
                        _playAcc = 0;

                        // Sanity check to ensure that essential states are not modified during the unlocked period
                                                    if (_ptrCaptureClient == NULL || _ptrAudioClientIn == NULL)
                        {
                            _UnLock();
                            WEBRTC_TRACE(kTraceCritical, kTraceAudioDevice, _id, "input state has been modified during unlocked period");
                            goto Exit;
                        }
                    }

                    // store remaining data which was not able to deliver as 10ms segment
                    MoveMemory(&syncBuffer[0], &syncBuffer[_recBlockSize*_recAudioFrameSize], (syncBufIndex - _recBlockSize)*_recAudioFrameSize);
                    syncBufIndex -= _recBlockSize;
                    sndCardRecDelay -= 10;
                }

                if (_AGC)
                {
                    uint32_t newMicLevel = _ptrAudioBuffer->NewMicLevel();
                    if (newMicLevel != 0)
                    {
                        // The VQE will only deliver non-zero microphone levels when a change is needed.
                        // Set this new mic level (received from the observer as return value in the callback).
                        WEBRTC_TRACE(kTraceStream, kTraceAudioDevice, _id, "AGC change of volume: new=%u", newMicLevel);
                        // We store this outside of the audio buffer to avoid
                        // having it overwritten by the getter thread.
                        _newMicLevel = newMicLevel;
                        SetEvent(_hSetCaptureVolumeEvent);
                    }
                }
            }
            else
            {
                // If GetBuffer returns AUDCLNT_E_BUFFER_ERROR, the thread consuming the audio samples
                // must wait for the next processing pass. The client might benefit from keeping a count
                // of the failed GetBuffer calls. If GetBuffer returns this error repeatedly, the client
                // can start a new processing loop after shutting down the current client by calling
                // IAudioClient::Stop, IAudioClient::Reset, and releasing the audio client.
                WEBRTC_TRACE(kTraceError, kTraceAudioDevice, _id,
                    "IAudioCaptureClient::GetBuffer returned AUDCLNT_E_BUFFER_ERROR, hr = 0x%08X", hr);
                goto Exit;
            }

            _UnLock();
        }

    // ---------------------------- THREAD LOOP ---------------------------- <<

            if (_ptrAudioClientIn)
    {
                    hr = _ptrAudioClientIn->Stop();
    }

Exit:
    if (FAILED(hr))
    {
                    _ptrAudioClientIn->Stop();
        _UnLock();
        _TraceCOMError(hr);
    }

    RevertCaptureThreadPriority();

    if (syncBuffer)
    {
        delete[] syncBuffer;
    }

    return (DWORD)hr;
}

声音合成

创建     webrtc::AudioConferenceMixer *audio_mixer_ = nullptr; 在使用多路声音的时候进行混音

int32_t AnyRtmpCore::RecordedDataIsAvailable(const void* audioSamples, const size_t nSamples,
    const size_t nBytesPerSample, const size_t nChannels, const uint32_t samplesPerSec, const uint32_t totalDelayMS,
    const int32_t clockDrift, const uint32_t currentMicLevel, const bool keyPressed, uint32_t& newMicLevel)
{
    std::cout << "[-----------] record data avaliable " << nSamples << nBytesPerSample << nChannels << samplesPerSec << std::endl;
    rtc::CritScope cs(&cs_audio_record_);

    if (microphone_enable_ && bgm_enable_) {
        audio_device_mixer_ptr_->RecordedDataIsAvailable(audioSamples, nSamples,
            nBytesPerSample, nChannels, samplesPerSec, totalDelayMS,
            clockDrift, currentMicLevel, keyPressed, newMicLevel);
        if (audio_mixer_) {
            audio_mixer_->Process();
        }        
    }
    else
    {
        // 当只有一种声音时，不进行混音
        if (audio_record_callback_) {
            if (audio_record_sample_hz_ != samplesPerSec || nChannels != audio_record_channels_) {
                int16_t temp_output[kMaxDataSizeSamples];
                int samples_per_channel_int = resampler_record_.Resample10Msec((int16_t*)audioSamples, samplesPerSec * nChannels,
                    audio_record_sample_hz_ * audio_record_channels_, 1, kMaxDataSizeSamples, temp_output);
                audio_record_callback_->OnRecordAudio(temp_output, audio_record_sample_hz_ / 100, nBytesPerSample, audio_record_channels_, audio_record_sample_hz_, totalDelayMS);
            }
            else {
                audio_record_callback_->OnRecordAudio(audioSamples, nSamples, nBytesPerSample, audio_record_channels_, samplesPerSec, totalDelayMS);
            }
        }
    }
        
    return 0;
}