语音交互前端实现方案

1、语音交互主要使用了Recorder插件，以来wav这个文件进行转换编码

<template>
    <a-row :class="['voice-control', {'extend-w': isShow}]">
        <div class="voice-l" @click="activeRecordVoice">
            <img :class="['robot-img', {'robot-img-extend': isShow}]" :src="robotImg">
            <img
                v-if="!isShow" class="pos"
                :src="soundWaveImg"
            >
        </div>
        <div :class="['voice-r',{'anim-in': isShow}, {'anim-out': !isShow}]">
            <img
                :class="['img-con', {'anim-in': isShow}, {'anim-out': !isShow}]"
                :src="recording ? soundWaveGif : soundBgImg"
            >
            <p>{{ voiceText }}</p>
        </div>
        <i
            v-show="isShow" class="close-info-window"
            @click="clickCloseBtn"
        >×</i>
    </a-row>
</template>
<script>
import {reactive, onMounted, toRefs, onBeforeUnmount, getCurrentInstance, watch} from 'vue';

import robotImg from '@/assets/img/robot.gif';
import soundBgImg from '@/assets/img/voice-bg.png';
import soundWaveImg from '@/assets/img/voice.png';
import soundWaveGif from '@/assets/img/wave.gif';
import Recorder from 'recorder-core';
import ReconnectingWebSocket from 'reconnecting-websocket';

export default {
    name: 'VoiceControl',
    props: {
        url: {
            type: String,
            default: 'ws://10.138.65.153:8887/client/speech'
        }
    },
    setup(props, content) {
        const {proxy} = getCurrentInstance();
        const state = reactive({
            voiceText: '',
            isShow: false,
            recording: false,
            robotImg,
            soundWaveImg,
            soundWaveGif,
            soundBgImg,
            ws: null,
            realTimeSendTryType: 'wav',
            realTimeSendTryEncBusy: 0,
            realTimeSendTryTime: 0,
            realTimeSendTryNumber: 0,
            transferUploadNumberMax: 0,
            realTimeSendTryChunk: null,
            sendInterval: 100,
            sampleRate: 16000, // 采样率
            bitRate: 16, // 比特率
            encStartTime: Date.now(),
            scriptNode: null // 录音插件元素
        });

        const clickStartBtn = () => {
            if (!state.isShow) {
                state.isShow = true;
                creatSocket();
            }
        };

        const clickCloseBtn = () => {
            state.isShow = false;
            stopRecord();
        };

        // 添加录音插件
        const addPlugin = () => {
            const wavUrl = '/wav.min.js';
            const scriptNode = document.createElement('script');
            scriptNode.setAttribute('type', 'text/javascript');
            scriptNode.setAttribute('src', wavUrl);
            document.body.appendChild(scriptNode);
            state.scriptNode = scriptNode;
        };

        // 移除录音插件
        const removePlugin = () => {
            if (state.scriptNode) {
                document.body.removeChild(state.scriptNode);
                state.scriptNode = null;
            }
        };

        // 录音
        const activeRecordVoice = () => {
            if (!state.recording) {
                state.encStartTime = Date.now();
                state.recorderInstance = Recorder({
                    type: 'wav',
                    sampleRate: state.sampleRate,
                    bitRate: state.bitRate,
                    onProcess: (buffers, powerLevel, bufferDuration,
                        bufferSampleRate, newBufferIdx, asyncEnd) => {
                        // state.voicePowerArr.push(powerLevel);
                        // state.wave.input(buffers[buffers.length - 1], powerLevel, bufferSampleRate);
                        realTimeSendTry(state.recorderInstance, false);
                    }
                });
                startRecord();
            } else {
                stopRecord();
            }
        };

        const startRecord = () => {
            state.recording = true;
            clickStartBtn();
            state.recorderInstance.open(
                () => {
                    state.recorderInstance.start();
                },
                (msg, isUserNotAllow) => {
                    console.log((isUserNotAllow ? 'UserNotAllow，' : '') + '无法录音:' + msg);
                    state.recording = false;
                }
            );
        };

        const stopRecord = () => {
            if (state.recorderInstance) {
                state.recorderInstance.close(() => {
                    state.realTimeSendTryTime = 0;
                    state.realTimeSendTryEncBusy = 0;
                    state.realTimeSendTryNumber = 0;
                    state.transferUploadNumberMax = 0;
                    state.realTimeSendTryChunk = null;
                    state.recording = false;
                    state.recorderInstance = false;
                    console.log('%c录音完成，关闭录音', 'color:#fff;background:green;');
                });
            }
            closeWs();
        };

        const realTimeSendTry = (rec, isClose) => {
            const t1 = Date.now();
            let endT = 0;
            const recImpl = Recorder.prototype;
            if (state.realTimeSendTryTime === 0) {
                state.realTimeSendTryTime = t1;
                state.realTimeSendTryEncBusy = 0;
                state.realTimeSendTryNumber = 0;
                state.transferUploadNumberMax = 0;
                state.realTimeSendTryChunk = null;
            }
            if (!isClose && t1 - state.realTimeSendTryTime < state.sendInterval) {
                console.log('%c控制缓冲达到指定间隔才进行传输', 'color: #fff;background: red;');
                return;
            }
            state.realTimeSendTryTime = t1;
            let number = ++state.realTimeSendTryNumber;
            const chunk = Recorder.SampleData(
                rec.buffers,
                rec.srcSampleRate,
                state.sampleRate,
                state.realTimeSendTryChunk,
                {
                    frameType: isClose ? '' : state.realTimeSendTryType
                }
            );
            // 清理已处理完的缓冲数据，释放内存以支持长时间录音，最后完成录音时不能调用stop，因为数据已经被清掉了
            for (let i = state.realTimeSendTryChunk ? state.realTimeSendTryChunk.index : 0; i < chunk.index; i++) {
                rec.buffers[i] = null;
            }
            state.realTimeSendTryChunk = chunk;
            if (chunk.data.length === 0 || isClose && chunk.data.length < 2000) {
                transferUpload(number, null, 0, null, isClose);
                console.log('%c没有新数据，或结束时的数据量太小，不能进行mock转码', 'color: #fff;background: red;');
                return;
            }
            // 实时编码队列阻塞处理
            if (!isClose && state.realTimeSendTryEncBusy >= 2) {
                console.log('%c编码队列阻塞，已丢弃一帧', 'color: #fff;background: red;');
                return;
            }
            state.realTimeSendTryEncBusy++;
            const encStartTime = Date.now();
            const recMock = Recorder({
                type: state.realTimeSendTryType,
                sampleRate: state.sampleRate,
                bitRate: state.bitRate
            });
            recMock.mock(chunk.data, chunk.sampleRate);
            recMock.stop((blob, duration) => {
                state.realTimeSendTryEncBusy && (state.realTimeSendTryEncBusy--);
                blob.encTime = Date.now() - encStartTime;
                // 转码好就推入传输
                transferUpload(number, blob, duration, recMock, isClose);
            }, (msg) => {
                state.realTimeSendTryEncBusy && (state.realTimeSendTryEncBusy--);
                // 转码错误
                console.log('%c出现的错误:' + msg, 'color: #fff;background: red;');
            });
        };

        const transferUpload = (number, blobOrNull, duration, blobRec, isClose) => {
            state.transferUploadNumberMax = Math.max(state.transferUploadNumberMax, number);
            if (blobOrNull) {
                const blob = blobOrNull;
                const encTime = blob.encTime;
                const reader = new FileReader();
                reader.readAsArrayBuffer(blob);
                reader.onload = () => {
                    const byteArray = Array.from(new Int8Array(reader.result));
                    sendWs(byteArray);
                };
            }
        };

        const creatSocket = () => {
            if (state.ws) {
                closeWs();
            }
            // 与后台建立websocket交互逻辑处理，socket使用了ReconnectingWebSocket插件
            state.ws = new ReconnectingWebSocket(props.url, [], {
                connectionTimeout: 3000,
                maxRetries: 10
            });
            state.ws?.addEventListener('open', () => {
                state.ws?.send(JSON.stringify(
                    {type: 'start'}
                ));
                state.ws?.addEventListener('message', (e) => {
                    const info = JSON.parse(e?.data)?.data;
                    if (info.type === 'text') {
                        state.voiceText = info?.data?.text || '';
                    }
                    if (info.type === 'intent') {
                        content.emit('commandOpe', info);
                    }
                });
            });
            state.ws?.addEventListener('close', () => {
                state.ws = null;
                state.voiceText = '';
            });
        };

        const sendWs = (message) => {
            if (state.ws) {
                const mes = {
                    'type': 'data', 'data': {'speechBytes': message}
                };
                if (state.ws?.readyState === 1) {
                    state.ws?.send(JSON.stringify(mes));
                }
            }
        };

        const closeWs = () => {
            if (state.ws && state.ws?.readyState === 1) {
                state.ws?.send(JSON.stringify({'type': 'stop'}));
                state.ws?.close();
            }
        };

        onMounted(() => {
            watch(() => state.isShow, isShow => {
                content.emit('clickHandler', isShow);
            });
            addPlugin();
        });

        onBeforeUnmount(() => {
            stopRecord();
            removePlugin();
        });

        return {
            ...toRefs(state),
            activeRecordVoice,
            clickCloseBtn,
            clickStartBtn
        };
    }
};
</script>
<style lang="less" scoped>
@import '~@/assets/less/variables.less';

.voice-control {
    position: relative;
    display: flex;
    flex-wrap: nowrap;
    max-height: 216px;
    margin: 0 @margin-base-lg * 3 0 0;
    font-size: @--font-size-md;
    font-weight: 500;
    color: @--color-white;
    // background:
    // linear-gradient(
    //     179deg,
    //     rgba(13, 45, 80, 0.8) 0%,
    //     rgba(0, 25, 64, 0.7) 100%
    // );
    background-image: linear-gradient(179deg, rgba(13, 45, 80, 0.14) 0%, rgba(0, 25, 64, 0.70) 100%);

    .time {
        animation-duration: 0.5s;
    }

    .voice-l {
        position: relative;
         216px;
        min- 216px;
        overflow: hidden;

        .robot-img {
            position: relative;
            top: -38%;
            left: 50%;
             150%;
            transform: translate(-50%, 0);
        }

        .robot-img-extend {
            top: 50%;
            transform: translate(-50%, -50%);
        }

        .pos {
            position: absolute;
            bottom: @margin-base-lg;
            bottom: @margin-base-md * 3;
            left: 50%;
             90%;
            transform: translate(-50%, 0);
        }
    }

    .voice-r {
        display: none;
        flex-wrap: wrap;
        align-content: center;
        justify-content: center;
         0;
        cursor: pointer;

        > p {
             100%;
            text-align: center;
        }
    }

    .img-con {
         0;
        max- 408px;
        height: 50%;
        max-height: 77px;
        background: url('~@/assets/img/voice.png') no-repeat 54% center;
        background-size: 60% 70%;
        animation-name: anim-in-name;
        .time();
    }

    .img-h {
        height: auto;
    }

    .anim-in {
        // display: block;
        display: flex;
        animation-name: anim-in-name;
        .extend-w();
        .time();
    }

    .anim-out {
         0;
        animation-name: anim-out-name;
        .time();
    }

    @keyframes anim-in-name {
        from {
             0;
        }

        to {
            .extend-s-w();
        }
    }

    @keyframes anim-out-name {
        from {
            .extend-s-w();
        }

        to {
             0;
        }
    }

    .close-info-window {
        position: absolute;
        top: -10px;
        right: -10px;
        box-sizing: border-box;
        display: inline-block;
         40px;
        height: 40px;
        font-size: 20px;
        font-style: normal;
        line-height: 40px;
        color: #fff;
        text-align: center;
        cursor: pointer;
        border: 1px solid #47bef7;
        box-shadow: inset 0 1px 45px 0 rgba(66, 212, 254, 0.8);

        // &:after {
        //     position: absolute;
        //     top: 0;
        //     right: 0;
        //     display: inline-block;
        //      4px;
        //     height: 38px;
        //     content: '';
        //     background: #fff;
        // }
    }
}

.extend-w {
     100%;
}

.extend-s-w {
     auto;
}
</style>

参考：https://xiangyuecn.gitee.io/recorder/assets/%E5%B7%A5%E5%85%B7-%E4%BB%A3%E7%A0%81%E8%BF%90%E8%A1%8C%E5%92%8C%E9%9D%99%E6%80%81%E5%88%86%E5%8F%91Runtime.html?jsname=teach.realtime.encode_transfer

拓展阅读：http://www.semlinker.com/you-dont-know-blob/