ffmpeg官方例程3学习-给视频播放加上声音

在上一篇中简单设计了一个视频播放器，但是没有声音，于是本篇的代码就是在上一篇的基础上加上声音。

在ffmpeg给的官方例程中，给视频添加声音是有问题的，这一块可以参考下面的这篇博客：

http://www.cnblogs.com/ansersion/p/5265033.html

其中播放声音比较ffmpeg以前版本增加了重采样的概念，而我添加声音这块代码也是参考这篇博客得来。

代码是在上一篇的基础上添加，对于之前的代码几乎没有删改，程序最终运行效果为视频跑的很快，但是声音则正常播放。

开发环境：

　　操作系统：ubuntu14

　　ffmpeg版本：3.2.2

　　sdl版本：2

编译与运行：

gcc -g main.c -o test -I /usr/local/ffmpeg/include -L /usr/local/ffmpeg/lib -lavutil -lavformat -lavcodec -lz -lavutil -lswscale -L /usr/lib/x86_64-linux-gnu -lSDL2 -lSDL2main
（其中路径按照自己的安装路径来 -g 是加GDB调试）

./test

#ifdef _cplusplus
extern "C"
{
#endif

#include<stdio.h>
#include<assert.h>
#include<libavcodec/avcodec.h>
#include<libavformat/avformat.h>
#include<libavutil/avutil.h>
#include<libswscale/swscale.h>
#include <libswresample/swresample.h>
#include<libavutil/avutil.h>
#include<libavutil/imgutils.h>
#include<SDL2/SDL.h>
#include<SDL2/SDL_thread.h>

//是否将YUV420P内容输出到文件
#define OUTPUT_YUV420P 0
//要播放的文件路径
#define filename "/home/sns/test.flv"
//要输出YUV420P内容的文件路径
#define outfilename "/home/sns/output.yuv"

#define SDL_AUDIO_BUFFER_SIZE 1024
#define MAX_AUDIO_FRAME_SIZE 192000
//结构体定义
typedef struct PacketQueue{
    AVPacketList *first_pkt,*last_pkt;//队首、队尾
    int nb_packets;    //包的个数
    int size;//队列的字节数
    SDL_mutex *mutex;//互斥量
    SDL_cond    *cond;//条件变量
}PacketQueue;

typedef struct AudioParams {
    int freq;
    int channels;
    int64_t channel_layout;
    enum AVSampleFormat fmt;
    int frame_size;
    int bytes_per_sec;
} AudioParams;
//函数定义
static void audio_callback(void *userdata, Uint8 * stream, int len);//
static int packet_queue_get(PacketQueue *q,AVPacket *pkt,int block);
int    packet_queue_put(PacketQueue *q,AVPacket *pkt);
void packet_queue_init(PacketQueue *q);
static int audio_decode_frame(AVCodecContext *aCodecCtx,uint8_t *audio_buf,int buf_size);
int resample(AVFrame *af,uint8_t *audio_buf,int *audio_buf_size);

//全局变量定义
int quit =0;
PacketQueue audioQ;
int sample_rate, nb_channels;
int64_t channel_layout;
struct SwrContext * swr_ctx = NULL;
AudioParams audio_hw_params_tgt;
AudioParams audio_hw_params_src;

//主函数
int main(int argc, char **argv)
{
    //变量定义*********************************************************************
    AVFormatContext *pFormatCtx;
    int i=0;
    int videoStream;
    int audioStream;
    AVCodecContext *pCodecCtx;
    AVCodecContext  *aCodecCtxOrig;
    AVCodecContext *aCodecCtx;
    AVCodec *pCodec;
    AVCodec *aCodec;
    AVFrame *pFrame;
    AVFrame *pFrameYUV;
    uint8_t *buffer;
    int numBytes;

    SDL_Window *screen;
    SDL_Renderer *sdlRender;
    SDL_Texture *sdlTexture;
    SDL_Rect sdlRect;
    int frameFinished;
    AVPacket packet;
    struct SwsContext *img_convert_ctx;
    int err_code;
    char buf[1024];
    FILE *fp_yuv;
    int y_size;
    SDL_AudioSpec audioSpec;
    SDL_AudioSpec spec;
    SDL_Event       event;
    //*******************************************************************************
    av_register_all();
    //1、打开视频文件*************************************************
    pFormatCtx = avformat_alloc_context();
    err_code = avformat_open_input(&pFormatCtx, filename, NULL, NULL);
    if (err_code != 0)
    {//打开文件失败
        av_strerror(err_code, buf, 1024);
        printf("coundn't open the file!,error code = %d(%s)
", err_code, buf);
        return -1;
    }
    if (avformat_find_stream_info(pFormatCtx, NULL) < 0)
    {
        printf("Couldn't find stream information.
");
        return -1;
    }
    // 打印信息
      av_dump_format(pFormatCtx, 0, filename, 0);
    //2、找到第一个视频流和第一个音频流****************************
    videoStream = -1;
    audioStream = -1;
    for (i = 0; i < pFormatCtx->nb_streams; i++)
    {
        if (pFormatCtx->streams[i]->codec->codec_type == AVMEDIA_TYPE_VIDEO && videoStream<0)
        {
            videoStream = i;//得到视频流的索引
        }
        if(pFormatCtx->streams[i]->codec->codec_type == AVMEDIA_TYPE_AUDIO &&audioStream < 0)
        {
            audioStream = i;//得到音频流的索引
        }
    }
    if (videoStream == -1)
    {
        printf("Didn't find a video stream.
");
        return -1;
    }
    if(audioStream == -1)
    {
        printf("coundn't find a audio stream!
");
        return -1;
    }
    /* 3、从视频流中得到一个音频和视频编解码上下文,里面包含了编解码器的所有信息和一个
    指向真正的编解码器     ,然后我们找到音频和视频编解码器*/
    pCodecCtx = pFormatCtx->streams[videoStream]->codec;
    aCodecCtxOrig = pFormatCtx->streams[audioStream]->codec;
    pCodec = avcodec_find_decoder(pCodecCtx->codec_id);
    aCodec = avcodec_find_decoder(aCodecCtxOrig->codec_id);
    if (pCodec == NULL)
    {
        fprintf(stderr, "Unsupported codec !
");
        return -1;
    }
    if(aCodec == NULL)
    {
        fprintf(stderr,"Unsupported codec!
");
        return -1;
    }
    //拷贝上下文
    aCodecCtx = avcodec_alloc_context3(aCodec);
    if(avcodec_copy_context(aCodecCtx,aCodecCtxOrig) != 0)
    {
        fprintf(stderr,"couldn't copy codec context!
");
        return -1;
    }
    //4、打开音频和视频编解码器
    if (avcodec_open2(pCodecCtx, pCodec, NULL) < 0)
    {
        printf("cann't open the codec!
");
        return -1;
    }
    if(avcodec_open2(aCodecCtx,aCodec,NULL) < 0 )
    {
        printf("cann't open the audio codec!
");
        return -1;
    }
    //设置声音参数
    sample_rate = aCodecCtx->sample_rate;
    nb_channels = aCodecCtx->channels;
    channel_layout = aCodecCtx->channel_layout;

//    printf("channel_layout=%" PRId64 "
", channel_layout);
    printf("nb_channels=%d
", nb_channels);
    printf("freq=%d
", sample_rate);

    if (!channel_layout|| nb_channels != av_get_channel_layout_nb_channels(channel_layout))
    {
        channel_layout = av_get_default_channel_layout(nb_channels);
        channel_layout &= ~AV_CH_LAYOUT_STEREO_DOWNMIX;
        printf("correction
");
    }
    /*通故编解码上下文中的所有信息来建立音频的信息*/
    audioSpec.freq = aCodecCtx->sample_rate;
    audioSpec.format = AUDIO_S16SYS;
    audioSpec.channels = aCodecCtx->channels;
    audioSpec.silence = 0;
    audioSpec.samples = SDL_AUDIO_BUFFER_SIZE;
    audioSpec.callback = audio_callback;
    audioSpec.userdata = aCodecCtx;
    //打开音频设备和初始化
    if(SDL_OpenAudio(&audioSpec,&spec) < 0)
    //其中回调函数在需要更多音频数据的时候被调用（即播放完后需要从回调取数据播放）
    {
        fprintf(stderr,"SDL_OpenAudio:    %s
",SDL_GetError());
        return -1;
    }
    printf("freq: %d	channels: %d
", spec.freq, spec.channels);
    //5、分配两个视频帧，一个保存得到的原始视频帧，一个保存为指定格式的视频帧（该帧通过原始帧转换得来）
    pFrame = av_frame_alloc();
    if (pFrame == NULL)
    {
        printf("pFrame alloc fail!
");
        return -1;
    }
    pFrameYUV = av_frame_alloc();
    if (pFrameYUV == NULL)
    {
        printf("pFrameYUV alloc fail!
");
        return -1;
    }
    //6、得到一帧视频截图的内存大小并分配内存，并将YUV数据填充进去
    numBytes = av_image_get_buffer_size(AV_PIX_FMT_YUV420P, pCodecCtx->width,
            pCodecCtx->height,1);
    buffer = (uint8_t*) av_mallocz(numBytes * sizeof(uint8_t));
    if (!buffer)
    {
        printf("numBytes :%d , buffer malloc 's mem 
", numBytes);
        return -1;
    }
    //打印信息
    printf("--------------- File Information ----------------
");
    av_dump_format(pFormatCtx, 0, filename, 0);
    printf("-------------------------------------------------
");
    av_image_fill_arrays(pFrameYUV->data, pFrameYUV->linesize,buffer,
            AV_PIX_FMT_YUV420P,pCodecCtx->width, pCodecCtx->height,1);
    //7、得到指定转换格式的上下文**********************************
    img_convert_ctx = sws_getContext(pCodecCtx->width, pCodecCtx->height,
            pCodecCtx->pix_fmt, pCodecCtx->width, pCodecCtx->height,
            AV_PIX_FMT_YUV420P,
            SWS_BICUBIC,
            NULL, NULL, NULL);
    if (img_convert_ctx == NULL)
    {
        fprintf(stderr, "Cannot initialize the conversion context!
");
        return -1;
    }
    //***********************************************************
#if OUTPUT_YUV420P
    fp_yuv = fopen(outfilename, "wb+");
#endif
    //8、SDL初始化和创建多重windows等准备工作
    if (SDL_Init(SDL_INIT_AUDIO | SDL_INIT_TIMER | SDL_INIT_VIDEO))
    {
        fprintf(stderr, "Could not initialize SDL - %s
", SDL_GetError());
        return -1;
    }
    //使用SDL_CreateWindow代替SDL_SetVideoMode
    //创建一个给定高度和宽度、位置和标示的windows。
    screen = SDL_CreateWindow("Simplest ffmpeg player's Window",
    SDL_WINDOWPOS_UNDEFINED, SDL_WINDOWPOS_UNDEFINED, pCodecCtx->width,
            pCodecCtx->height, SDL_WINDOW_OPENGL);
    if (!screen)
    {
        fprintf(stderr, "SDL: could not create window - exiting - %s
",SDL_GetError());
        return -1;
    }
    //对该window创建一个2D渲染上下文
    sdlRender = SDL_CreateRenderer(screen, -1, 0);
    if (!sdlRender)
    {
        fprintf(stderr, "SDL:cound not create render :    %s
", SDL_GetError());
        return -1;
    }
    //Create a texture for a rendering context.
    //为一个渲染上下文创建一个纹理
    //IYUV: Y + U + V  (3 planes)
    //YV12: Y + V + U  (3 planes)
    sdlTexture = SDL_CreateTexture(sdlRender, SDL_PIXELFORMAT_IYUV,
            SDL_TEXTUREACCESS_STREAMING, pCodecCtx->width, pCodecCtx->height);
    if (!sdlTexture)
    {
        fprintf(stderr, "SDL:cound not create Texture :    %s
", SDL_GetError());
        return -1;
    }
    //建立一个矩形变量，提供后面使用
    sdlRect.x = 0;
    sdlRect.y = 0;
    sdlRect.w = pCodecCtx->width;
    sdlRect.h = pCodecCtx->height;
    //*****************************************************
    //声音部分代码
    audio_hw_params_tgt.fmt = AV_SAMPLE_FMT_S16;
    audio_hw_params_tgt.freq = spec.freq;
    audio_hw_params_tgt.channel_layout = channel_layout;
    audio_hw_params_tgt.channels = spec.channels;
    audio_hw_params_tgt.frame_size = av_samples_get_buffer_size(NULL,
    audio_hw_params_tgt.channels, 1, audio_hw_params_tgt.fmt, 1);
    audio_hw_params_tgt.bytes_per_sec = av_samples_get_buffer_size(NULL,
            audio_hw_params_tgt.channels, audio_hw_params_tgt.freq,
            audio_hw_params_tgt.fmt, 1);
    if (audio_hw_params_tgt.bytes_per_sec <= 0|| audio_hw_params_tgt.frame_size <= 0)
    {
        printf("size error
");
        return -1;
    }
    audio_hw_params_src = audio_hw_params_tgt;
    //*****************************************************
    packet_queue_init(&audioQ);
    SDL_PauseAudio(0);
    //9、正式开始读取数据*****************************************
    while (av_read_frame(pFormatCtx, &packet) >= 0)
    {
        //如果读取的包来自视频流
        if (packet.stream_index == videoStream)
        {
            //从包中得到解码后的帧
            if (avcodec_decode_video2(pCodecCtx, pFrame, &frameFinished,&packet) < 0)
            {
                printf("Decode Error!
");
                return -1;
            }
            //如果确定完成得到该视频帧
            if (frameFinished)
            {
                //转换帧数据格式
                sws_scale(img_convert_ctx, pFrame->data, pFrame->linesize, 0,
                        pCodecCtx->height,
                        pFrameYUV->data,
                        pFrameYUV->linesize);
#if OUTPUT_YUV420P
                y_size = pCodecCtx->width * pCodecCtx->height;
                fwrite(pFrameYUV->data[0], 1, y_size, fp_yuv);    //Y
                fwrite(pFrameYUV->data[1], 1, y_size / 4, fp_yuv);  //U
                fwrite(pFrameYUV->data[2], 1, y_size / 4, fp_yuv);  //V
#endif
                //SDL显示~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#if 0
                SDL_UpdateTexture(sdlTexture,NULL,pFrameYUV->data[0],pFrameYUV->linesize[0]);
#else
                SDL_UpdateYUVTexture(sdlTexture, &sdlRect, pFrameYUV->data[0],
                        pFrameYUV->linesize[0], pFrameYUV->data[1],
                        pFrameYUV->linesize[1], pFrameYUV->data[2],
                        pFrameYUV->linesize[2]);
#endif
                SDL_RenderClear(sdlRender);
                SDL_RenderCopy(sdlRender, sdlTexture, NULL, &sdlRect);
                SDL_RenderPresent(sdlRender);
                //结束SDL~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
            }
        }
        else if(packet.stream_index == audioStream)
        {
                packet_queue_put(&audioQ,&packet);
        }else{
                av_free_packet(&packet);//释放读出来的包
        }
        SDL_PollEvent(&event);
        switch (event.type)
        {
        case SDL_QUIT:
            quit = 1;
            SDL_Quit();
            exit(0);
            break;
        default:
            break;
        }
    }
    while(1) SDL_Delay(1000);
    //**************************************************************************************
    //10、释放分配的内存或关闭文件等操作
#if OUTPUT_YUV420P
    fclose(fp_yuv);
#endif
    sws_freeContext(img_convert_ctx);
    SDL_Quit();
    av_free(buffer);
    av_free(pFrame);
    av_free(pFrameYUV);
    avcodec_close(pCodecCtx);
    avcodec_close(aCodecCtxOrig);
    avcodec_close(aCodecCtx);
    avformat_close_input(&pFormatCtx);
    return EXIT_SUCCESS;
}


static void audio_callback(void *userdata, Uint8 * stream, int len)
{
    AVCodecContext *aCodecCtx = (AVCodecContext*)userdata;
    int len1,audio_size;
    static uint8_t audio_buf[(MAX_AUDIO_FRAME_SIZE*3)/2];
    static unsigned int audio_buf_size = 0;
    static unsigned int audio_buf_index =0;
    while(len>0){
        if(audio_buf_index >= audio_buf_size){
            audio_size = audio_decode_frame(aCodecCtx,audio_buf,sizeof(audio_buf));
            if(audio_size < 0){
                audio_buf_size = 1024;
                memset(audio_buf,0,audio_buf_size);
            }else{
                audio_buf_size = audio_size;
            }
            audio_buf_index = 0;
        }
        len1 = audio_buf_size - audio_buf_index;
        if(len1 > len)
        len1 = len;
        memcpy(stream, (uint8_t *)audio_buf + audio_buf_index, len1);
        len -= len1;
        stream += len1;
        audio_buf_index += len1;
    }
}

void packet_queue_init(PacketQueue *q)
{
    memset(q,0,sizeof(PacketQueue));
    q->mutex = SDL_CreateMutex();
    q->cond = SDL_CreateCond();
}

int    packet_queue_put(PacketQueue *q,AVPacket *pkt)
{
    AVPacketList *pkt1;
    if(av_dup_packet(pkt)<0)
    {
        printf("dup packet error!
");
        return -1;
    }
    pkt1 = av_malloc(sizeof(AVPacketList));
    if(!pkt1)
    {
        printf("malloc AVPacketList error!
");
        return -1;
    }
    pkt1->pkt = *pkt;
    pkt1->next = NULL;
    SDL_LockMutex(q->mutex);
    if(!q->last_pkt){
        q->first_pkt = pkt1;
    }else{
        q->last_pkt->next = pkt1;
    }
    q->last_pkt = pkt1;
    q->nb_packets ++;
    q->size +=pkt1->pkt.size;
    SDL_CondSignal(q->cond);
    SDL_UnlockMutex(q->mutex);
    return 0;
}
static int packet_queue_get(PacketQueue *q,AVPacket *pkt,int block){
    int ret;
    AVPacketList *pkt1;
    SDL_LockMutex(q->mutex);
    for(;;){
        if(quit){
            printf("packet_queue has quit!
");
            ret =-1;
            break;
        }
        pkt1 = q->first_pkt;
        if(pkt1){
            q->first_pkt = pkt1->next;
            if(!q->first_pkt){
                q->last_pkt = NULL;
            }
            q->nb_packets--;
            q->size -= pkt1->pkt.size;
            *pkt = pkt1->pkt;
            av_free(pkt1);
            ret =1;
            break;
        }else if(!block){
            ret =0;
            break;
        }else{
            SDL_CondWait(q->cond,q->mutex);//做了解锁互斥量的动作
        }
    }
    SDL_UnlockMutex(q->mutex);
    return ret;
}
static int audio_decode_frame(AVCodecContext *aCodecCtx,uint8_t *audio_buf,int buf_size)
{
    static AVPacket pkt;
    static uint8_t *audio_pkt_data = NULL;
    static int audio_pkt_size = 0;
    int len1,data_size=0;
    static AVFrame frame;
    int got_frame=0;

    for(;;){
        while(audio_pkt_size >0 ){
            len1 = avcodec_decode_audio4(aCodecCtx, &frame, &got_frame, &pkt);
            if (len1 < 0)
            {
                /* if error, skip frame */
                audio_pkt_size = 0;
                break;
            }
            audio_pkt_data += len1;
            audio_pkt_size -= len1;
            data_size =0;
            if(got_frame){
                data_size = resample(&frame, audio_buf, &buf_size);
                assert(data_size <= buf_size);
            }
            if (data_size <= 0)
            {
                /* No data yet, get more frames */
                continue;
            }
            /* We have data, return it and come back for more later */
            return data_size;
        }
        if (pkt.data)
            av_free_packet(&pkt);
        if (quit)
        {
            return -1;
        }
        if (packet_queue_get(&audioQ, &pkt, 1) < 0)
        {
            printf("packet_queue_get error!
");
            return -1;
        }
        audio_pkt_data = pkt.data;
        audio_pkt_size = pkt.size;
    }
    return 1;
}

//重采样
int resample(AVFrame *af, uint8_t *audio_buf, int *audio_buf_size)
{
    int data_size = 0;
    int resampled_data_size = 0;
    int64_t dec_channel_layout;
    data_size = av_samples_get_buffer_size(NULL,
            av_frame_get_channels(af),
            af->nb_samples,
            af->format, 1);
    dec_channel_layout =(af->channel_layout&&
            av_frame_get_channels(af)== av_get_channel_layout_nb_channels(
                                    af->channel_layout)) ?
                    af->channel_layout :
                    av_get_default_channel_layout(av_frame_get_channels(af));
    if (af->format != audio_hw_params_src.fmt
            || af->sample_rate != audio_hw_params_src.freq
            || dec_channel_layout != audio_hw_params_src.channel_layout
            || !swr_ctx)
    {
        swr_free(&swr_ctx);
        swr_ctx = swr_alloc_set_opts(NULL, audio_hw_params_tgt.channel_layout,
                audio_hw_params_tgt.fmt, audio_hw_params_tgt.freq,
                dec_channel_layout, af->format, af->sample_rate, 0, NULL);
        if (!swr_ctx || swr_init(swr_ctx) < 0)
        {
            av_log(NULL, AV_LOG_ERROR,
                    "Cannot create sample rate converter for conversion of %d Hz %s %d channels to %d Hz %s %d channels!
",
                    af->sample_rate, av_get_sample_fmt_name(af->format),
                    av_frame_get_channels(af), audio_hw_params_tgt.freq,
                    av_get_sample_fmt_name(audio_hw_params_tgt.fmt),
                    audio_hw_params_tgt.channels);
            swr_free(&swr_ctx);
            return -1;
        }
        printf("swr_init
");
        audio_hw_params_src.channels = av_frame_get_channels(af);
        audio_hw_params_src.fmt = af->format;
        audio_hw_params_src.freq = af->sample_rate;
    }

    if (swr_ctx)
    {
        const uint8_t **in = (const uint8_t **) af->extended_data;
        uint8_t **out = &audio_buf;
        int out_count = (int64_t) af->nb_samples * audio_hw_params_tgt.freq
                / af->sample_rate + 256;
        int out_size = av_samples_get_buffer_size(NULL,
                audio_hw_params_tgt.channels, out_count,
                audio_hw_params_tgt.fmt, 0);
        int len2;
        if (out_size < 0)
        {
            av_log(NULL, AV_LOG_ERROR, "av_samples_get_buffer_size() failed
");
            return -1;
        }
        av_fast_malloc(&audio_buf, audio_buf_size, out_size);
        if (!audio_buf)
            return AVERROR(ENOMEM);
        len2 = swr_convert(swr_ctx, out, out_count, in, af->nb_samples);
        if (len2 < 0)
        {
            av_log(NULL, AV_LOG_ERROR, "swr_convert() failed
");
            return -1;
        }
        if (len2 == out_count)
        {
            av_log(NULL, AV_LOG_WARNING,
                    "audio buffer is probably too small
");
            if (swr_init(swr_ctx) < 0)
                swr_free(&swr_ctx);
        }
        resampled_data_size = len2 * audio_hw_params_tgt.channels
                * av_get_bytes_per_sample(audio_hw_params_tgt.fmt);
    }
    else
    {
        audio_buf = af->data[0];
        resampled_data_size = data_size;
    }

    return resampled_data_size;
}

#ifdef _cplusplus
}
#endif

代码那么多，不好理解呀～～～～～

所以程序还是最好模块化和分文件来做才行，或者封装成一个类。