=====================================================
=====================================================
打开流：stream_component_open（）/* open a given stream. Return 0 if OK */ /** * @brief stream_component_open * @param is * @param stream_index 流索引 * @return Return 0 if OK */ static int stream_component_open(VideoState *is, int stream_index) { AVFormatContext *ic = is->ic; AVCodecContext *avctx; AVCodec *codec; const char *forced_codec_name = NULL; AVDictionary *opts = NULL; AVDictionaryEntry *t = NULL; int sample_rate, nb_channels; int64_t channel_layout; int ret = 0; int stream_lowres = lowres; if (stream_index < 0 || stream_index >= ic->nb_streams) return -1; /* 为解码器分配一个编解码器上下文结构体 */ avctx = avcodec_alloc_context3(NULL); if (!avctx) return AVERROR(ENOMEM); /* 将码流中的编解码器信息拷贝到新分配的编解码器上下文结构体 */ ret = avcodec_parameters_to_context(avctx, ic->streams[stream_index]->codecpar); if (ret < 0) goto fail; // 设置pkt_timebase avctx->pkt_timebase = ic->streams[stream_index]->time_base; /* 根据codec_id查找解码器 */ codec = avcodec_find_decoder(avctx->codec_id); switch(avctx->codec_type){ case AVMEDIA_TYPE_AUDIO : is->last_audio_stream = stream_index; forced_codec_name = audio_codec_name; break; case AVMEDIA_TYPE_SUBTITLE: is->last_subtitle_stream = stream_index; forced_codec_name = subtitle_codec_name; break; case AVMEDIA_TYPE_VIDEO : is->last_video_stream = stream_index; forced_codec_name = video_codec_name; break; } if (forced_codec_name) codec = avcodec_find_decoder_by_name(forced_codec_name); if (!codec) { if (forced_codec_name) av_log(NULL, AV_LOG_WARNING, "No codec could be found with name '%s' ", forced_codec_name); else av_log(NULL, AV_LOG_WARNING, "No decoder could be found for codec %s ", avcodec_get_name(avctx->codec_id)); ret = AVERROR(EINVAL); goto fail; } avctx->codec_id = codec->id; if (stream_lowres > codec->max_lowres) { av_log(avctx, AV_LOG_WARNING, "The maximum value for lowres supported by the decoder is %d ", codec->max_lowres); stream_lowres = codec->max_lowres; } avctx->lowres = stream_lowres; if (fast) avctx->flags2 |= AV_CODEC_FLAG2_FAST; opts = filter_codec_opts(codec_opts, avctx->codec_id, ic, ic->streams[stream_index], codec); if (!av_dict_get(opts, "threads", NULL, 0)) av_dict_set(&opts, "threads", "auto", 0); if (stream_lowres) av_dict_set_int(&opts, "lowres", stream_lowres, 0); if (avctx->codec_type == AVMEDIA_TYPE_VIDEO || avctx->codec_type == AVMEDIA_TYPE_AUDIO) av_dict_set(&opts, "refcounted_frames", "1", 0); if ((ret = avcodec_open2(avctx, codec, &opts)) < 0) { goto fail; } if ((t = av_dict_get(opts, "", NULL, AV_DICT_IGNORE_SUFFIX))) { av_log(NULL, AV_LOG_ERROR, "Option %s not found. ", t->key); ret = AVERROR_OPTION_NOT_FOUND; goto fail; } is->eof = 0; ic->streams[stream_index]->discard = AVDISCARD_DEFAULT; switch (avctx->codec_type) { case AVMEDIA_TYPE_AUDIO: #if CONFIG_AVFILTER { AVFilterContext *sink; is->audio_filter_src.freq = avctx->sample_rate; is->audio_filter_src.channels = avctx->channels; is->audio_filter_src.channel_layout = get_valid_channel_layout(avctx->channel_layout, avctx->channels); is->audio_filter_src.fmt = avctx->sample_fmt; if ((ret = configure_audio_filters(is, afilters, 0)) < 0) goto fail; sink = is->out_audio_filter; sample_rate = av_buffersink_get_sample_rate(sink); nb_channels = av_buffersink_get_channels(sink); channel_layout = av_buffersink_get_channel_layout(sink); } #else sample_rate = avctx->sample_rate; nb_channels = avctx->channels; channel_layout = avctx->channel_layout; #endif /* prepare audio output 准备音频输出*/ if ((ret = audio_open(is, channel_layout, nb_channels, sample_rate, &is->audio_tgt)) < 0) goto fail; is->audio_hw_buf_size = ret; is->audio_src = is->audio_tgt; is->audio_buf_size = 0; is->audio_buf_index = 0; /* init averaging filter 初始化averaging滤镜, 非audio master时使用 */ is->audio_diff_avg_coef = exp(log(0.01) / AUDIO_DIFF_AVG_NB); is->audio_diff_avg_count = 0; /* 由于我们没有精确的音频数据填充FIFO,故只有在大于该阈值时才进行校正音频同步*/ is->audio_diff_threshold = (double)(is->audio_hw_buf_size) / is->audio_tgt.bytes_per_sec; is->audio_stream = stream_index; // 获取audio的stream索引 is->audio_st = ic->streams[stream_index]; // 获取audio的stream指针 // 初始化ffplay封装的音频解码器 decoder_init(&is->auddec, avctx, &is->audioq, is->continue_read_thread); if ((is->ic->iformat->flags & (AVFMT_NOBINSEARCH | AVFMT_NOGENSEARCH | AVFMT_NO_BYTE_SEEK)) && !is->ic->iformat->read_seek) { is->auddec.start_pts = is->audio_st->start_time; is->auddec.start_pts_tb = is->audio_st->time_base; } // 启动音频解码线程 if ((ret = decoder_start(&is->auddec, audio_thread, "audio_decoder", is)) < 0) goto out; SDL_PauseAudioDevice(audio_dev, 0); break; case AVMEDIA_TYPE_VIDEO: is->video_stream = stream_index; // 获取video的stream索引 is->video_st = ic->streams[stream_index];// 获取video的stream指针 // 初始化ffplay封装的视频解码器 decoder_init(&is->viddec, avctx, &is->videoq, is->continue_read_thread); // 启动视频频解码线程 if ((ret = decoder_start(&is->viddec, video_thread, "video_decoder", is)) < 0) goto out; is->queue_attachments_req = 1; // 使能请求mp3、aac等音频文件的封面 break; case AVMEDIA_TYPE_SUBTITLE: // 视频是类似逻辑处理 is->subtitle_stream = stream_index; is->subtitle_st = ic->streams[stream_index]; decoder_init(&is->subdec, avctx, &is->subtitleq, is->continue_read_thread); if ((ret = decoder_start(&is->subdec, subtitle_thread, "subtitle_decoder", is)) < 0) goto out; break; default: break; } goto out; fail: avcodec_free_context(&avctx); out: av_dict_free(&opts); return ret; }
启动解码线程 ：decoder_start()
/** * 创建解码线程, audio/video有独立的线程 */ static int decoder_start(Decoder *d, int (*fn)(void *), const char *thread_name, void* arg) { packet_queue_start(d->queue); // 启用对应的packet 队列 d->decoder_tid = SDL_CreateThread(fn, thread_name, arg); // 创建解码线程 if (!d->decoder_tid) { av_log(NULL, AV_LOG_ERROR, "SDL_CreateThread(): %s ", SDL_GetError()); return AVERROR(ENOMEM); } return 0; }
启动packet队列 packet_queue_start()
static void packet_queue_start(PacketQueue *q) { SDL_LockMutex(q->mutex); q->abort_request = 0; packet_queue_put_private(q, &flush_pkt); //这里放入了一个flush_pkt SDL_UnlockMutex(q->mutex); }
pakcet包插入队列：packet_queue_put_private()
static int packet_queue_put_private(PacketQueue *q, AVPacket *pkt) { MyAVPacketList *pkt1; if (q->abort_request) //如果已中止，则放入失败 return -1; pkt1 = av_malloc(sizeof(MyAVPacketList)); //分配节点内存 if (!pkt1) //内存不足，则放入失败 return -1; // 没有做引用计数，那这里也说明av_read_frame不会释放替用户释放buffer。 pkt1->pkt = *pkt; //拷贝AVPacket(浅拷贝，AVPacket.data等内存并没有拷贝) pkt1->next = NULL; if (pkt == &flush_pkt)//如果放入的是flush_pkt，需要增加队列的播放序列号，以区分不连续的两段数据 { q->serial++; printf("q->serial = %d ", q->serial); } pkt1->serial = q->serial; //用队列序列号标记节点 /* 队列操作：如果last_pkt为空，说明队列是空的，新增节点为队头； * 否则，队列有数据，则让原队尾的next为新增节点。 最后将队尾指向新增节点 */ if (!q->last_pkt) q->first_pkt = pkt1; else q->last_pkt->next = pkt1; q->last_pkt = pkt1; //队列属性操作：增加节点数、cache大小、cache总时长, 用来控制队列的大小 q->nb_packets++; q->size += pkt1->pkt.size + sizeof(*pkt1); q->duration += pkt1->pkt.duration; /* XXX: should duplicate packet data in DV case */ //发出信号，表明当前队列中有数据了，通知等待中的读线程可以取数据了 SDL_CondSignal(q->cond); return 0; }
flush_pkt包用来区分连续不同的流，这里有几个地方用到：
- 队列初始化的时候
- seek的时候，插入，可以冲刷之前缓存的流
视频解码线程：video_thread()
// 视频解码线程 static int video_thread(void *arg) { VideoState *is = arg; AVFrame *frame = av_frame_alloc(); // 分配解码帧 double pts; // pts double duration; // 帧持续时间 int ret; //1 获取stream timebase AVRational tb = is->video_st->time_base; // 获取stream timebase //2 获取帧率，以便计算每帧picture的duration AVRational frame_rate = av_guess_frame_rate(is->ic, is->video_st, NULL); if (!frame) return AVERROR(ENOMEM); for (;;) { // 循环取出视频解码的帧数据 // 3 获取解码后的视频帧 ret = get_video_frame(is, frame); if (ret < 0) goto the_end; //解码结束, 什么时候会结束 if (!ret) //没有解码得到画面, 什么情况下会得不到解后的帧 continue; // 4 计算帧持续时间和换算pts值为秒 // 1/帧率 = duration 单位秒, 没有帧率时则设置为0, 有帧率帧计算出帧间隔 duration = (frame_rate.num && frame_rate.den ? av_q2d((AVRational){frame_rate.den, frame_rate.num}) : 0); // 根据AVStream timebase计算出pts值, 单位为秒 pts = (frame->pts == AV_NOPTS_VALUE) ? NAN : frame->pts * av_q2d(tb); // 5 将解码后的视频帧插入队列 ret = queue_picture(is, frame, pts, duration, frame->pkt_pos, is->viddec.pkt_serial); // 6 释放frame对应的数据 av_frame_unref(frame); if (ret < 0) // 返回值小于0则退出线程 goto the_end; } the_end: av_frame_free(&frame); return 0; }
获取视频帧：get_video_frame()
/** * @brief 获取视频帧 * @param is * @param frame 指向获取的视频帧 * @return */ static int get_video_frame(VideoState *is, AVFrame *frame) { int got_picture; // 1. 获取解码后的视频帧 if ((got_picture = decoder_decode_frame(&is->viddec, frame, NULL)) < 0) { return -1; // 返回-1意味着要退出解码线程, 所以要分析decoder_decode_frame什么情况下返回-1 } if (got_picture) { // 2. 分析获取到的该帧是否要drop掉, 该机制的目的是在放入帧队列前先drop掉过时的视频帧 double dpts = NAN; if (frame->pts != AV_NOPTS_VALUE) dpts = av_q2d(is->video_st->time_base) * frame->pts; //计算出秒为单位的pts frame->sample_aspect_ratio = av_guess_sample_aspect_ratio(is->ic, is->video_st, frame); if (framedrop>0 || // 允许drop帧 (framedrop && get_master_sync_type(is) != AV_SYNC_VIDEO_MASTER))//非视频同步模式 { if (frame->pts != AV_NOPTS_VALUE) { // pts值有效 double diff = dpts - get_master_clock(is); if (!isnan(diff) && // 差值有效 fabs(diff) < AV_NOSYNC_THRESHOLD && // 差值在可同步范围呢 diff - is->frame_last_filter_delay < 0 && // 和过滤器有关系 is->viddec.pkt_serial == is->vidclk.serial && // 同一序列的包 is->videoq.nb_packets) { // packet队列至少有1帧数据 is->frame_drops_early++; printf("%s(%d) diff:%lfs, drop frame, drops:%d ", __FUNCTION__, __LINE__, diff, is->frame_drops_early); av_frame_unref(frame); got_picture = 0; } } } } return got_picture; }
decoder_decode_frame()
static int decoder_decode_frame(Decoder *d, AVFrame *frame, AVSubtitle *sub) { int ret = AVERROR(EAGAIN); for (;;) { AVPacket pkt; // 1. 流连续情况下获取解码后的帧 if (d->queue->serial == d->pkt_serial) { // 1.1 先判断是否是同一播放序列的数据 do { if (d->queue->abort_request) return -1; // 是否请求退出 // 1.2. 获取解码帧 switch (d->avctx->codec_type) { case AVMEDIA_TYPE_VIDEO: ret = avcodec_receive_frame(d->avctx, frame); //printf("frame pts:%ld, dts:%ld ", frame->pts, frame->pkt_dts); if (ret >= 0) { if (decoder_reorder_pts == -1) { frame->pts = frame->best_effort_timestamp; } else if (!decoder_reorder_pts) { frame->pts = frame->pkt_dts; } } break; case AVMEDIA_TYPE_AUDIO: ret = avcodec_receive_frame(d->avctx, frame); if (ret >= 0) { AVRational tb = (AVRational){1, frame->sample_rate}; if (frame->pts != AV_NOPTS_VALUE) frame->pts = av_rescale_q(frame->pts, d->avctx->pkt_timebase, tb); else if (d->next_pts != AV_NOPTS_VALUE) frame->pts = av_rescale_q(d->next_pts, d->next_pts_tb, tb); if (frame->pts != AV_NOPTS_VALUE) { d->next_pts = frame->pts + frame->nb_samples; d->next_pts_tb = tb; } } break; } // 1.3. 检查解码是否已经结束，解码结束返回0 if (ret == AVERROR_EOF) { d->finished = d->pkt_serial; printf("avcodec_flush_buffers %s(%d) ", __FUNCTION__, __LINE__); avcodec_flush_buffers(d->avctx); return 0; } // 1.4. 正常解码返回1 if (ret >= 0) return 1; } while (ret != AVERROR(EAGAIN)); // 1.5 没帧可读时ret返回EAGIN，需要继续送packet } // 2 获取一个packet，如果播放序列不一致(数据不连续)则过滤掉“过时”的packet do { // 2.1 如果没有数据可读则唤醒read_thread, 实际是continue_read_thread SDL_cond if (d->queue->nb_packets == 0) // 没有数据可读 SDL_CondSignal(d->empty_queue_cond);// 通知read_thread放入packet // 2.2 如果还有pending的packet则使用它 if (d->packet_pending) { av_packet_move_ref(&pkt, &d->pkt); d->packet_pending = 0; } else { // 2.3 阻塞式读取packet if (packet_queue_get(d->queue, &pkt, 1, &d->pkt_serial) < 0) return -1; } if(d->queue->serial != d->pkt_serial) { // darren自己的代码 printf("%s(%d) discontinue:queue->serial:%d,pkt_serial:%d ", __FUNCTION__, __LINE__, d->queue->serial, d->pkt_serial); av_packet_unref(&pkt); // fixed me? 释放要过滤的packet } } while (d->queue->serial != d->pkt_serial);// 如果不是同一播放序列(流不连续)则继续读取 // 3 将packet送入解码器 if (pkt.data == flush_pkt.data) {// // when seeking or when switching to a different stream avcodec_flush_buffers(d->avctx); //清空里面的缓存帧 d->finished = 0; // 重置为0 d->next_pts = d->start_pts; // 主要用在了audio d->next_pts_tb = d->start_pts_tb;// 主要用在了audio } else { if (d->avctx->codec_type == AVMEDIA_TYPE_SUBTITLE) { int got_frame = 0; ret = avcodec_decode_subtitle2(d->avctx, sub, &got_frame, &pkt); if (ret < 0) { ret = AVERROR(EAGAIN); } else { if (got_frame && !pkt.data) { d->packet_pending = 1; av_packet_move_ref(&d->pkt, &pkt); } ret = got_frame ? 0 : (pkt.data ? AVERROR(EAGAIN) : AVERROR_EOF); } } else { if (avcodec_send_packet(d->avctx, &pkt) == AVERROR(EAGAIN)) { av_log(d->avctx, AV_LOG_ERROR, "Receive_frame and send_packet both returned EAGAIN, which is an API violation. "); d->packet_pending = 1; av_packet_move_ref(&d->pkt, &pkt); } } av_packet_unref(&pkt); // 一定要自己去释放音视频数据 } } }
packet_pending含义：我们主要分析这个端代码：
if (avcodec_send_packet(d->avctx, &pkt) == AVERROR(EAGAIN)) { av_log(d->avctx, AV_LOG_ERROR, "Receive_frame and send_packet both returned EAGAIN, which is an API violation. "); d->packet_pending = 1; av_packet_move_ref(&d->pkt, &pkt); }
我们看看ffmpeg源码：
int attribute_align_arg avcodec_send_packet(AVCodecContext *avctx, const AVPacket *avpkt) { AVCodecInternal *avci = avctx->internal; int ret; if (!avcodec_is_open(avctx) || !av_codec_is_decoder(avctx->codec)) return AVERROR(EINVAL); if (avctx->internal->draining) return AVERROR_EOF; if (avpkt && !avpkt->size && avpkt->data) return AVERROR(EINVAL); av_packet_unref(avci->buffer_pkt); if (avpkt && (avpkt->data || avpkt->side_data_elems)) { ret = av_packet_ref(avci->buffer_pkt, avpkt); if (ret < 0) return ret; } ret = av_bsf_send_packet(avci->filter.bsfs[0], avci->buffer_pkt); if (ret < 0) { av_packet_unref(avci->buffer_pkt); return ret; } if (!avci->buffer_frame->buf[0]) { ret = decode_receive_frame_internal(avctx, avci->buffer_frame); if (ret < 0 && ret != AVERROR(EAGAIN) && ret != AVERROR_EOF) return ret; } return 0; } int av_bsf_send_packet(AVBSFContext *ctx, AVPacket *pkt) { int ret; if (!pkt || (!pkt->data && !pkt->side_data_elems)) { ctx->internal->eof = 1; return 0; } if (ctx->internal->eof) { av_log(ctx, AV_LOG_ERROR, "A non-NULL packet sent after an EOF. "); return AVERROR(EINVAL); } if (ctx->internal->buffer_pkt->data || ctx->internal->buffer_pkt->side_data_elems) return AVERROR(EAGAIN); ret = av_packet_make_refcounted(pkt); if (ret < 0) return ret; av_packet_move_ref(ctx->internal->buffer_pkt, pkt); return 0; }
可以看到，返回AVERROR(EAGAIN)表示有数据，这个其实没有送进去，所以这里就比较好理解了，就是缓存一下数据。