libavcodec/libstagefright.cpp
Go to the documentation of this file.
00001 /*
00002  * Interface to the Android Stagefright library for
00003  * H/W accelerated H.264 decoding
00004  *
00005  * Copyright (C) 2011 Mohamed Naufal
00006  * Copyright (C) 2011 Martin Storsjö
00007  *
00008  * This file is part of FFmpeg.
00009  *
00010  * FFmpeg is free software; you can redistribute it and/or
00011  * modify it under the terms of the GNU Lesser General Public
00012  * License as published by the Free Software Foundation; either
00013  * version 2.1 of the License, or (at your option) any later version.
00014  *
00015  * FFmpeg is distributed in the hope that it will be useful,
00016  * but WITHOUT ANY WARRANTY; without even the implied warranty of
00017  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00018  * Lesser General Public License for more details.
00019  *
00020  * You should have received a copy of the GNU Lesser General Public
00021  * License along with FFmpeg; if not, write to the Free Software
00022  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
00023  */
00024 
00025 #include <binder/ProcessState.h>
00026 #include <media/stagefright/MetaData.h>
00027 #include <media/stagefright/MediaBufferGroup.h>
00028 #include <media/stagefright/MediaDebug.h>
00029 #include <media/stagefright/MediaDefs.h>
00030 #include <media/stagefright/OMXClient.h>
00031 #include <media/stagefright/OMXCodec.h>
00032 #include <utils/List.h>
00033 #include <new>
00034 
00035 extern "C" {
00036 #include "avcodec.h"
00037 #include "libavutil/imgutils.h"
00038 }
00039 
00040 #define OMX_QCOM_COLOR_FormatYVU420SemiPlanar 0x7FA30C00
00041 
00042 using namespace android;
00043 
00044 struct Frame {
00045     status_t status;
00046     size_t size;
00047     int64_t time;
00048     int key;
00049     uint8_t *buffer;
00050     MediaBuffer* mbuffer;
00051     int32_t w, h;
00052 };
00053 
00054 class CustomSource;
00055 
00056 struct StagefrightContext {
00057     AVCodecContext *avctx;
00058     AVBitStreamFilterContext *bsfc;
00059     uint8_t* orig_extradata;
00060     int orig_extradata_size;
00061     sp<MediaSource> *source;
00062     List<Frame*> *in_queue, *out_queue;
00063     pthread_mutex_t in_mutex, out_mutex;
00064     pthread_cond_t condition;
00065     pthread_t decode_thread_id;
00066 
00067     Frame *end_frame;
00068     bool source_done;
00069     volatile sig_atomic_t thread_started, thread_exited, stop_decode;
00070 
00071     AVFrame ret_frame;
00072 
00073     uint8_t *dummy_buf;
00074     int dummy_bufsize;
00075 
00076     OMXClient *client;
00077     sp<MediaSource> *decoder;
00078     const char *decoder_component;
00079 };
00080 
00081 class CustomSource : public MediaSource {
00082 public:
00083     CustomSource(AVCodecContext *avctx, sp<MetaData> meta) {
00084         s = (StagefrightContext*)avctx->priv_data;
00085         source_meta = meta;
00086         frame_size  = (avctx->width * avctx->height * 3) / 2;
00087         buf_group.add_buffer(new MediaBuffer(frame_size));
00088     }
00089 
00090     virtual sp<MetaData> getFormat() {
00091         return source_meta;
00092     }
00093 
00094     virtual status_t start(MetaData *params) {
00095         return OK;
00096     }
00097 
00098     virtual status_t stop() {
00099         return OK;
00100     }
00101 
00102     virtual status_t read(MediaBuffer **buffer,
00103                           const MediaSource::ReadOptions *options) {
00104         Frame *frame;
00105         status_t ret;
00106 
00107         if (s->thread_exited)
00108             return ERROR_END_OF_STREAM;
00109         pthread_mutex_lock(&s->in_mutex);
00110 
00111         while (s->in_queue->empty())
00112             pthread_cond_wait(&s->condition, &s->in_mutex);
00113 
00114         frame = *s->in_queue->begin();
00115         ret = frame->status;
00116 
00117         if (ret == OK) {
00118             ret = buf_group.acquire_buffer(buffer);
00119             if (ret == OK) {
00120                 memcpy((*buffer)->data(), frame->buffer, frame->size);
00121                 (*buffer)->set_range(0, frame->size);
00122                 (*buffer)->meta_data()->clear();
00123                 (*buffer)->meta_data()->setInt32(kKeyIsSyncFrame,frame->key);
00124                 (*buffer)->meta_data()->setInt64(kKeyTime, frame->time);
00125             } else {
00126                 av_log(s->avctx, AV_LOG_ERROR, "Failed to acquire MediaBuffer\n");
00127             }
00128             av_freep(&frame->buffer);
00129         }
00130 
00131         s->in_queue->erase(s->in_queue->begin());
00132         pthread_mutex_unlock(&s->in_mutex);
00133 
00134         av_freep(&frame);
00135         return ret;
00136     }
00137 
00138 private:
00139     MediaBufferGroup buf_group;
00140     sp<MetaData> source_meta;
00141     StagefrightContext *s;
00142     int frame_size;
00143 };
00144 
00145 void* decode_thread(void *arg)
00146 {
00147     AVCodecContext *avctx = (AVCodecContext*)arg;
00148     StagefrightContext *s = (StagefrightContext*)avctx->priv_data;
00149     Frame* frame;
00150     MediaBuffer *buffer;
00151     int decode_done = 0;
00152     do {
00153         buffer = NULL;
00154         frame = (Frame*)av_mallocz(sizeof(Frame));
00155         if (!frame) {
00156             frame         = s->end_frame;
00157             frame->status = AVERROR(ENOMEM);
00158             decode_done   = 1;
00159             s->end_frame  = NULL;
00160         } else {
00161             frame->status = (*s->decoder)->read(&buffer);
00162             if (frame->status == OK) {
00163                 sp<MetaData> outFormat = (*s->decoder)->getFormat();
00164                 outFormat->findInt32(kKeyWidth , &frame->w);
00165                 outFormat->findInt32(kKeyHeight, &frame->h);
00166                 frame->size    = buffer->range_length();
00167                 frame->mbuffer = buffer;
00168             } else if (frame->status == INFO_FORMAT_CHANGED) {
00169                 if (buffer)
00170                     buffer->release();
00171                 av_free(frame);
00172                 continue;
00173             } else {
00174                 decode_done = 1;
00175             }
00176         }
00177         while (true) {
00178             pthread_mutex_lock(&s->out_mutex);
00179             if (s->out_queue->size() >= 10) {
00180                 pthread_mutex_unlock(&s->out_mutex);
00181                 usleep(10000);
00182                 continue;
00183             }
00184             break;
00185         }
00186         s->out_queue->push_back(frame);
00187         pthread_mutex_unlock(&s->out_mutex);
00188     } while (!decode_done && !s->stop_decode);
00189 
00190     s->thread_exited = true;
00191 
00192     return 0;
00193 }
00194 
00195 static av_cold int Stagefright_init(AVCodecContext *avctx)
00196 {
00197     StagefrightContext *s = (StagefrightContext*)avctx->priv_data;
00198     sp<MetaData> meta, outFormat;
00199     int32_t colorFormat = 0;
00200     int ret;
00201 
00202     if (!avctx->extradata || !avctx->extradata_size || avctx->extradata[0] != 1)
00203         return -1;
00204 
00205     s->avctx = avctx;
00206     s->bsfc  = av_bitstream_filter_init("h264_mp4toannexb");
00207     if (!s->bsfc) {
00208         av_log(avctx, AV_LOG_ERROR, "Cannot open the h264_mp4toannexb BSF!\n");
00209         return -1;
00210     }
00211 
00212     s->orig_extradata_size = avctx->extradata_size;
00213     s->orig_extradata = (uint8_t*) av_mallocz(avctx->extradata_size +
00214                                               FF_INPUT_BUFFER_PADDING_SIZE);
00215     if (!s->orig_extradata) {
00216         ret = AVERROR(ENOMEM);
00217         goto fail;
00218     }
00219     memcpy(s->orig_extradata, avctx->extradata, avctx->extradata_size);
00220 
00221     meta = new MetaData;
00222     if (meta == NULL) {
00223         ret = AVERROR(ENOMEM);
00224         goto fail;
00225     }
00226     meta->setCString(kKeyMIMEType, MEDIA_MIMETYPE_VIDEO_AVC);
00227     meta->setInt32(kKeyWidth, avctx->width);
00228     meta->setInt32(kKeyHeight, avctx->height);
00229     meta->setData(kKeyAVCC, kTypeAVCC, avctx->extradata, avctx->extradata_size);
00230 
00231     android::ProcessState::self()->startThreadPool();
00232 
00233     s->source    = new sp<MediaSource>();
00234     *s->source   = new CustomSource(avctx, meta);
00235     s->in_queue  = new List<Frame*>;
00236     s->out_queue = new List<Frame*>;
00237     s->client    = new OMXClient;
00238     s->end_frame = (Frame*)av_mallocz(sizeof(Frame));
00239     if (s->source == NULL || !s->in_queue || !s->out_queue || !s->client ||
00240         !s->end_frame) {
00241         ret = AVERROR(ENOMEM);
00242         goto fail;
00243     }
00244 
00245     if (s->client->connect() !=  OK) {
00246         av_log(avctx, AV_LOG_ERROR, "Cannot connect OMX client\n");
00247         ret = -1;
00248         goto fail;
00249     }
00250 
00251     s->decoder  = new sp<MediaSource>();
00252     *s->decoder = OMXCodec::Create(s->client->interface(), meta,
00253                                   false, *s->source, NULL,
00254                                   OMXCodec::kClientNeedsFramebuffer);
00255     if ((*s->decoder)->start() !=  OK) {
00256         av_log(avctx, AV_LOG_ERROR, "Cannot start decoder\n");
00257         ret = -1;
00258         s->client->disconnect();
00259         goto fail;
00260     }
00261 
00262     outFormat = (*s->decoder)->getFormat();
00263     outFormat->findInt32(kKeyColorFormat, &colorFormat);
00264     if (colorFormat == OMX_QCOM_COLOR_FormatYVU420SemiPlanar ||
00265         colorFormat == OMX_COLOR_FormatYUV420SemiPlanar)
00266         avctx->pix_fmt = PIX_FMT_NV21;
00267     else
00268         avctx->pix_fmt = PIX_FMT_YUV420P;
00269 
00270     outFormat->findCString(kKeyDecoderComponent, &s->decoder_component);
00271     if (s->decoder_component)
00272         s->decoder_component = av_strdup(s->decoder_component);
00273 
00274     pthread_mutex_init(&s->in_mutex, NULL);
00275     pthread_mutex_init(&s->out_mutex, NULL);
00276     pthread_cond_init(&s->condition, NULL);
00277     return 0;
00278 
00279 fail:
00280     av_bitstream_filter_close(s->bsfc);
00281     av_freep(&s->orig_extradata);
00282     av_freep(&s->end_frame);
00283     delete s->in_queue;
00284     delete s->out_queue;
00285     delete s->client;
00286     return ret;
00287 }
00288 
00289 static int Stagefright_decode_frame(AVCodecContext *avctx, void *data,
00290                                     int *data_size, AVPacket *avpkt)
00291 {
00292     StagefrightContext *s = (StagefrightContext*)avctx->priv_data;
00293     Frame *frame;
00294     MediaBuffer *mbuffer;
00295     status_t status;
00296     size_t size;
00297     uint8_t *buf;
00298     const uint8_t *src_data[3];
00299     int w, h;
00300     int src_linesize[3];
00301     int orig_size = avpkt->size;
00302     AVPacket pkt = *avpkt;
00303     int ret;
00304 
00305     if (!s->thread_started) {
00306         pthread_create(&s->decode_thread_id, NULL, &decode_thread, avctx);
00307         s->thread_started = true;
00308     }
00309 
00310     if (avpkt && avpkt->data) {
00311         av_bitstream_filter_filter(s->bsfc, avctx, NULL, &pkt.data, &pkt.size,
00312                                    avpkt->data, avpkt->size, avpkt->flags & AV_PKT_FLAG_KEY);
00313         avpkt = &pkt;
00314     }
00315 
00316     if (!s->source_done) {
00317         if(!s->dummy_buf) {
00318             s->dummy_buf = (uint8_t*)av_malloc(avpkt->size);
00319             if (!s->dummy_buf)
00320                 return AVERROR(ENOMEM);
00321             s->dummy_bufsize = avpkt->size;
00322             memcpy(s->dummy_buf, avpkt->data, avpkt->size);
00323         }
00324 
00325         frame = (Frame*)av_mallocz(sizeof(Frame));
00326         if (avpkt->data) {
00327             frame->status  = OK;
00328             frame->size    = avpkt->size;
00329             // Stagefright can't handle negative timestamps -
00330             // if needed, work around this by offsetting them manually?
00331             if (avpkt->pts >= 0)
00332                 frame->time    = avpkt->pts;
00333             frame->key     = avpkt->flags & AV_PKT_FLAG_KEY ? 1 : 0;
00334             frame->buffer  = (uint8_t*)av_malloc(avpkt->size);
00335             if (!frame->buffer) {
00336                 av_freep(&frame);
00337                 return AVERROR(ENOMEM);
00338             }
00339             uint8_t *ptr = avpkt->data;
00340             // The OMX.SEC decoder fails without this.
00341             if (avpkt->size == orig_size + avctx->extradata_size) {
00342                 ptr += avctx->extradata_size;
00343                 frame->size = orig_size;
00344             }
00345             memcpy(frame->buffer, ptr, orig_size);
00346         } else {
00347             frame->status  = ERROR_END_OF_STREAM;
00348             s->source_done = true;
00349         }
00350 
00351         while (true) {
00352             if (s->thread_exited) {
00353                 s->source_done = true;
00354                 break;
00355             }
00356             pthread_mutex_lock(&s->in_mutex);
00357             if (s->in_queue->size() >= 10) {
00358                 pthread_mutex_unlock(&s->in_mutex);
00359                 usleep(10000);
00360                 continue;
00361             }
00362             s->in_queue->push_back(frame);
00363             pthread_cond_signal(&s->condition);
00364             pthread_mutex_unlock(&s->in_mutex);
00365             break;
00366         }
00367     }
00368     while (true) {
00369         pthread_mutex_lock(&s->out_mutex);
00370         if (!s->out_queue->empty()) break;
00371         pthread_mutex_unlock(&s->out_mutex);
00372         if (s->source_done) {
00373             usleep(10000);
00374             continue;
00375         } else {
00376             return orig_size;
00377         }
00378     }
00379 
00380     frame = *s->out_queue->begin();
00381     s->out_queue->erase(s->out_queue->begin());
00382     pthread_mutex_unlock(&s->out_mutex);
00383 
00384     mbuffer = frame->mbuffer;
00385     status  = frame->status;
00386     size    = frame->size;
00387     w       = frame->w;
00388     h       = frame->h;
00389     av_freep(&frame);
00390 
00391     if (status == ERROR_END_OF_STREAM)
00392         return 0;
00393     if (status != OK) {
00394         if (status == AVERROR(ENOMEM))
00395             return status;
00396         av_log(avctx, AV_LOG_ERROR, "Decode failed: %x\n", status);
00397         return -1;
00398     }
00399 
00400     // The OMX.SEC decoder doesn't signal the modified width/height
00401     if (s->decoder_component && !strncmp(s->decoder_component, "OMX.SEC", 7) &&
00402         (w & 15 || h & 15)) {
00403         if (((w + 15)&~15) * ((h + 15)&~15) * 3/2 == size) {
00404             w = (w + 15)&~15;
00405             h = (h + 15)&~15;
00406         }
00407     }
00408 
00409     if (!avctx->width || !avctx->height || avctx->width > w || avctx->height > h) {
00410         avctx->width  = w;
00411         avctx->height = h;
00412     }
00413 
00414     ret = avctx->reget_buffer(avctx, &s->ret_frame);
00415     if (ret < 0) {
00416         av_log(avctx, AV_LOG_ERROR, "reget buffer() failed\n");
00417         goto end;
00418     }
00419 
00420     src_linesize[0] = w;
00421     if (avctx->pix_fmt == PIX_FMT_YUV420P)
00422         src_linesize[1] = src_linesize[2] = w/2;
00423     else if (avctx->pix_fmt == PIX_FMT_NV21)
00424         src_linesize[1] = w;
00425 
00426     buf = (uint8_t*)mbuffer->data();
00427     src_data[0] = buf;
00428     src_data[1] = buf + src_linesize[0] * h;
00429     src_data[2] = src_data[1] + src_linesize[1] * h/2;
00430     av_image_copy(s->ret_frame.data, s->ret_frame.linesize,
00431                   src_data, src_linesize,
00432                   avctx->pix_fmt, avctx->width, avctx->height);
00433 
00434     *data_size = sizeof(AVFrame);
00435     *(AVFrame*)data = s->ret_frame;
00436     ret = orig_size;
00437 end:
00438     mbuffer->release();
00439     return ret;
00440 }
00441 
00442 static av_cold int Stagefright_close(AVCodecContext *avctx)
00443 {
00444     StagefrightContext *s = (StagefrightContext*)avctx->priv_data;
00445     Frame *frame;
00446 
00447     if (s->thread_started) {
00448         if (!s->thread_exited) {
00449             s->stop_decode = 1;
00450 
00451             // Make sure decode_thread() doesn't get stuck
00452             pthread_mutex_lock(&s->out_mutex);
00453             while (!s->out_queue->empty()) {
00454                 frame = *s->out_queue->begin();
00455                 s->out_queue->erase(s->out_queue->begin());
00456                 if (frame->size)
00457                     frame->mbuffer->release();
00458                 av_freep(&frame);
00459             }
00460             pthread_mutex_unlock(&s->out_mutex);
00461 
00462             // Feed a dummy frame prior to signalling EOF.
00463             // This is required to terminate the decoder(OMX.SEC)
00464             // when only one frame is read during stream info detection.
00465             if (s->dummy_buf && (frame = (Frame*)av_mallocz(sizeof(Frame)))) {
00466                 frame->status = OK;
00467                 frame->size   = s->dummy_bufsize;
00468                 frame->key    = 1;
00469                 frame->buffer = s->dummy_buf;
00470                 pthread_mutex_lock(&s->in_mutex);
00471                 s->in_queue->push_back(frame);
00472                 pthread_cond_signal(&s->condition);
00473                 pthread_mutex_unlock(&s->in_mutex);
00474                 s->dummy_buf = NULL;
00475             }
00476 
00477             pthread_mutex_lock(&s->in_mutex);
00478             s->end_frame->status = ERROR_END_OF_STREAM;
00479             s->in_queue->push_back(s->end_frame);
00480             pthread_cond_signal(&s->condition);
00481             pthread_mutex_unlock(&s->in_mutex);
00482             s->end_frame = NULL;
00483         }
00484 
00485         pthread_join(s->decode_thread_id, NULL);
00486 
00487         if (s->ret_frame.data[0])
00488             avctx->release_buffer(avctx, &s->ret_frame);
00489 
00490         s->thread_started = false;
00491     }
00492 
00493     while (!s->in_queue->empty()) {
00494         frame = *s->in_queue->begin();
00495         s->in_queue->erase(s->in_queue->begin());
00496         if (frame->size)
00497             av_freep(&frame->buffer);
00498         av_freep(&frame);
00499     }
00500 
00501     while (!s->out_queue->empty()) {
00502         frame = *s->out_queue->begin();
00503         s->out_queue->erase(s->out_queue->begin());
00504         if (frame->size)
00505             frame->mbuffer->release();
00506         av_freep(&frame);
00507     }
00508 
00509     (*s->decoder)->stop();
00510     s->client->disconnect();
00511 
00512     if (s->decoder_component)
00513         av_freep(&s->decoder_component);
00514     av_freep(&s->dummy_buf);
00515     av_freep(&s->end_frame);
00516 
00517     // Reset the extradata back to the original mp4 format, so that
00518     // the next invocation (both when decoding and when called from
00519     // av_find_stream_info) get the original mp4 format extradata.
00520     av_freep(&avctx->extradata);
00521     avctx->extradata = s->orig_extradata;
00522     avctx->extradata_size = s->orig_extradata_size;
00523 
00524     delete s->in_queue;
00525     delete s->out_queue;
00526     delete s->client;
00527     delete s->decoder;
00528     delete s->source;
00529 
00530     pthread_mutex_destroy(&s->in_mutex);
00531     pthread_mutex_destroy(&s->out_mutex);
00532     pthread_cond_destroy(&s->condition);
00533     av_bitstream_filter_close(s->bsfc);
00534     return 0;
00535 }
00536 
00537 AVCodec ff_libstagefright_h264_decoder = {
00538     "libstagefright_h264",
00539     AVMEDIA_TYPE_VIDEO,
00540     CODEC_ID_H264,
00541     sizeof(StagefrightContext),
00542     Stagefright_init,
00543     NULL, //encode
00544     Stagefright_close,
00545     Stagefright_decode_frame,
00546     CODEC_CAP_DELAY,
00547     NULL, //next
00548     NULL, //flush
00549     NULL, //supported_framerates
00550     NULL, //pixel_formats
00551     NULL_IF_CONFIG_SMALL("libstagefright H.264"),
00552 };