/*
* Copyright (C) 2010 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
//#define LOG_NDEBUG 0
#define LOG_TAG "NuPlayerRenderer"
#include <utils/Log.h>
#include "NuPlayerRenderer.h"
#include <media/stagefright/foundation/ABuffer.h>
#include <media/stagefright/foundation/ADebug.h>
#include <media/stagefright/foundation/AMessage.h>
namespace android {
// static
const int64_t NuPlayer::Renderer::kMinPositionUpdateDelayUs = 100000ll;
NuPlayer::Renderer::Renderer(
const sp<MediaPlayerBase::AudioSink> &sink,
const sp<AMessage> ¬ify,
uint32_t flags)
: mAudioSink(sink),
mNotify(notify),
mFlags(flags),
mNumFramesWritten(0),
mDrainAudioQueuePending(false),
mDrainVideoQueuePending(false),
mAudioQueueGeneration(0),
mVideoQueueGeneration(0),
mAnchorTimeMediaUs(-1),
mAnchorTimeRealUs(-1),
mFlushingAudio(false),
mFlushingVideo(false),
mHasAudio(false),
mHasVideo(false),
mSyncQueues(false),
mPaused(false),
mVideoRenderingStarted(false),
mLastPositionUpdateUs(-1ll),
mVideoLateByUs(0ll) {
}
NuPlayer::Renderer::~Renderer() {
}
void NuPlayer::Renderer::queueBuffer(
bool audio,
const sp<ABuffer> &buffer,
const sp<AMessage> ¬ifyConsumed) {
sp<AMessage> msg = new AMessage(kWhatQueueBuffer, id());
msg->setInt32("audio", static_cast<int32_t>(audio));
msg->setBuffer("buffer", buffer);
msg->setMessage("notifyConsumed", notifyConsumed);
msg->post();
}
void NuPlayer::Renderer::queueEOS(bool audio, status_t finalResult) {
CHECK_NE(finalResult, (status_t)OK);
sp<AMessage> msg = new AMessage(kWhatQueueEOS, id());
msg->setInt32("audio", static_cast<int32_t>(audio));
msg->setInt32("finalResult", finalResult);
msg->post();
}
void NuPlayer::Renderer::flush(bool audio) {
{
Mutex::Autolock autoLock(mFlushLock);
if (audio) {
CHECK(!mFlushingAudio);
mFlushingAudio = true;
} else {
CHECK(!mFlushingVideo);
mFlushingVideo = true;
}
}
sp<AMessage> msg = new AMessage(kWhatFlush, id());
msg->setInt32("audio", static_cast<int32_t>(audio));
msg->post();
}
void NuPlayer::Renderer::signalTimeDiscontinuity() {
CHECK(mAudioQueue.empty());
CHECK(mVideoQueue.empty());
mAnchorTimeMediaUs = -1;
mAnchorTimeRealUs = -1;
mSyncQueues = mHasAudio && mHasVideo;
}
void NuPlayer::Renderer::pause() {
(new AMessage(kWhatPause, id()))->post();
}
void NuPlayer::Renderer::resume() {
(new AMessage(kWhatResume, id()))->post();
}
void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) {
switch (msg->what()) {
case kWhatDrainAudioQueue:
{
int32_t generation;
CHECK(msg->findInt32("generation", &generation));
if (generation != mAudioQueueGeneration) {
break;
}
mDrainAudioQueuePending = false;
if (onDrainAudioQueue()) {
uint32_t numFramesPlayed;
CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed),
(status_t)OK);
uint32_t numFramesPendingPlayout =
mNumFramesWritten - numFramesPlayed;
// This is how long the audio sink will have data to
// play back.
int64_t delayUs =
mAudioSink->msecsPerFrame()
* numFramesPendingPlayout * 1000ll;
// Let's give it more data after about half that time
// has elapsed.
postDrainAudioQueue(delayUs / 2);
}
break;
}
case kWhatDrainVideoQueue:
{
int32_t generation;
CHECK(msg->findInt32("generation", &generation));
if (generation != mVideoQueueGeneration) {
break;
}
mDrainVideoQueuePending = false;
onDrainVideoQueue();
postDrainVideoQueue();
break;
}
case kWhatQueueBuffer:
{
onQueueBuffer(msg);
break;
}
case kWhatQueueEOS:
{
onQueueEOS(msg);
break;
}
case kWhatFlush:
{
onFlush(msg);
break;
}
case kWhatAudioSinkChanged:
{
onAudioSinkChanged();
break;
}
case kWhatPause:
{
onPause();
break;
}
case kWhatResume:
{
onResume();
break;
}
default:
TRESPASS();
break;
}
}
void NuPlayer::Renderer::postDrainAudioQueue(int64_t delayUs) {
if (mDrainAudioQueuePending || mSyncQueues || mPaused) {
return;
}
if (mAudioQueue.empty()) {
return;
}
mDrainAudioQueuePending = true;
sp<AMessage> msg = new AMessage(kWhatDrainAudioQueue, id());
msg->setInt32("generation", mAudioQueueGeneration);
msg->post(delayUs);
}
void NuPlayer::Renderer::signalAudioSinkChanged() {
(new AMessage(kWhatAudioSinkChanged, id()))->post();
}
bool NuPlayer::Renderer::onDrainAudioQueue() {
uint32_t numFramesPlayed;
if (mAudioSink->getPosition(&numFramesPlayed) != OK) {
return false;
}
ssize_t numFramesAvailableToWrite =
mAudioSink->frameCount() - (mNumFramesWritten - numFramesPlayed);
#if 0
if (numFramesAvailableToWrite == mAudioSink->frameCount()) {
ALOGI("audio sink underrun");
} else {
ALOGV("audio queue has %d frames left to play",
mAudioSink->frameCount() - numFramesAvailableToWrite);
}
#endif
size_t numBytesAvailableToWrite =
numFramesAvailableToWrite * mAudioSink->frameSize();
while (numBytesAvailableToWrite > 0 && !mAudioQueue.empty()) {
QueueEntry *entry = &*mAudioQueue.begin();
if (entry->mBuffer == NULL) {
// EOS
notifyEOS(true /* audio */, entry->mFinalResult);
mAudioQueue.erase(mAudioQueue.begin());
entry = NULL;
return false;
}
if (entry->mOffset == 0) {
int64_t mediaTimeUs;
CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
ALOGV("rendering audio at media time %.2f secs", mediaTimeUs / 1E6);
mAnchorTimeMediaUs = mediaTimeUs;
uint32_t numFramesPlayed;
CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed), (status_t)OK);
uint32_t numFramesPendingPlayout =
mNumFramesWritten - numFramesPlayed;
int64_t realTimeOffsetUs =
(mAudioSink->latency() / 2 /* XXX */
+ numFramesPendingPlayout
* mAudioSink->msecsPerFrame()) * 1000ll;
// ALOGI("realTimeOffsetUs = %lld us", realTimeOffsetUs);
mAnchorTimeRealUs =
ALooper::GetNowUs() + realTimeOffsetUs;
}
size_t copy = entry->mBuffer->size() - entry->mOffset;
if (copy > numBytesAvailableToWrite) {
copy = numBytesAvailableToWrite;
}
CHECK_EQ(mAudioSink->write(
entry->mBuffer->data() + entry->mOffset, copy),
(ssize_t)copy);
entry->mOffset += copy;
if (entry->mOffset == entry->mBuffer->size()) {
entry->mNotifyConsumed->post();
mAudioQueue.erase(mAudioQueue.begin());
entry = NULL;
}
numBytesAvailableToWrite -= copy;
size_t copiedFrames = copy / mAudioSink->frameSize();
mNumFramesWritten += copiedFrames;
}
notifyPosition();
return !mAudioQueue.empty();
}
void NuPlayer::Renderer::postDrainVideoQueue() {
if (mDrainVideoQueuePending || mSyncQueues || mPaused) {
return;
}
if (mVideoQueue.empty()) {
return;
}
QueueEntry &entry = *mVideoQueue.begin();
sp<AMessage> msg = new AMessage(kWhatDrainVideoQueue, id());
msg->setInt32("generation", mVideoQueueGeneration);
int64_t delayUs;
if (entry.mBuffer == NULL) {
// EOS doesn't carry a timestamp.
delayUs = 0;
} else if (mFlags & FLAG_REAL_TIME) {
int64_t mediaTimeUs;
CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
delayUs = mediaTimeUs - ALooper::GetNowUs();
} else {
int64_t mediaTimeUs;
CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
if (mAnchorTimeMediaUs < 0) {
delayUs = 0;
if (!mHasAudio) {
mAnchorTimeMediaUs = mediaTimeUs;
mAnchorTimeRealUs = ALooper::GetNowUs();
}
} else {
int64_t realTimeUs =
(mediaTimeUs - mAnchorTimeMediaUs) + mAnchorTimeRealUs;
delayUs = realTimeUs - ALooper::GetNowUs();
}
}
msg->post(delayUs);
mDrainVideoQueuePending = true;
}
void NuPlayer::Renderer::onDrainVideoQueue() {
if (mVideoQueue.empty()) {
return;
}
QueueEntry *entry = &*mVideoQueue.begin();
if (entry->mBuffer == NULL) {
// EOS
notifyEOS(false /* audio */, entry->mFinalResult);
mVideoQueue.erase(mVideoQueue.begin());
entry = NULL;
mVideoLateByUs = 0ll;
notifyPosition();
return;
}
int64_t realTimeUs;
if (mFlags & FLAG_REAL_TIME) {
CHECK(entry->mBuffer->meta()->findInt64("timeUs", &realTimeUs));
} else {
int64_t mediaTimeUs;
CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
realTimeUs = mediaTimeUs - mAnchorTimeMediaUs + mAnchorTimeRealUs;
}
mVideoLateByUs = ALooper::GetNowUs() - realTimeUs;
bool tooLate = (mVideoLateByUs > 40000);
if (tooLate) {
ALOGV("video late by %lld us (%.2f secs)",
mVideoLateByUs, mVideoLateByUs / 1E6);
} else {
ALOGV("rendering video at media time %.2f secs", mediaTimeUs / 1E6);
}
entry->mNotifyConsumed->setInt32("render", !tooLate);
entry->mNotifyConsumed->post();
mVideoQueue.erase(mVideoQueue.begin());
entry = NULL;
if (!mVideoRenderingStarted) {
mVideoRenderingStarted = true;
notifyVideoRenderingStart();
}
notifyPosition();
}
void NuPlayer::Renderer::notifyVideoRenderingStart() {
sp<AMessage> notify = mNotify->dup();
notify->setInt32("what", kWhatVideoRenderingStart);
notify->post();
}
void NuPlayer::Renderer::notifyEOS(bool audio, status_t finalResult) {
sp<AMessage> notify = mNotify->dup();
notify->setInt32("what", kWhatEOS);
notify->setInt32("audio", static_cast<int32_t>(audio));
notify->setInt32("finalResult", finalResult);
notify->post();
}
void NuPlayer::Renderer::onQueueBuffer(const sp<AMessage> &msg) {
int32_t audio;
CHECK(msg->findInt32("audio", &audio));
if (audio) {
mHasAudio = true;
} else {
mHasVideo = true;
}
if (dropBufferWhileFlushing(audio, msg)) {
return;
}
sp<ABuffer> buffer;
CHECK(msg->findBuffer("buffer", &buffer));
sp<AMessage> notifyConsumed;
CHECK(msg->findMessage("notifyConsumed", ¬ifyConsumed));
QueueEntry entry;
entry.mBuffer = buffer;
entry.mNotifyConsumed = notifyConsumed;
entry.mOffset = 0;
entry.mFinalResult = OK;
if (audio) {
mAudioQueue.push_back(entry);
postDrainAudioQueue();
} else {
mVideoQueue.push_back(entry);
postDrainVideoQueue();
}
if (!mSyncQueues || mAudioQueue.empty() || mVideoQueue.empty()) {
return;
}
sp<ABuffer> firstAudioBuffer = (*mAudioQueue.begin()).mBuffer;
sp<ABuffer> firstVideoBuffer = (*mVideoQueue.begin()).mBuffer;
if (firstAudioBuffer == NULL || firstVideoBuffer == NULL) {
// EOS signalled on either queue.
syncQueuesDone();
return;
}
int64_t firstAudioTimeUs;
int64_t firstVideoTimeUs;
CHECK(firstAudioBuffer->meta()
->findInt64("timeUs", &firstAudioTimeUs));
CHECK(firstVideoBuffer->meta()
->findInt64("timeUs", &firstVideoTimeUs));
int64_t diff = firstVideoTimeUs - firstAudioTimeUs;
ALOGV("queueDiff = %.2f secs", diff / 1E6);
if (diff > 100000ll) {
// Audio data starts More than 0.1 secs before video.
// Drop some audio.
(*mAudioQueue.begin()).mNotifyConsumed->post();
mAudioQueue.erase(mAudioQueue.begin());
return;
}
syncQueuesDone();
}
void NuPlayer::Renderer::syncQueuesDone() {
if (!mSyncQueues) {
return;
}
mSyncQueues = false;
if (!mAudioQueue.empty()) {
postDrainAudioQueue();
}
if (!mVideoQueue.empty()) {
postDrainVideoQueue();
}
}
void NuPlayer::Renderer::onQueueEOS(const sp<AMessage> &msg) {
int32_t audio;
CHECK(msg->findInt32("audio", &audio));
if (dropBufferWhileFlushing(audio, msg)) {
return;
}
int32_t finalResult;
CHECK(msg->findInt32("finalResult", &finalResult));
QueueEntry entry;
entry.mOffset = 0;
entry.mFinalResult = finalResult;
if (audio) {
if (mAudioQueue.empty() && mSyncQueues) {
syncQueuesDone();
}
mAudioQueue.push_back(entry);
postDrainAudioQueue();
} else {
if (mVideoQueue.empty() && mSyncQueues) {
syncQueuesDone();
}
mVideoQueue.push_back(entry);
postDrainVideoQueue();
}
}
void NuPlayer::Renderer::onFlush(const sp<AMessage> &msg) {
int32_t audio;
CHECK(msg->findInt32("audio", &audio));
// If we're currently syncing the queues, i.e. dropping audio while
// aligning the first audio/video buffer times and only one of the
// two queues has data, we may starve that queue by not requesting
// more buffers from the decoder. If the other source then encounters
// a discontinuity that leads to flushing, we'll never find the
// corresponding discontinuity on the other queue.
// Therefore we'll stop syncing the queues if at least one of them
// is flushed.
syncQueuesDone();
if (audio) {
flushQueue(&mAudioQueue);
Mutex::Autolock autoLock(mFlushLock);
mFlushingAudio = false;
mDrainAudioQueuePending = false;
++mAudioQueueGeneration;
} else {
flushQueue(&mVideoQueue);
Mutex::Autolock autoLock(mFlushLock);
mFlushingVideo = false;
mDrainVideoQueuePending = false;
++mVideoQueueGeneration;
}
notifyFlushComplete(audio);
}
void NuPlayer::Renderer::flushQueue(List<QueueEntry> *queue) {
while (!queue->empty()) {
QueueEntry *entry = &*queue->begin();
if (entry->mBuffer != NULL) {
entry->mNotifyConsumed->post();
}
queue->erase(queue->begin());
entry = NULL;
}
}
void NuPlayer::Renderer::notifyFlushComplete(bool audio) {
sp<AMessage> notify = mNotify->dup();
notify->setInt32("what", kWhatFlushComplete);
notify->setInt32("audio", static_cast<int32_t>(audio));
notify->post();
}
bool NuPlayer::Renderer::dropBufferWhileFlushing(
bool audio, const sp<AMessage> &msg) {
bool flushing = false;
{
Mutex::Autolock autoLock(mFlushLock);
if (audio) {
flushing = mFlushingAudio;
} else {
flushing = mFlushingVideo;
}
}
if (!flushing) {
return false;
}
sp<AMessage> notifyConsumed;
if (msg->findMessage("notifyConsumed", ¬ifyConsumed)) {
notifyConsumed->post();
}
return true;
}
void NuPlayer::Renderer::onAudioSinkChanged() {
CHECK(!mDrainAudioQueuePending);
mNumFramesWritten = 0;
uint32_t written;
if (mAudioSink->getFramesWritten(&written) == OK) {
mNumFramesWritten = written;
}
}
void NuPlayer::Renderer::notifyPosition() {
if (mAnchorTimeRealUs < 0 || mAnchorTimeMediaUs < 0) {
return;
}
int64_t nowUs = ALooper::GetNowUs();
if (mLastPositionUpdateUs >= 0
&& nowUs < mLastPositionUpdateUs + kMinPositionUpdateDelayUs) {
return;
}
mLastPositionUpdateUs = nowUs;
int64_t positionUs = (nowUs - mAnchorTimeRealUs) + mAnchorTimeMediaUs;
sp<AMessage> notify = mNotify->dup();
notify->setInt32("what", kWhatPosition);
notify->setInt64("positionUs", positionUs);
notify->setInt64("videoLateByUs", mVideoLateByUs);
notify->post();
}
void NuPlayer::Renderer::onPause() {
CHECK(!mPaused);
mDrainAudioQueuePending = false;
++mAudioQueueGeneration;
mDrainVideoQueuePending = false;
++mVideoQueueGeneration;
if (mHasAudio) {
mAudioSink->pause();
}
ALOGV("now paused audio queue has %d entries, video has %d entries",
mAudioQueue.size(), mVideoQueue.size());
mPaused = true;
}
void NuPlayer::Renderer::onResume() {
if (!mPaused) {
return;
}
if (mHasAudio) {
mAudioSink->start();
}
mPaused = false;
if (!mAudioQueue.empty()) {
postDrainAudioQueue();
}
if (!mVideoQueue.empty()) {
postDrainVideoQueue();
}
}
} // namespace android