// Copyright 2013 The Chromium Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. #include "media/cast/video_receiver/video_receiver.h" #include <algorithm> #include "base/bind.h" #include "base/logging.h" #include "base/message_loop/message_loop.h" #include "crypto/encryptor.h" #include "crypto/symmetric_key.h" #include "media/cast/cast_defines.h" #include "media/cast/framer/framer.h" #include "media/cast/video_receiver/video_decoder.h" namespace media { namespace cast { const int64 kMinSchedulingDelayMs = 1; static const int64 kMinTimeBetweenOffsetUpdatesMs = 2000; static const int kTimeOffsetFilter = 8; static const int64_t kMinProcessIntervalMs = 5; // Local implementation of RtpData (defined in rtp_rtcp_defines.h). // Used to pass payload data into the video receiver. class LocalRtpVideoData : public RtpData { public: explicit LocalRtpVideoData(VideoReceiver* video_receiver) : video_receiver_(video_receiver) {} virtual ~LocalRtpVideoData() {} virtual void OnReceivedPayloadData(const uint8* payload_data, size_t payload_size, const RtpCastHeader* rtp_header) OVERRIDE { video_receiver_->IncomingParsedRtpPacket(payload_data, payload_size, *rtp_header); } private: VideoReceiver* video_receiver_; }; // Local implementation of RtpPayloadFeedback (defined in rtp_defines.h) // Used to convey cast-specific feedback from receiver to sender. // Callback triggered by the Framer (cast message builder). class LocalRtpVideoFeedback : public RtpPayloadFeedback { public: explicit LocalRtpVideoFeedback(VideoReceiver* video_receiver) : video_receiver_(video_receiver) { } virtual void CastFeedback(const RtcpCastMessage& cast_message) OVERRIDE { video_receiver_->CastFeedback(cast_message); } private: VideoReceiver* video_receiver_; }; // Local implementation of RtpReceiverStatistics (defined by rtcp.h). // Used to pass statistics data from the RTP module to the RTCP module. class LocalRtpReceiverStatistics : public RtpReceiverStatistics { public: explicit LocalRtpReceiverStatistics(RtpReceiver* rtp_receiver) : rtp_receiver_(rtp_receiver) { } virtual void GetStatistics(uint8* fraction_lost, uint32* cumulative_lost, // 24 bits valid. uint32* extended_high_sequence_number, uint32* jitter) OVERRIDE { rtp_receiver_->GetStatistics(fraction_lost, cumulative_lost, extended_high_sequence_number, jitter); } private: RtpReceiver* rtp_receiver_; }; VideoReceiver::VideoReceiver(scoped_refptr<CastEnvironment> cast_environment, const VideoReceiverConfig& video_config, PacedPacketSender* const packet_sender) : cast_environment_(cast_environment), codec_(video_config.codec), target_delay_delta_( base::TimeDelta::FromMilliseconds(video_config.rtp_max_delay_ms)), frame_delay_(base::TimeDelta::FromMilliseconds( 1000 / video_config.max_frame_rate)), incoming_payload_callback_(new LocalRtpVideoData(this)), incoming_payload_feedback_(new LocalRtpVideoFeedback(this)), rtp_receiver_(cast_environment_->Clock(), NULL, &video_config, incoming_payload_callback_.get()), rtp_video_receiver_statistics_( new LocalRtpReceiverStatistics(&rtp_receiver_)), time_incoming_packet_updated_(false), incoming_rtp_timestamp_(0), weak_factory_(this) { int max_unacked_frames = video_config.rtp_max_delay_ms * video_config.max_frame_rate / 1000; DCHECK(max_unacked_frames) << "Invalid argument"; if (video_config.aes_iv_mask.size() == kAesKeySize && video_config.aes_key.size() == kAesKeySize) { iv_mask_ = video_config.aes_iv_mask; crypto::SymmetricKey* key = crypto::SymmetricKey::Import( crypto::SymmetricKey::AES, video_config.aes_key); decryptor_.reset(new crypto::Encryptor()); decryptor_->Init(key, crypto::Encryptor::CTR, std::string()); } else if (video_config.aes_iv_mask.size() != 0 || video_config.aes_key.size() != 0) { DCHECK(false) << "Invalid crypto configuration"; } framer_.reset(new Framer(cast_environment->Clock(), incoming_payload_feedback_.get(), video_config.incoming_ssrc, video_config.decoder_faster_than_max_frame_rate, max_unacked_frames)); if (!video_config.use_external_decoder) { video_decoder_.reset(new VideoDecoder(video_config, cast_environment)); } rtcp_.reset( new Rtcp(cast_environment_, NULL, packet_sender, NULL, rtp_video_receiver_statistics_.get(), video_config.rtcp_mode, base::TimeDelta::FromMilliseconds(video_config.rtcp_interval), video_config.feedback_ssrc, video_config.incoming_ssrc, video_config.rtcp_c_name)); } VideoReceiver::~VideoReceiver() {} void VideoReceiver::InitializeTimers() { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); ScheduleNextRtcpReport(); ScheduleNextCastMessage(); } void VideoReceiver::GetRawVideoFrame( const VideoFrameDecodedCallback& callback) { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); GetEncodedVideoFrame(base::Bind(&VideoReceiver::DecodeVideoFrame, base::Unretained(this), callback)); } // Called when we have a frame to decode. void VideoReceiver::DecodeVideoFrame( const VideoFrameDecodedCallback& callback, scoped_ptr<EncodedVideoFrame> encoded_frame, const base::TimeTicks& render_time) { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); // Hand the ownership of the encoded frame to the decode thread. cast_environment_->PostTask(CastEnvironment::VIDEO_DECODER, FROM_HERE, base::Bind(&VideoReceiver::DecodeVideoFrameThread, base::Unretained(this), base::Passed(&encoded_frame), render_time, callback)); } // Utility function to run the decoder on a designated decoding thread. void VideoReceiver::DecodeVideoFrameThread( scoped_ptr<EncodedVideoFrame> encoded_frame, const base::TimeTicks render_time, const VideoFrameDecodedCallback& frame_decoded_callback) { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::VIDEO_DECODER)); DCHECK(video_decoder_); if (!(video_decoder_->DecodeVideoFrame(encoded_frame.get(), render_time, frame_decoded_callback))) { // This will happen if we decide to decode but not show a frame. cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE, base::Bind(&VideoReceiver::GetRawVideoFrame, base::Unretained(this), frame_decoded_callback)); } } bool VideoReceiver::DecryptVideoFrame( scoped_ptr<EncodedVideoFrame>* video_frame) { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); DCHECK(decryptor_) << "Invalid state"; if (!decryptor_->SetCounter(GetAesNonce((*video_frame)->frame_id, iv_mask_))) { NOTREACHED() << "Failed to set counter"; return false; } std::string decrypted_video_data; if (!decryptor_->Decrypt((*video_frame)->data, &decrypted_video_data)) { VLOG(1) << "Decryption error"; // Give up on this frame, release it from jitter buffer. framer_->ReleaseFrame((*video_frame)->frame_id); return false; } (*video_frame)->data.swap(decrypted_video_data); return true; } // Called from the main cast thread. void VideoReceiver::GetEncodedVideoFrame( const VideoFrameEncodedCallback& callback) { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); scoped_ptr<EncodedVideoFrame> encoded_frame(new EncodedVideoFrame()); uint32 rtp_timestamp = 0; bool next_frame = false; if (!framer_->GetEncodedVideoFrame(encoded_frame.get(), &rtp_timestamp, &next_frame)) { // We have no video frames. Wait for new packet(s). queued_encoded_callbacks_.push_back(callback); return; } if (decryptor_ && !DecryptVideoFrame(&encoded_frame)) { // Logging already done. queued_encoded_callbacks_.push_back(callback); return; } base::TimeTicks render_time; if (PullEncodedVideoFrame(rtp_timestamp, next_frame, &encoded_frame, &render_time)) { cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE, base::Bind(callback, base::Passed(&encoded_frame), render_time)); } else { // We have a video frame; however we are missing packets and we have time // to wait for new packet(s). queued_encoded_callbacks_.push_back(callback); } } // Should we pull the encoded video frame from the framer? decided by if this is // the next frame or we are running out of time and have to pull the following // frame. // If the frame is too old to be rendered we set the don't show flag in the // video bitstream where possible. bool VideoReceiver::PullEncodedVideoFrame(uint32 rtp_timestamp, bool next_frame, scoped_ptr<EncodedVideoFrame>* encoded_frame, base::TimeTicks* render_time) { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); base::TimeTicks now = cast_environment_->Clock()->NowTicks(); *render_time = GetRenderTime(now, rtp_timestamp); // TODO(mikhal): Store actual render time and not diff. cast_environment_->Logging()->InsertFrameEventWithDelay(kVideoRenderDelay, rtp_timestamp, (*encoded_frame)->frame_id, now - *render_time); // Minimum time before a frame is due to be rendered before we pull it for // decode. base::TimeDelta min_wait_delta = frame_delay_; base::TimeDelta time_until_render = *render_time - now; if (!next_frame && (time_until_render > min_wait_delta)) { // Example: // We have decoded frame 1 and we have received the complete frame 3, but // not frame 2. If we still have time before frame 3 should be rendered we // will wait for 2 to arrive, however if 2 never show up this timer will hit // and we will pull out frame 3 for decoding and rendering. base::TimeDelta time_until_release = time_until_render - min_wait_delta; cast_environment_->PostDelayedTask(CastEnvironment::MAIN, FROM_HERE, base::Bind(&VideoReceiver::PlayoutTimeout, weak_factory_.GetWeakPtr()), time_until_release); VLOG(1) << "Wait before releasing frame " << static_cast<int>((*encoded_frame)->frame_id) << " time " << time_until_release.InMilliseconds(); return false; } base::TimeDelta dont_show_timeout_delta = base::TimeDelta::FromMilliseconds(-kDontShowTimeoutMs); if (codec_ == kVp8 && time_until_render < dont_show_timeout_delta) { (*encoded_frame)->data[0] &= 0xef; VLOG(1) << "Don't show frame " << static_cast<int>((*encoded_frame)->frame_id) << " time_until_render:" << time_until_render.InMilliseconds(); } else { VLOG(1) << "Show frame " << static_cast<int>((*encoded_frame)->frame_id) << " time_until_render:" << time_until_render.InMilliseconds(); } // We have a copy of the frame, release this one. framer_->ReleaseFrame((*encoded_frame)->frame_id); (*encoded_frame)->codec = codec_; return true; } void VideoReceiver::PlayoutTimeout() { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); if (queued_encoded_callbacks_.empty()) return; uint32 rtp_timestamp = 0; bool next_frame = false; scoped_ptr<EncodedVideoFrame> encoded_frame(new EncodedVideoFrame()); if (!framer_->GetEncodedVideoFrame(encoded_frame.get(), &rtp_timestamp, &next_frame)) { // We have no video frames. Wait for new packet(s). // Since the application can post multiple VideoFrameEncodedCallback and // we only check the next frame to play out we might have multiple timeout // events firing after each other; however this should be a rare event. VLOG(1) << "Failed to retrieved a complete frame at this point in time"; return; } VLOG(1) << "PlayoutTimeout retrieved frame " << static_cast<int>(encoded_frame->frame_id); if (decryptor_ && !DecryptVideoFrame(&encoded_frame)) { // Logging already done. return; } base::TimeTicks render_time; if (PullEncodedVideoFrame(rtp_timestamp, next_frame, &encoded_frame, &render_time)) { if (!queued_encoded_callbacks_.empty()) { VideoFrameEncodedCallback callback = queued_encoded_callbacks_.front(); queued_encoded_callbacks_.pop_front(); cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE, base::Bind(callback, base::Passed(&encoded_frame), render_time)); } } else { // We have a video frame; however we are missing packets and we have time // to wait for new packet(s). } } base::TimeTicks VideoReceiver::GetRenderTime(base::TimeTicks now, uint32 rtp_timestamp) { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); // Senders time in ms when this frame was captured. // Note: the senders clock and our local clock might not be synced. base::TimeTicks rtp_timestamp_in_ticks; if (time_offset_.InMilliseconds() == 0) { if (!rtcp_->RtpTimestampInSenderTime(kVideoFrequency, incoming_rtp_timestamp_, &rtp_timestamp_in_ticks)) { // We have not received any RTCP to sync the stream play it out as soon as // possible. return now; } time_offset_ = time_incoming_packet_ - rtp_timestamp_in_ticks; } else if (time_incoming_packet_updated_) { if (rtcp_->RtpTimestampInSenderTime(kVideoFrequency, incoming_rtp_timestamp_, &rtp_timestamp_in_ticks)) { // Time to update the time_offset. base::TimeDelta time_offset = time_incoming_packet_ - rtp_timestamp_in_ticks; time_offset_ = ((kTimeOffsetFilter - 1) * time_offset_ + time_offset) / kTimeOffsetFilter; } } // Reset |time_incoming_packet_updated_| to enable a future measurement. time_incoming_packet_updated_ = false; if (!rtcp_->RtpTimestampInSenderTime(kVideoFrequency, rtp_timestamp, &rtp_timestamp_in_ticks)) { // This can fail if we have not received any RTCP packets in a long time. return now; } return (rtp_timestamp_in_ticks + time_offset_ + target_delay_delta_); } void VideoReceiver::IncomingPacket(const uint8* packet, size_t length, const base::Closure callback) { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); if (Rtcp::IsRtcpPacket(packet, length)) { rtcp_->IncomingRtcpPacket(packet, length); } else { rtp_receiver_.ReceivedPacket(packet, length); } cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE, callback); } void VideoReceiver::IncomingParsedRtpPacket(const uint8* payload_data, size_t payload_size, const RtpCastHeader& rtp_header) { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); base::TimeTicks now = cast_environment_->Clock()->NowTicks(); if (time_incoming_packet_.is_null() || now - time_incoming_packet_ > base::TimeDelta::FromMilliseconds(kMinTimeBetweenOffsetUpdatesMs)) { if (time_incoming_packet_.is_null()) InitializeTimers(); incoming_rtp_timestamp_ = rtp_header.webrtc.header.timestamp; time_incoming_packet_ = now; time_incoming_packet_updated_ = true; } cast_environment_->Logging()->InsertPacketEvent(kPacketReceived, rtp_header.webrtc.header.timestamp, rtp_header.frame_id, rtp_header.packet_id, rtp_header.max_packet_id, payload_size); bool complete = framer_->InsertPacket(payload_data, payload_size, rtp_header); if (!complete) return; // Video frame not complete; wait for more packets. if (queued_encoded_callbacks_.empty()) return; // No pending callback. VideoFrameEncodedCallback callback = queued_encoded_callbacks_.front(); queued_encoded_callbacks_.pop_front(); cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE, base::Bind(&VideoReceiver::GetEncodedVideoFrame, weak_factory_.GetWeakPtr(), callback)); } // Send a cast feedback message. Actual message created in the framer (cast // message builder). void VideoReceiver::CastFeedback(const RtcpCastMessage& cast_message) { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); // TODO(pwestin): wire up log messages. rtcp_->SendRtcpFromRtpReceiver(&cast_message, NULL); time_last_sent_cast_message_= cast_environment_->Clock()->NowTicks(); } // Cast messages should be sent within a maximum interval. Schedule a call // if not triggered elsewhere, e.g. by the cast message_builder. void VideoReceiver::ScheduleNextCastMessage() { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); base::TimeTicks send_time; framer_->TimeToSendNextCastMessage(&send_time); base::TimeDelta time_to_send = send_time - cast_environment_->Clock()->NowTicks(); time_to_send = std::max(time_to_send, base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs)); cast_environment_->PostDelayedTask(CastEnvironment::MAIN, FROM_HERE, base::Bind(&VideoReceiver::SendNextCastMessage, weak_factory_.GetWeakPtr()), time_to_send); } void VideoReceiver::SendNextCastMessage() { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); framer_->SendCastMessage(); // Will only send a message if it is time. ScheduleNextCastMessage(); } // Schedule the next RTCP report to be sent back to the sender. void VideoReceiver::ScheduleNextRtcpReport() { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); base::TimeDelta time_to_next = rtcp_->TimeToSendNextRtcpReport() - cast_environment_->Clock()->NowTicks(); time_to_next = std::max(time_to_next, base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs)); cast_environment_->PostDelayedTask(CastEnvironment::MAIN, FROM_HERE, base::Bind(&VideoReceiver::SendNextRtcpReport, weak_factory_.GetWeakPtr()), time_to_next); } void VideoReceiver::SendNextRtcpReport() { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); rtcp_->SendRtcpFromRtpReceiver(NULL, NULL); ScheduleNextRtcpReport(); } } // namespace cast } // namespace media