507 lines
17 KiB
C++
507 lines
17 KiB
C++
/*
|
|
* Copyright (c) 2012, The WebRTC project authors. All rights reserved.
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions are
|
|
* met:
|
|
*
|
|
* * Redistributions of source code must retain the above copyright
|
|
* notice, this list of conditions and the following disclaimer.
|
|
*
|
|
* * Redistributions in binary form must reproduce the above copyright
|
|
* notice, this list of conditions and the following disclaimer in
|
|
* the documentation and/or other materials provided with the
|
|
* distribution.
|
|
*
|
|
* * Neither the name of Google nor the names of its contributors may
|
|
* be used to endorse or promote products derived from this software
|
|
* without specific prior written permission.
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
* HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
*/
|
|
|
|
#ifndef WEBRTCGMPVIDEOCODEC_H_
|
|
#define WEBRTCGMPVIDEOCODEC_H_
|
|
|
|
#include <queue>
|
|
#include <string>
|
|
|
|
#include "nsThreadUtils.h"
|
|
#include "mozilla/Monitor.h"
|
|
#include "mozilla/Mutex.h"
|
|
#include "mozilla/Telemetry.h"
|
|
|
|
#include "mozIGeckoMediaPluginService.h"
|
|
#include "MediaConduitInterface.h"
|
|
#include "AudioConduit.h"
|
|
#include "PerformanceRecorder.h"
|
|
#include "VideoConduit.h"
|
|
#include "api/video/video_frame_type.h"
|
|
#include "modules/video_coding/include/video_codec_interface.h"
|
|
#include "common_video/h264/h264_bitstream_parser.h"
|
|
|
|
#include "gmp-video-host.h"
|
|
#include "GMPVideoDecoderProxy.h"
|
|
#include "GMPVideoEncoderProxy.h"
|
|
|
|
#include "jsapi/PeerConnectionImpl.h"
|
|
|
|
namespace mozilla {
|
|
|
|
class GmpInitDoneRunnable : public Runnable {
|
|
public:
|
|
explicit GmpInitDoneRunnable(std::string aPCHandle)
|
|
: Runnable("GmpInitDoneRunnable"),
|
|
mResult(WEBRTC_VIDEO_CODEC_OK),
|
|
mPCHandle(std::move(aPCHandle)) {}
|
|
|
|
NS_IMETHOD Run() override {
|
|
Telemetry::Accumulate(Telemetry::WEBRTC_GMP_INIT_SUCCESS,
|
|
mResult == WEBRTC_VIDEO_CODEC_OK);
|
|
if (mResult == WEBRTC_VIDEO_CODEC_OK) {
|
|
// Might be useful to notify the PeerConnection about successful init
|
|
// someday.
|
|
return NS_OK;
|
|
}
|
|
|
|
PeerConnectionWrapper wrapper(mPCHandle);
|
|
if (wrapper.impl()) {
|
|
wrapper.impl()->OnMediaError(mError);
|
|
}
|
|
return NS_OK;
|
|
}
|
|
|
|
void Dispatch(int32_t aResult, const std::string& aError = "") {
|
|
mResult = aResult;
|
|
mError = aError;
|
|
nsCOMPtr<nsIThread> mainThread(do_GetMainThread());
|
|
if (mainThread) {
|
|
// For some reason, the compiler on CI is treating |this| as a const
|
|
// pointer, despite the fact that we're in a non-const function. And,
|
|
// interestingly enough, correcting this doesn't require a const_cast.
|
|
mainThread->Dispatch(do_AddRef(static_cast<nsIRunnable*>(this)),
|
|
NS_DISPATCH_NORMAL);
|
|
}
|
|
}
|
|
|
|
int32_t Result() { return mResult; }
|
|
|
|
private:
|
|
int32_t mResult;
|
|
const std::string mPCHandle;
|
|
std::string mError;
|
|
};
|
|
|
|
// Hold a frame for later decode
|
|
class GMPDecodeData {
|
|
public:
|
|
GMPDecodeData(const webrtc::EncodedImage& aInputImage, bool aMissingFrames,
|
|
int64_t aRenderTimeMs)
|
|
: mImage(aInputImage),
|
|
mMissingFrames(aMissingFrames),
|
|
mRenderTimeMs(aRenderTimeMs) {
|
|
// We want to use this for queuing, and the calling code recycles the
|
|
// buffer on return from Decode()
|
|
MOZ_RELEASE_ASSERT(aInputImage.size() <
|
|
(std::numeric_limits<size_t>::max() >> 1));
|
|
}
|
|
|
|
~GMPDecodeData() = default;
|
|
|
|
const webrtc::EncodedImage mImage;
|
|
const bool mMissingFrames;
|
|
const int64_t mRenderTimeMs;
|
|
};
|
|
|
|
class RefCountedWebrtcVideoEncoder {
|
|
public:
|
|
NS_INLINE_DECL_PURE_VIRTUAL_REFCOUNTING
|
|
|
|
// Implement sort of WebrtcVideoEncoder interface and support refcounting.
|
|
// (We cannot use |Release|, since that's needed for nsRefPtr)
|
|
virtual int32_t InitEncode(
|
|
const webrtc::VideoCodec* aCodecSettings,
|
|
const webrtc::VideoEncoder::Settings& aSettings) = 0;
|
|
|
|
virtual int32_t Encode(
|
|
const webrtc::VideoFrame& aInputImage,
|
|
const std::vector<webrtc::VideoFrameType>* aFrameTypes) = 0;
|
|
|
|
virtual int32_t RegisterEncodeCompleteCallback(
|
|
webrtc::EncodedImageCallback* aCallback) = 0;
|
|
|
|
virtual int32_t Shutdown() = 0;
|
|
|
|
virtual int32_t SetRates(
|
|
const webrtc::VideoEncoder::RateControlParameters& aParameters) = 0;
|
|
|
|
virtual MediaEventSource<uint64_t>* InitPluginEvent() = 0;
|
|
|
|
virtual MediaEventSource<uint64_t>* ReleasePluginEvent() = 0;
|
|
|
|
virtual WebrtcVideoEncoder::EncoderInfo GetEncoderInfo() const = 0;
|
|
|
|
protected:
|
|
virtual ~RefCountedWebrtcVideoEncoder() = default;
|
|
};
|
|
|
|
class WebrtcGmpVideoEncoder : public GMPVideoEncoderCallbackProxy,
|
|
public RefCountedWebrtcVideoEncoder {
|
|
public:
|
|
NS_INLINE_DECL_THREADSAFE_REFCOUNTING(WebrtcGmpVideoEncoder, final);
|
|
|
|
WebrtcGmpVideoEncoder(const webrtc::SdpVideoFormat& aFormat,
|
|
std::string aPCHandle);
|
|
|
|
// Implement VideoEncoder interface, sort of.
|
|
// (We cannot use |Release|, since that's needed for nsRefPtr)
|
|
int32_t InitEncode(const webrtc::VideoCodec* aCodecSettings,
|
|
const webrtc::VideoEncoder::Settings& aSettings) override;
|
|
|
|
int32_t Encode(
|
|
const webrtc::VideoFrame& aInputImage,
|
|
const std::vector<webrtc::VideoFrameType>* aFrameTypes) override;
|
|
|
|
int32_t RegisterEncodeCompleteCallback(
|
|
webrtc::EncodedImageCallback* aCallback) override;
|
|
|
|
int32_t Shutdown() override;
|
|
|
|
int32_t SetRates(
|
|
const webrtc::VideoEncoder::RateControlParameters& aParameters) override;
|
|
|
|
WebrtcVideoEncoder::EncoderInfo GetEncoderInfo() const override;
|
|
|
|
MediaEventSource<uint64_t>* InitPluginEvent() override {
|
|
return &mInitPluginEvent;
|
|
}
|
|
|
|
MediaEventSource<uint64_t>* ReleasePluginEvent() override {
|
|
return &mReleasePluginEvent;
|
|
}
|
|
|
|
// GMPVideoEncoderCallback virtual functions.
|
|
virtual void Terminated() override;
|
|
|
|
virtual void Encoded(GMPVideoEncodedFrame* aEncodedFrame,
|
|
const nsTArray<uint8_t>& aCodecSpecificInfo) override;
|
|
|
|
virtual void Error(GMPErr aError) override {}
|
|
|
|
private:
|
|
virtual ~WebrtcGmpVideoEncoder();
|
|
|
|
static void InitEncode_g(const RefPtr<WebrtcGmpVideoEncoder>& aThis,
|
|
const GMPVideoCodec& aCodecParams,
|
|
int32_t aNumberOfCores, uint32_t aMaxPayloadSize,
|
|
const RefPtr<GmpInitDoneRunnable>& aInitDone);
|
|
int32_t GmpInitDone(GMPVideoEncoderProxy* aGMP, GMPVideoHost* aHost,
|
|
const GMPVideoCodec& aCodecParams,
|
|
std::string* aErrorOut);
|
|
int32_t GmpInitDone(GMPVideoEncoderProxy* aGMP, GMPVideoHost* aHost,
|
|
std::string* aErrorOut);
|
|
int32_t InitEncoderForSize(unsigned short aWidth, unsigned short aHeight,
|
|
std::string* aErrorOut);
|
|
static void ReleaseGmp_g(const RefPtr<WebrtcGmpVideoEncoder>& aEncoder);
|
|
void Close_g();
|
|
|
|
class InitDoneCallback : public GetGMPVideoEncoderCallback {
|
|
public:
|
|
InitDoneCallback(const RefPtr<WebrtcGmpVideoEncoder>& aEncoder,
|
|
const RefPtr<GmpInitDoneRunnable>& aInitDone,
|
|
const GMPVideoCodec& aCodecParams)
|
|
: mEncoder(aEncoder),
|
|
mInitDone(aInitDone),
|
|
mCodecParams(aCodecParams) {}
|
|
|
|
virtual void Done(GMPVideoEncoderProxy* aGMP,
|
|
GMPVideoHost* aHost) override {
|
|
std::string errorOut;
|
|
int32_t result =
|
|
mEncoder->GmpInitDone(aGMP, aHost, mCodecParams, &errorOut);
|
|
|
|
mInitDone->Dispatch(result, errorOut);
|
|
}
|
|
|
|
private:
|
|
const RefPtr<WebrtcGmpVideoEncoder> mEncoder;
|
|
const RefPtr<GmpInitDoneRunnable> mInitDone;
|
|
const GMPVideoCodec mCodecParams;
|
|
};
|
|
|
|
static void Encode_g(const RefPtr<WebrtcGmpVideoEncoder>& aEncoder,
|
|
webrtc::VideoFrame aInputImage,
|
|
std::vector<webrtc::VideoFrameType> aFrameTypes);
|
|
void RegetEncoderForResolutionChange(
|
|
uint32_t aWidth, uint32_t aHeight,
|
|
const RefPtr<GmpInitDoneRunnable>& aInitDone);
|
|
|
|
class InitDoneForResolutionChangeCallback
|
|
: public GetGMPVideoEncoderCallback {
|
|
public:
|
|
InitDoneForResolutionChangeCallback(
|
|
const RefPtr<WebrtcGmpVideoEncoder>& aEncoder,
|
|
const RefPtr<GmpInitDoneRunnable>& aInitDone, uint32_t aWidth,
|
|
uint32_t aHeight)
|
|
: mEncoder(aEncoder),
|
|
mInitDone(aInitDone),
|
|
mWidth(aWidth),
|
|
mHeight(aHeight) {}
|
|
|
|
virtual void Done(GMPVideoEncoderProxy* aGMP,
|
|
GMPVideoHost* aHost) override {
|
|
std::string errorOut;
|
|
int32_t result = mEncoder->GmpInitDone(aGMP, aHost, &errorOut);
|
|
if (result != WEBRTC_VIDEO_CODEC_OK) {
|
|
mInitDone->Dispatch(result, errorOut);
|
|
return;
|
|
}
|
|
|
|
result = mEncoder->InitEncoderForSize(mWidth, mHeight, &errorOut);
|
|
mInitDone->Dispatch(result, errorOut);
|
|
}
|
|
|
|
private:
|
|
const RefPtr<WebrtcGmpVideoEncoder> mEncoder;
|
|
const RefPtr<GmpInitDoneRunnable> mInitDone;
|
|
const uint32_t mWidth;
|
|
const uint32_t mHeight;
|
|
};
|
|
|
|
static int32_t SetRates_g(RefPtr<WebrtcGmpVideoEncoder> aThis,
|
|
uint32_t aNewBitRateKbps, Maybe<double> aFrameRate);
|
|
|
|
nsCOMPtr<mozIGeckoMediaPluginService> mMPS;
|
|
nsCOMPtr<nsIThread> mGMPThread;
|
|
GMPVideoEncoderProxy* mGMP;
|
|
// Used to handle a race where Release() is called while init is in progress
|
|
bool mInitting;
|
|
GMPVideoHost* mHost;
|
|
GMPVideoCodec mCodecParams;
|
|
uint32_t mMaxPayloadSize;
|
|
const webrtc::CodecParameterMap mFormatParams;
|
|
webrtc::CodecSpecificInfo mCodecSpecificInfo;
|
|
webrtc::H264BitstreamParser mH264BitstreamParser;
|
|
// Protects mCallback
|
|
Mutex mCallbackMutex MOZ_UNANNOTATED;
|
|
webrtc::EncodedImageCallback* mCallback;
|
|
Maybe<uint64_t> mCachedPluginId;
|
|
const std::string mPCHandle;
|
|
|
|
struct InputImageData {
|
|
int64_t timestamp_us;
|
|
};
|
|
// Map rtp time -> input image data
|
|
DataMutex<std::map<uint64_t, InputImageData>> mInputImageMap;
|
|
|
|
MediaEventProducer<uint64_t> mInitPluginEvent;
|
|
MediaEventProducer<uint64_t> mReleasePluginEvent;
|
|
};
|
|
|
|
// Basically a strong ref to a RefCountedWebrtcVideoEncoder, that also
|
|
// translates from Release() to RefCountedWebrtcVideoEncoder::Shutdown(),
|
|
// since we need RefCountedWebrtcVideoEncoder::Release() for managing the
|
|
// refcount. The webrtc.org code gets one of these, so it doesn't unilaterally
|
|
// delete the "real" encoder.
|
|
class WebrtcVideoEncoderProxy : public WebrtcVideoEncoder {
|
|
public:
|
|
explicit WebrtcVideoEncoderProxy(
|
|
RefPtr<RefCountedWebrtcVideoEncoder> aEncoder)
|
|
: mEncoderImpl(std::move(aEncoder)) {}
|
|
|
|
virtual ~WebrtcVideoEncoderProxy() {
|
|
RegisterEncodeCompleteCallback(nullptr);
|
|
}
|
|
|
|
MediaEventSource<uint64_t>* InitPluginEvent() override {
|
|
return mEncoderImpl->InitPluginEvent();
|
|
}
|
|
|
|
MediaEventSource<uint64_t>* ReleasePluginEvent() override {
|
|
return mEncoderImpl->ReleasePluginEvent();
|
|
}
|
|
|
|
int32_t InitEncode(const webrtc::VideoCodec* aCodecSettings,
|
|
const WebrtcVideoEncoder::Settings& aSettings) override {
|
|
return mEncoderImpl->InitEncode(aCodecSettings, aSettings);
|
|
}
|
|
|
|
int32_t Encode(
|
|
const webrtc::VideoFrame& aInputImage,
|
|
const std::vector<webrtc::VideoFrameType>* aFrameTypes) override {
|
|
return mEncoderImpl->Encode(aInputImage, aFrameTypes);
|
|
}
|
|
|
|
int32_t RegisterEncodeCompleteCallback(
|
|
webrtc::EncodedImageCallback* aCallback) override {
|
|
return mEncoderImpl->RegisterEncodeCompleteCallback(aCallback);
|
|
}
|
|
|
|
int32_t Release() override { return mEncoderImpl->Shutdown(); }
|
|
|
|
void SetRates(const RateControlParameters& aParameters) override {
|
|
mEncoderImpl->SetRates(aParameters);
|
|
}
|
|
|
|
EncoderInfo GetEncoderInfo() const override {
|
|
return mEncoderImpl->GetEncoderInfo();
|
|
}
|
|
|
|
private:
|
|
const RefPtr<RefCountedWebrtcVideoEncoder> mEncoderImpl;
|
|
};
|
|
|
|
class WebrtcGmpVideoDecoder : public GMPVideoDecoderCallbackProxy {
|
|
public:
|
|
WebrtcGmpVideoDecoder(std::string aPCHandle, TrackingId aTrackingId);
|
|
NS_INLINE_DECL_THREADSAFE_REFCOUNTING(WebrtcGmpVideoDecoder, final);
|
|
|
|
// Implement VideoEncoder interface, sort of.
|
|
// (We cannot use |Release|, since that's needed for nsRefPtr)
|
|
virtual bool Configure(const webrtc::VideoDecoder::Settings& settings);
|
|
virtual int32_t Decode(const webrtc::EncodedImage& aInputImage,
|
|
bool aMissingFrames, int64_t aRenderTimeMs);
|
|
virtual int32_t RegisterDecodeCompleteCallback(
|
|
webrtc::DecodedImageCallback* aCallback);
|
|
|
|
virtual int32_t ReleaseGmp();
|
|
|
|
MediaEventSource<uint64_t>* InitPluginEvent() { return &mInitPluginEvent; }
|
|
|
|
MediaEventSource<uint64_t>* ReleasePluginEvent() {
|
|
return &mReleasePluginEvent;
|
|
}
|
|
|
|
// GMPVideoDecoderCallbackProxy
|
|
virtual void Terminated() override;
|
|
|
|
virtual void Decoded(GMPVideoi420Frame* aDecodedFrame) override;
|
|
|
|
virtual void ReceivedDecodedReferenceFrame(
|
|
const uint64_t aPictureId) override {
|
|
MOZ_CRASH();
|
|
}
|
|
|
|
virtual void ReceivedDecodedFrame(const uint64_t aPictureId) override {
|
|
MOZ_CRASH();
|
|
}
|
|
|
|
virtual void InputDataExhausted() override {}
|
|
|
|
virtual void DrainComplete() override {}
|
|
|
|
virtual void ResetComplete() override {}
|
|
|
|
virtual void Error(GMPErr aError) override { mDecoderStatus = aError; }
|
|
|
|
private:
|
|
virtual ~WebrtcGmpVideoDecoder();
|
|
|
|
static void Configure_g(const RefPtr<WebrtcGmpVideoDecoder>& aThis,
|
|
const webrtc::VideoDecoder::Settings& settings,
|
|
const RefPtr<GmpInitDoneRunnable>& aInitDone);
|
|
int32_t GmpInitDone(GMPVideoDecoderProxy* aGMP, GMPVideoHost* aHost,
|
|
std::string* aErrorOut);
|
|
static void ReleaseGmp_g(const RefPtr<WebrtcGmpVideoDecoder>& aDecoder);
|
|
void Close_g();
|
|
|
|
class InitDoneCallback : public GetGMPVideoDecoderCallback {
|
|
public:
|
|
explicit InitDoneCallback(const RefPtr<WebrtcGmpVideoDecoder>& aDecoder,
|
|
const RefPtr<GmpInitDoneRunnable>& aInitDone)
|
|
: mDecoder(aDecoder), mInitDone(aInitDone) {}
|
|
|
|
virtual void Done(GMPVideoDecoderProxy* aGMP,
|
|
GMPVideoHost* aHost) override {
|
|
std::string errorOut;
|
|
int32_t result = mDecoder->GmpInitDone(aGMP, aHost, &errorOut);
|
|
|
|
mInitDone->Dispatch(result, errorOut);
|
|
}
|
|
|
|
private:
|
|
const RefPtr<WebrtcGmpVideoDecoder> mDecoder;
|
|
const RefPtr<GmpInitDoneRunnable> mInitDone;
|
|
};
|
|
|
|
static void Decode_g(const RefPtr<WebrtcGmpVideoDecoder>& aThis,
|
|
UniquePtr<GMPDecodeData>&& aDecodeData);
|
|
|
|
nsCOMPtr<mozIGeckoMediaPluginService> mMPS;
|
|
nsCOMPtr<nsIThread> mGMPThread;
|
|
GMPVideoDecoderProxy* mGMP; // Addref is held for us
|
|
// Used to handle a race where Release() is called while init is in progress
|
|
bool mInitting;
|
|
// Frames queued for decode while mInitting is true
|
|
nsTArray<UniquePtr<GMPDecodeData>> mQueuedFrames;
|
|
GMPVideoHost* mHost;
|
|
// Protects mCallback
|
|
Mutex mCallbackMutex MOZ_UNANNOTATED;
|
|
webrtc::DecodedImageCallback* mCallback;
|
|
Maybe<uint64_t> mCachedPluginId;
|
|
Atomic<GMPErr, ReleaseAcquire> mDecoderStatus;
|
|
const std::string mPCHandle;
|
|
const TrackingId mTrackingId;
|
|
PerformanceRecorderMulti<DecodeStage> mPerformanceRecorder;
|
|
|
|
MediaEventProducer<uint64_t> mInitPluginEvent;
|
|
MediaEventProducer<uint64_t> mReleasePluginEvent;
|
|
};
|
|
|
|
// Basically a strong ref to a WebrtcGmpVideoDecoder, that also translates
|
|
// from Release() to WebrtcGmpVideoDecoder::ReleaseGmp(), since we need
|
|
// WebrtcGmpVideoDecoder::Release() for managing the refcount.
|
|
// The webrtc.org code gets one of these, so it doesn't unilaterally delete
|
|
// the "real" encoder.
|
|
class WebrtcVideoDecoderProxy : public WebrtcVideoDecoder {
|
|
public:
|
|
explicit WebrtcVideoDecoderProxy(std::string aPCHandle,
|
|
TrackingId aTrackingId)
|
|
: mDecoderImpl(new WebrtcGmpVideoDecoder(std::move(aPCHandle),
|
|
std::move(aTrackingId))) {}
|
|
|
|
virtual ~WebrtcVideoDecoderProxy() {
|
|
RegisterDecodeCompleteCallback(nullptr);
|
|
}
|
|
|
|
MediaEventSource<uint64_t>* InitPluginEvent() override {
|
|
return mDecoderImpl->InitPluginEvent();
|
|
}
|
|
|
|
MediaEventSource<uint64_t>* ReleasePluginEvent() override {
|
|
return mDecoderImpl->ReleasePluginEvent();
|
|
}
|
|
|
|
bool Configure(const Settings& settings) override {
|
|
return mDecoderImpl->Configure(settings);
|
|
}
|
|
|
|
int32_t Decode(const webrtc::EncodedImage& aInputImage, bool aMissingFrames,
|
|
int64_t aRenderTimeMs) override {
|
|
return mDecoderImpl->Decode(aInputImage, aMissingFrames, aRenderTimeMs);
|
|
}
|
|
|
|
int32_t RegisterDecodeCompleteCallback(
|
|
webrtc::DecodedImageCallback* aCallback) override {
|
|
return mDecoderImpl->RegisterDecodeCompleteCallback(aCallback);
|
|
}
|
|
|
|
int32_t Release() override { return mDecoderImpl->ReleaseGmp(); }
|
|
|
|
private:
|
|
const RefPtr<WebrtcGmpVideoDecoder> mDecoderImpl;
|
|
};
|
|
|
|
} // namespace mozilla
|
|
|
|
#endif
|