From 20928d20b69fa98f87923930181cd2940923a813 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20Jos=C3=A9=20Pereira?= Date: Tue, 22 Dec 2020 16:15:24 -0300 Subject: [PATCH] Enable and configure hardware decode priority (#9213) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * VideoReceiver: Enable and put hardware decode with higher priority over software decode Signed-off-by: Patrick José Pereira * VideoReceiver: Move from old decodebin to decodebin3 decodebin3 is a newer and smarter version of the old decodebin. It can handle different stream types and a better connection between gstreamer elements, fixing some weird video problems. Signed-off-by: Patrick José Pereira * Settings: Add forceVideoDecoder Signed-off-by: Patrick José Pereira * VideoReceiver: Break blacklist to be called outside gstreamer initialization function Signed-off-by: Patrick José Pereira * ScreenTools: Add isLinux property Signed-off-by: Patrick José Pereira * GeneralSettings: Add forceVideoDecoder option Signed-off-by: Patrick José Pereira * FactSystem: Add removeEnumInfo function Signed-off-by: Patrick José Pereira * Settins: Use enum string over string for forceVideoDecode Signed-off-by: Patrick José Pereira * VideoMananager: Call GStreamer blacklist Signed-off-by: Patrick José Pereira --- src/FactSystem/FactMetaData.cc | 12 +++ src/FactSystem/FactMetaData.h | 3 + src/QmlControls/ScreenTools.qml | 1 + src/Settings/Video.SettingsGroup.json | 10 +++ src/Settings/VideoSettings.cc | 36 ++++++++ src/Settings/VideoSettings.h | 10 +++ src/VideoManager/VideoManager.cc | 6 ++ src/VideoReceiver/GStreamer.cc | 54 +++++++++--- src/VideoReceiver/GStreamer.h | 1 + src/VideoReceiver/GstVideoReceiver.cc | 145 ++------------------------------- src/VideoReceiver/GstVideoReceiver.h | 6 +- src/ui/preferences/GeneralSettings.qml | 13 +++ 12 files changed, 142 insertions(+), 155 deletions(-) diff --git a/src/FactSystem/FactMetaData.cc b/src/FactSystem/FactMetaData.cc index 7e1c95a..da864b1 100644 --- a/src/FactSystem/FactMetaData.cc +++ b/src/FactSystem/FactMetaData.cc @@ -709,6 +709,18 @@ void FactMetaData::addEnumInfo(const QString& name, const QVariant& value) _enumValues << value; } +void FactMetaData::removeEnumInfo(const QVariant& value) +{ + const int index = _enumValues.indexOf(value); + if (index < 0) { + qWarning() << "Value does not exist in fact:" << value; + return; + } + + _enumValues.removeAt(index); + _enumStrings.removeAt(index); +} + void FactMetaData::setTranslators(Translator rawTranslator, Translator cookedTranslator) { _rawTranslator = rawTranslator; diff --git a/src/FactSystem/FactMetaData.h b/src/FactSystem/FactMetaData.h index bcc2f74..ae5835b 100644 --- a/src/FactSystem/FactMetaData.h +++ b/src/FactSystem/FactMetaData.h @@ -148,6 +148,9 @@ public: /// Used to add new values to the enum lists after the meta data has been loaded void addEnumInfo(const QString& name, const QVariant& value); + /// Used to remove values from the enum lists after the meta data has been loaded + void removeEnumInfo(const QVariant& value); + void setDecimalPlaces (int decimalPlaces) { _decimalPlaces = decimalPlaces; } void setRawDefaultValue (const QVariant& rawDefaultValue); void setBitmaskInfo (const QStringList& strings, const QVariantList& values); diff --git a/src/QmlControls/ScreenTools.qml b/src/QmlControls/ScreenTools.qml index a151a49..b64ccbc 100644 --- a/src/QmlControls/ScreenTools.qml +++ b/src/QmlControls/ScreenTools.qml @@ -81,6 +81,7 @@ Item { property bool isWindows: ScreenToolsController.isWindows property bool isDebug: ScreenToolsController.isDebug property bool isMac: ScreenToolsController.isMacOS + property bool isLinux: ScreenToolsController.isLinux property bool isTinyScreen: (Screen.width / realPixelDensity) < 120 // 120mm property bool isShortScreen: ((Screen.height / realPixelDensity) < 120) || (ScreenToolsController.isMobile && ((Screen.height / Screen.width) < 0.6)) property bool isHugeScreen: (Screen.width / realPixelDensity) >= (23.5 * 25.4) // 27" monitor diff --git a/src/Settings/Video.SettingsGroup.json b/src/Settings/Video.SettingsGroup.json index 3dc0b68..2d6dfc0 100644 --- a/src/Settings/Video.SettingsGroup.json +++ b/src/Settings/Video.SettingsGroup.json @@ -128,6 +128,16 @@ "longDesc": "If this option is enabled, the rtpjitterbuffer is removed and the video sink is set to assynchronous mode, reducing the latency by about 200 ms.", "type": "bool", "default": false +}, +{ + "name": "forceVideoDecoder", + "shortDesc": "Force specific category of video decode", + "longDesc": "Force the change of prioritization between video decode methods, allowing the user to force some video hardware decode plugins if necessary.", + "type": "uint32", + "enumStrings": "Default,Force software decoder,Force NVIDIA decoder,Force VA-API decoder,Force DirectX3D 11 decoder", + "enumValues": "0,1,2,3,4", + "default": 0, + "qgcRebootRequired": true } ] } diff --git a/src/Settings/VideoSettings.cc b/src/Settings/VideoSettings.cc index 89ae6f6..4801b79 100644 --- a/src/Settings/VideoSettings.cc +++ b/src/Settings/VideoSettings.cc @@ -63,6 +63,20 @@ DECLARE_SETTINGGROUP(Video, "Video") videoSourceVarList.append(QVariant::fromValue(videoSource)); } _nameToMetaDataMap[videoSourceName]->setEnumInfo(videoSourceList, videoSourceVarList); + + const QVariantList removeForceVideoDecodeList{ +#ifdef Q_OS_LINUX + VideoDecoderOptions::ForceVideoDecoderDirectX3D, +#endif +#ifdef Q_OS_WIN + VideoDecoderOptions::ForceVideoDecoderVAAPI, +#endif + }; + + for(const auto& value : removeForceVideoDecodeList) { + _nameToMetaDataMap[forceVideoDecoderName]->removeEnumInfo(value); + } + // Set default value for videoSource _setDefaults(); } @@ -105,6 +119,28 @@ DECLARE_SETTINGSFACT_NO_FUNC(VideoSettings, videoSource) return _videoSourceFact; } +DECLARE_SETTINGSFACT_NO_FUNC(VideoSettings, forceVideoDecoder) +{ + if (!_forceVideoDecoderFact) { + _forceVideoDecoderFact = _createSettingsFact(forceVideoDecoderName); + + _forceVideoDecoderFact->setVisible( +#ifdef Q_OS_LINUX + true +#else +#ifdef Q_OS_WIN + true +#else + false +#endif +#endif + ); + + connect(_forceVideoDecoderFact, &Fact::valueChanged, this, &VideoSettings::_configChanged); + } + return _forceVideoDecoderFact; +} + DECLARE_SETTINGSFACT_NO_FUNC(VideoSettings, udpPort) { if (!_udpPortFact) { diff --git a/src/Settings/VideoSettings.h b/src/Settings/VideoSettings.h index 24373dc..d7ad6c0 100644 --- a/src/Settings/VideoSettings.h +++ b/src/Settings/VideoSettings.h @@ -35,6 +35,16 @@ public: DEFINE_SETTINGFACT(streamEnabled) DEFINE_SETTINGFACT(disableWhenDisarmed) DEFINE_SETTINGFACT(lowLatencyMode) + DEFINE_SETTINGFACT(forceVideoDecoder) + + enum VideoDecoderOptions { + ForceVideoDecoderDefault = 0, + ForceVideoDecoderSoftware, + ForceVideoDecoderNVIDIA, + ForceVideoDecoderVAAPI, + ForceVideoDecoderDirectX3D, + }; + Q_ENUM(VideoDecoderOptions) Q_PROPERTY(bool streamConfigured READ streamConfigured NOTIFY streamConfiguredChanged) Q_PROPERTY(QString rtspVideoSource READ rtspVideoSource CONSTANT) diff --git a/src/VideoManager/VideoManager.cc b/src/VideoManager/VideoManager.cc index 75f0bca..e301428 100644 --- a/src/VideoManager/VideoManager.cc +++ b/src/VideoManager/VideoManager.cc @@ -30,6 +30,7 @@ #if defined(QGC_GST_STREAMING) #include "GStreamer.h" +#include "VideoSettings.h" #else #include "GLVideoItemStub.h" #endif @@ -104,6 +105,11 @@ VideoManager::setToolbox(QGCToolbox *toolbox) connect(pVehicleMgr, &MultiVehicleManager::activeVehicleChanged, this, &VideoManager::_setActiveVehicle); #if defined(QGC_GST_STREAMING) + bool forceSoftware = _videoSettings->forceVideoDecoder()->rawValue() == VideoSettings::VideoDecoderOptions::ForceVideoDecoderSoftware; + bool forceNVIDIA = _videoSettings->forceVideoDecoder()->rawValue() == VideoSettings::VideoDecoderOptions::ForceVideoDecoderNVIDIA; + bool forceVAAPI = _videoSettings->forceVideoDecoder()->rawValue() == VideoSettings::VideoDecoderOptions::ForceVideoDecoderVAAPI; + bool forceD3D11 = _videoSettings->forceVideoDecoder()->rawValue() == VideoSettings::VideoDecoderOptions::ForceVideoDecoderDirectX3D; + GStreamer::blacklist(forceSoftware, forceVAAPI, forceNVIDIA, forceD3D11); #ifndef QGC_DISABLE_UVC // If we are using a UVC camera setup the device name _updateUVC(); diff --git a/src/VideoReceiver/GStreamer.cc b/src/VideoReceiver/GStreamer.cc index 3695947..cbb615d 100644 --- a/src/VideoReceiver/GStreamer.cc +++ b/src/VideoReceiver/GStreamer.cc @@ -105,20 +105,56 @@ static void qgcputenv(const QString& key, const QString& root, const QString& pa } #endif -static void -blacklist() +void +GStreamer::blacklist(bool forceSoftware, bool forceVAAPI, bool forceNVIDIA, bool forceD3D11) { - GstRegistry* reg; + GstRegistry* registry = gst_registry_get(); - if ((reg = gst_registry_get()) == nullptr) { + if (registry == nullptr) { + qCCritical(GStreamerLog) << "Failed to get gstreamer registry."; return; } - GstPluginFeature* plugin; + auto changeRank = [registry](const char* featureName, uint16_t rank) { + GstPluginFeature* feature = gst_registry_lookup_feature(registry, featureName); + if (feature == nullptr) { + qCDebug(GStreamerLog) << "Failed to change ranking of feature:" << featureName; + return; + } + + qCInfo(GStreamerLog) << "Changing feature (" << featureName << ") to use rank:" << rank; + gst_plugin_feature_set_rank(feature, rank); + gst_registry_add_feature(registry, feature); + gst_object_unref(feature); + }; + + // Set rank for specific features + changeRank("bcmdec", GST_RANK_NONE); + + // Force software decode + if (forceSoftware) { + changeRank("avdec_h264", GST_RANK_PRIMARY + 1); + } + + // Enable VAAPI drivers + if (forceVAAPI) { + for(auto name : {"vaapimpeg2dec", "vaapimpeg4dec", "vaapih263dec", "vaapih264dec", "vaapivc1dec"}) { + changeRank(name, GST_RANK_PRIMARY + 1); + } + } - if ((plugin = gst_registry_lookup_feature(reg, "bcmdec")) != nullptr) { - qCCritical(GStreamerLog) << "Disable bcmdec"; - gst_plugin_feature_set_rank(plugin, GST_RANK_NONE); + // Enable NVIDIA's proprietary APIs for hardware video acceleration + if (forceNVIDIA) { + for(auto name : {"nvh265dec", "nvh265sldec", "nvh264dec", "nvh264sldec"}) { + changeRank(name, GST_RANK_PRIMARY + 1); + } + } + + // Enable DirectX3D 11 decoders + if (forceD3D11) { + for(auto name : {"d3d11vp9dec", "d3d11h265dec", "d3d11h264dec"}) { + changeRank(name, GST_RANK_PRIMARY + 1); + } } } @@ -190,8 +226,6 @@ GStreamer::initialize(int argc, char* argv[], int debuglevel) gst_ios_post_init(); #endif - blacklist(); - /* the plugin must be loaded before loading the qml file to register the * GstGLVideoItem qml item * FIXME Add a QQmlExtensionPlugin into qmlglsink to register GstGLVideoItem diff --git a/src/VideoReceiver/GStreamer.h b/src/VideoReceiver/GStreamer.h index 8d96e5d..aec81c3 100644 --- a/src/VideoReceiver/GStreamer.h +++ b/src/VideoReceiver/GStreamer.h @@ -7,6 +7,7 @@ class GStreamer { public: + static void blacklist(bool forceSoftware = false, bool forceVAAPI = false, bool forceNVIDIA = false, bool forceD3D11 = false); static void initialize(int argc, char* argv[], int debuglevel); static void* createVideoSink(QObject* parent, QQuickItem* widget); static void releaseVideoSink(void* sink); diff --git a/src/VideoReceiver/GstVideoReceiver.cc b/src/VideoReceiver/GstVideoReceiver.cc index 345783a..4603157 100644 --- a/src/VideoReceiver/GstVideoReceiver.cc +++ b/src/VideoReceiver/GstVideoReceiver.cc @@ -759,8 +759,6 @@ GstVideoReceiver::_makeSource(const QString& uri) break; } - g_signal_connect(parser, "autoplug-query", G_CALLBACK(_filterParserCaps), nullptr); - gst_bin_add_many(GST_BIN(bin), source, parser, nullptr); // FIXME: AV: Android does not determine MPEG2-TS via parsebin - have to explicitly state which demux to use @@ -848,17 +846,15 @@ GstVideoReceiver::_makeSource(const QString& uri) GstElement* GstVideoReceiver::_makeDecoder(GstCaps* caps, GstElement* videoSink) { - Q_UNUSED(caps); - + Q_UNUSED(caps) + Q_UNUSED(videoSink) GstElement* decoder = nullptr; do { - if ((decoder = gst_element_factory_make("decodebin", nullptr)) == nullptr) { - qCCritical(VideoReceiverLog) << "gst_element_factory_make('decodebin') failed"; + if ((decoder = gst_element_factory_make("decodebin3", nullptr)) == nullptr) { + qCCritical(VideoReceiverLog) << "gst_element_factory_make('decodebin3') failed"; break; } - - g_signal_connect(decoder, "autoplug-query", G_CALLBACK(_autoplugQuery), videoSink); } while(0); return decoder; @@ -1020,7 +1016,7 @@ GstVideoReceiver::_addDecoder(GstElement* src) gst_object_unref(srcpad); srcpad = nullptr; - if ((_decoder = _makeDecoder(caps, _videoSink)) == nullptr) { + if ((_decoder = _makeDecoder()) == nullptr) { qCCritical(VideoReceiverLog) << "_makeDecoder() failed"; gst_caps_unref(caps); caps = nullptr; @@ -1449,137 +1445,6 @@ GstVideoReceiver::_padProbe(GstElement* element, GstPad* pad, gpointer user_data return TRUE; } -gboolean -GstVideoReceiver::_filterParserCaps(GstElement* bin, GstPad* pad, GstElement* element, GstQuery* query, gpointer data) -{ - Q_UNUSED(bin) - Q_UNUSED(pad) - Q_UNUSED(element) - Q_UNUSED(data) - - if (GST_QUERY_TYPE(query) != GST_QUERY_CAPS) { - return FALSE; - } - - GstCaps* srcCaps; - - gst_query_parse_caps(query, &srcCaps); - - if (srcCaps == nullptr || gst_caps_is_any(srcCaps)) { - return FALSE; - } - - GstCaps* sinkCaps = nullptr; - - GstCaps* filter; - - if (sinkCaps == nullptr && (filter = gst_caps_from_string("video/x-h264")) != nullptr) { - if (gst_caps_can_intersect(srcCaps, filter)) { - sinkCaps = gst_caps_from_string("video/x-h264,stream-format=avc"); - } - - gst_caps_unref(filter); - filter = nullptr; - } else if (sinkCaps == nullptr && (filter = gst_caps_from_string("video/x-h265")) != nullptr) { - if (gst_caps_can_intersect(srcCaps, filter)) { - sinkCaps = gst_caps_from_string("video/x-h265,stream-format=hvc1"); - } - - gst_caps_unref(filter); - filter = nullptr; - } - - if (sinkCaps == nullptr) { - return FALSE; - } - - gst_query_set_caps_result(query, sinkCaps); - - gst_caps_unref(sinkCaps); - sinkCaps = nullptr; - - return TRUE; -} - -gboolean -GstVideoReceiver::_autoplugQueryCaps(GstElement* bin, GstPad* pad, GstElement* element, GstQuery* query, gpointer data) -{ - Q_UNUSED(bin) - Q_UNUSED(pad) - Q_UNUSED(element) - - GstElement* glupload = (GstElement* )data; - - GstPad* sinkpad; - - if ((sinkpad = gst_element_get_static_pad(glupload, "sink")) == nullptr) { - qCCritical(VideoReceiverLog) << "No sink pad found"; - return FALSE; - } - - GstCaps* filter; - - gst_query_parse_caps(query, &filter); - - GstCaps* sinkcaps = gst_pad_query_caps(sinkpad, filter); - - gst_query_set_caps_result(query, sinkcaps); - - const gboolean ret = !gst_caps_is_empty(sinkcaps); - - gst_caps_unref(sinkcaps); - sinkcaps = nullptr; - - gst_object_unref(sinkpad); - sinkpad = nullptr; - - return ret; -} - -gboolean -GstVideoReceiver::_autoplugQueryContext(GstElement* bin, GstPad* pad, GstElement* element, GstQuery* query, gpointer data) -{ - Q_UNUSED(bin) - Q_UNUSED(pad) - Q_UNUSED(element) - - GstElement* glsink = (GstElement* )data; - - GstPad* sinkpad; - - if ((sinkpad = gst_element_get_static_pad(glsink, "sink")) == nullptr){ - qCCritical(VideoReceiverLog) << "No sink pad found"; - return FALSE; - } - - const gboolean ret = gst_pad_query(sinkpad, query); - - gst_object_unref(sinkpad); - sinkpad = nullptr; - - return ret; -} - -gboolean -GstVideoReceiver::_autoplugQuery(GstElement* bin, GstPad* pad, GstElement* element, GstQuery* query, gpointer data) -{ - gboolean ret; - - switch (GST_QUERY_TYPE(query)) { - case GST_QUERY_CAPS: - ret = _autoplugQueryCaps(bin, pad, element, query, data); - break; - case GST_QUERY_CONTEXT: - ret = _autoplugQueryContext(bin, pad, element, query, data); - break; - default: - ret = FALSE; - break; - } - - return ret; -} - GstPadProbeReturn GstVideoReceiver::_teeProbe(GstPad* pad, GstPadProbeInfo* info, gpointer user_data) { diff --git a/src/VideoReceiver/GstVideoReceiver.h b/src/VideoReceiver/GstVideoReceiver.h index e9b00b7..82f5555 100644 --- a/src/VideoReceiver/GstVideoReceiver.h +++ b/src/VideoReceiver/GstVideoReceiver.h @@ -103,7 +103,7 @@ protected slots: protected: virtual GstElement* _makeSource(const QString& uri); - virtual GstElement* _makeDecoder(GstCaps* caps, GstElement* videoSink); + virtual GstElement* _makeDecoder(GstCaps* caps = nullptr, GstElement* videoSink = nullptr); virtual GstElement* _makeFileSink(const QString& videoFile, FILE_FORMAT format); virtual void _onNewSourcePad(GstPad* pad); @@ -125,10 +125,6 @@ protected: static void _wrapWithGhostPad(GstElement* element, GstPad* pad, gpointer data); static void _linkPad(GstElement* element, GstPad* pad, gpointer data); static gboolean _padProbe(GstElement* element, GstPad* pad, gpointer user_data); - static gboolean _filterParserCaps(GstElement* bin, GstPad* pad, GstElement* element, GstQuery* query, gpointer data); - static gboolean _autoplugQueryCaps(GstElement* bin, GstPad* pad, GstElement* element, GstQuery* query, gpointer data); - static gboolean _autoplugQueryContext(GstElement* bin, GstPad* pad, GstElement* element, GstQuery* query, gpointer data); - static gboolean _autoplugQuery(GstElement* bin, GstPad* pad, GstElement* element, GstQuery* query, gpointer data); static GstPadProbeReturn _teeProbe(GstPad* pad, GstPadProbeInfo* info, gpointer user_data); static GstPadProbeReturn _videoSinkProbe(GstPad* pad, GstPadProbeInfo* info, gpointer user_data); static GstPadProbeReturn _eosProbe(GstPad* pad, GstPadProbeInfo* info, gpointer user_data); diff --git a/src/ui/preferences/GeneralSettings.qml b/src/ui/preferences/GeneralSettings.qml index 9df8291..c59d397 100644 --- a/src/ui/preferences/GeneralSettings.qml +++ b/src/ui/preferences/GeneralSettings.qml @@ -313,6 +313,19 @@ Rectangle { visible: _showSaveVideoSettings && _videoSettings.enableStorageLimit.value && maxSavedVideoStorageLabel.visible } + QGCLabel { + id: videoDecodeLabel + text: qsTr("Video decode priority") + visible: forceVideoDecoderComboBox.visible + } + FactComboBox { + id: forceVideoDecoderComboBox + Layout.preferredWidth: _comboFieldWidth + fact: _videoSettings.forceVideoDecoder + visible: fact.visible + indexModel: false + } + Item { width: 1; height: 1} FactCheckBox { text: qsTr("Disable When Disarmed")