Enable and configure hardware decode priority (#9213)

* VideoReceiver: Enable and put hardware decode with higher priority over software decode

Signed-off-by: Patrick José Pereira <patrickelectric@gmail.com>

* VideoReceiver: Move from old decodebin to decodebin3

decodebin3 is a newer and smarter version of the old decodebin.
It can handle different stream types and a better connection
between gstreamer elements, fixing some weird video problems.

Signed-off-by: Patrick José Pereira <patrickelectric@gmail.com>

* Settings: Add forceVideoDecoder

Signed-off-by: Patrick José Pereira <patrickelectric@gmail.com>

* VideoReceiver: Break blacklist to be called outside gstreamer initialization function

Signed-off-by: Patrick José Pereira <patrickelectric@gmail.com>

* ScreenTools: Add isLinux property

Signed-off-by: Patrick José Pereira <patrickelectric@gmail.com>

* GeneralSettings: Add forceVideoDecoder option

Signed-off-by: Patrick José Pereira <patrickelectric@gmail.com>

* FactSystem: Add removeEnumInfo function

Signed-off-by: Patrick José Pereira <patrickelectric@gmail.com>

* Settins: Use enum string over string for forceVideoDecode

Signed-off-by: Patrick José Pereira <patrickelectric@gmail.com>

* VideoMananager: Call GStreamer blacklist

Signed-off-by: Patrick José Pereira <patrickelectric@gmail.com>

src/FactSystem/FactMetaData.cc

@@ -709,6 +709,18 @@ void FactMetaData::addEnumInfo(const QString& name, const QVariant& value)
     _enumValues << value;
 }
 
+void FactMetaData::removeEnumInfo(const QVariant& value)
+{
+    const int index = _enumValues.indexOf(value);
+    if (index < 0) {
+        qWarning() << "Value does not exist in fact:" << value;
+        return;
+    }
+
+    _enumValues.removeAt(index);
+    _enumStrings.removeAt(index);
+}
+
 void FactMetaData::setTranslators(Translator rawTranslator, Translator cookedTranslator)
 {
     _rawTranslator = rawTranslator;

src/FactSystem/FactMetaData.h

@@ -148,6 +148,9 @@ public:
     /// Used to add new values to the enum lists after the meta data has been loaded
     void addEnumInfo(const QString& name, const QVariant& value);
 
+    /// Used to remove values from the enum lists after the meta data has been loaded
+    void removeEnumInfo(const QVariant& value);
+
     void setDecimalPlaces           (int decimalPlaces)                 { _decimalPlaces = decimalPlaces; }
     void setRawDefaultValue         (const QVariant& rawDefaultValue);
     void setBitmaskInfo             (const QStringList& strings, const QVariantList& values);

src/QmlControls/ScreenTools.qml

@@ -81,6 +81,7 @@ Item {
     property bool isWindows:                        ScreenToolsController.isWindows
     property bool isDebug:                          ScreenToolsController.isDebug
     property bool isMac:                            ScreenToolsController.isMacOS
+    property bool isLinux:                          ScreenToolsController.isLinux
     property bool isTinyScreen:                     (Screen.width / realPixelDensity) < 120 // 120mm
     property bool isShortScreen:                    ((Screen.height / realPixelDensity) < 120) || (ScreenToolsController.isMobile && ((Screen.height / Screen.width) < 0.6))
     property bool isHugeScreen:                     (Screen.width / realPixelDensity) >= (23.5 * 25.4) // 27" monitor

src/Settings/Video.SettingsGroup.json

@@ -128,6 +128,16 @@
     "longDesc":  "If this option is enabled, the rtpjitterbuffer is removed and the video sink is set to assynchronous mode, reducing the latency by about 200 ms.",
     "type":             "bool",
     "default":     false
+},
+{
+    "name":             "forceVideoDecoder",
+    "shortDesc":        "Force specific category of video decode",
+    "longDesc":         "Force the change of prioritization between video decode methods, allowing the user to force some video hardware decode plugins if necessary.",
+    "type":             "uint32",
+    "enumStrings":      "Default,Force software decoder,Force NVIDIA decoder,Force VA-API decoder,Force DirectX3D 11 decoder",
+    "enumValues":       "0,1,2,3,4",
+    "default":           0,
+    "qgcRebootRequired": true
 }
 ]
 }

src/Settings/VideoSettings.cc

@@ -63,6 +63,20 @@ DECLARE_SETTINGGROUP(Video, "Video")
         videoSourceVarList.append(QVariant::fromValue(videoSource));
     }
     _nameToMetaDataMap[videoSourceName]->setEnumInfo(videoSourceList, videoSourceVarList);
+
+    const QVariantList removeForceVideoDecodeList{
+#ifdef Q_OS_LINUX
+        VideoDecoderOptions::ForceVideoDecoderDirectX3D,
+#endif
+#ifdef Q_OS_WIN
+        VideoDecoderOptions::ForceVideoDecoderVAAPI,
+#endif
+    };
+
+    for(const auto& value : removeForceVideoDecodeList) {
+        _nameToMetaDataMap[forceVideoDecoderName]->removeEnumInfo(value);
+    }
+
     // Set default value for videoSource
     _setDefaults();
 }
@@ -105,6 +119,28 @@ DECLARE_SETTINGSFACT_NO_FUNC(VideoSettings, videoSource)
     return _videoSourceFact;
 }
 
+DECLARE_SETTINGSFACT_NO_FUNC(VideoSettings, forceVideoDecoder)
+{
+    if (!_forceVideoDecoderFact) {
+        _forceVideoDecoderFact = _createSettingsFact(forceVideoDecoderName);
+
+        _forceVideoDecoderFact->setVisible(
+#ifdef Q_OS_LINUX
+            true
+#else
+#ifdef Q_OS_WIN
+            true
+#else
+            false
+#endif
+#endif
+        );
+
+        connect(_forceVideoDecoderFact, &Fact::valueChanged, this, &VideoSettings::_configChanged);
+    }
+    return _forceVideoDecoderFact;
+}
+
 DECLARE_SETTINGSFACT_NO_FUNC(VideoSettings, udpPort)
 {
     if (!_udpPortFact) {

src/Settings/VideoSettings.h

@@ -35,6 +35,16 @@ public:
     DEFINE_SETTINGFACT(streamEnabled)
     DEFINE_SETTINGFACT(disableWhenDisarmed)
     DEFINE_SETTINGFACT(lowLatencyMode)
+    DEFINE_SETTINGFACT(forceVideoDecoder)
+
+    enum VideoDecoderOptions {
+        ForceVideoDecoderDefault = 0,
+        ForceVideoDecoderSoftware,
+        ForceVideoDecoderNVIDIA,
+        ForceVideoDecoderVAAPI,
+        ForceVideoDecoderDirectX3D,
+    };
+    Q_ENUM(VideoDecoderOptions)
 
     Q_PROPERTY(bool     streamConfigured        READ streamConfigured       NOTIFY streamConfiguredChanged)
     Q_PROPERTY(QString  rtspVideoSource         READ rtspVideoSource        CONSTANT)

src/VideoManager/VideoManager.cc

@@ -30,6 +30,7 @@
 
 #if defined(QGC_GST_STREAMING)
 #include "GStreamer.h"
+#include "VideoSettings.h"
 #else
 #include "GLVideoItemStub.h"
 #endif
@@ -104,6 +105,11 @@ VideoManager::setToolbox(QGCToolbox *toolbox)
    connect(pVehicleMgr, &MultiVehicleManager::activeVehicleChanged, this, &VideoManager::_setActiveVehicle);
 
 #if defined(QGC_GST_STREAMING)
+    bool forceSoftware = _videoSettings->forceVideoDecoder()->rawValue() == VideoSettings::VideoDecoderOptions::ForceVideoDecoderSoftware;
+    bool forceNVIDIA = _videoSettings->forceVideoDecoder()->rawValue() == VideoSettings::VideoDecoderOptions::ForceVideoDecoderNVIDIA;
+    bool forceVAAPI = _videoSettings->forceVideoDecoder()->rawValue() == VideoSettings::VideoDecoderOptions::ForceVideoDecoderVAAPI;
+    bool forceD3D11 = _videoSettings->forceVideoDecoder()->rawValue() == VideoSettings::VideoDecoderOptions::ForceVideoDecoderDirectX3D;
+    GStreamer::blacklist(forceSoftware, forceVAAPI, forceNVIDIA, forceD3D11);
 #ifndef QGC_DISABLE_UVC
    // If we are using a UVC camera setup the device name
    _updateUVC();

src/VideoReceiver/GStreamer.cc

@@ -105,20 +105,56 @@ static void qgcputenv(const QString& key, const QString& root, const QString& pa
 }
 #endif
 
-static void
+void
-blacklist()
+GStreamer::blacklist(bool forceSoftware, bool forceVAAPI, bool forceNVIDIA, bool forceD3D11)
 {
-    GstRegistry* reg;
+    GstRegistry* registry = gst_registry_get();
+
+    if (registry == nullptr) {
+        qCCritical(GStreamerLog) << "Failed to get gstreamer registry.";
+        return;
+    }
 
-    if ((reg = gst_registry_get()) == nullptr) {
+    auto changeRank = [registry](const char* featureName, uint16_t rank) {
+        GstPluginFeature* feature = gst_registry_lookup_feature(registry, featureName);
+        if (feature == nullptr) {
+            qCDebug(GStreamerLog) << "Failed to change ranking of feature:" << featureName;
             return;
         }
 
-    GstPluginFeature* plugin;
+        qCInfo(GStreamerLog) << "Changing feature (" << featureName << ") to use rank:" << rank;
+        gst_plugin_feature_set_rank(feature, rank);
+        gst_registry_add_feature(registry, feature);
+        gst_object_unref(feature);
+    };
 
-    if ((plugin = gst_registry_lookup_feature(reg, "bcmdec")) != nullptr) {
+    // Set rank for specific features
-        qCCritical(GStreamerLog) << "Disable bcmdec";
+    changeRank("bcmdec", GST_RANK_NONE);
-        gst_plugin_feature_set_rank(plugin, GST_RANK_NONE);
+
+    // Force software decode
+    if (forceSoftware) {
+        changeRank("avdec_h264", GST_RANK_PRIMARY + 1);
+    }
+
+    // Enable VAAPI drivers
+    if (forceVAAPI) {
+        for(auto name : {"vaapimpeg2dec", "vaapimpeg4dec", "vaapih263dec", "vaapih264dec", "vaapivc1dec"}) {
+            changeRank(name, GST_RANK_PRIMARY + 1);
+        }
+    }
+
+    // Enable NVIDIA's proprietary APIs for hardware video acceleration
+    if (forceNVIDIA) {
+        for(auto name : {"nvh265dec", "nvh265sldec", "nvh264dec", "nvh264sldec"}) {
+            changeRank(name, GST_RANK_PRIMARY + 1);
+        }
+    }
+
+    // Enable DirectX3D 11 decoders
+    if (forceD3D11) {
+        for(auto name : {"d3d11vp9dec", "d3d11h265dec", "d3d11h264dec"}) {
+            changeRank(name, GST_RANK_PRIMARY + 1);
+        }
     }
 }
 
@@ -190,8 +226,6 @@ GStreamer::initialize(int argc, char* argv[], int debuglevel)
     gst_ios_post_init();
 #endif
 
-    blacklist();
-
     /* the plugin must be loaded before loading the qml file to register the
      * GstGLVideoItem qml item
      * FIXME Add a QQmlExtensionPlugin into qmlglsink to register GstGLVideoItem

src/VideoReceiver/GStreamer.h

@@ -7,6 +7,7 @@
 
 class GStreamer {
 public:
+    static void blacklist(bool forceSoftware = false, bool forceVAAPI = false, bool forceNVIDIA = false, bool forceD3D11 = false);
     static void initialize(int argc, char* argv[], int debuglevel);
     static void* createVideoSink(QObject* parent, QQuickItem* widget);
     static void releaseVideoSink(void* sink);

src/VideoReceiver/GstVideoReceiver.cc

@@ -759,8 +759,6 @@ GstVideoReceiver::_makeSource(const QString& uri)
             break;
         }
 
-        g_signal_connect(parser, "autoplug-query", G_CALLBACK(_filterParserCaps), nullptr);
-
         gst_bin_add_many(GST_BIN(bin), source, parser, nullptr);
 
         // FIXME: AV: Android does not determine MPEG2-TS via parsebin - have to explicitly state which demux to use
@@ -848,17 +846,15 @@ GstVideoReceiver::_makeSource(const QString& uri)
 GstElement*
 GstVideoReceiver::_makeDecoder(GstCaps* caps, GstElement* videoSink)
 {
-    Q_UNUSED(caps);
+    Q_UNUSED(caps)
-
+    Q_UNUSED(videoSink)
     GstElement* decoder = nullptr;
 
     do {
-        if ((decoder = gst_element_factory_make("decodebin", nullptr)) == nullptr) {
+        if ((decoder = gst_element_factory_make("decodebin3", nullptr)) == nullptr) {
-            qCCritical(VideoReceiverLog) << "gst_element_factory_make('decodebin') failed";
+            qCCritical(VideoReceiverLog) << "gst_element_factory_make('decodebin3') failed";
             break;
         }
-
-        g_signal_connect(decoder, "autoplug-query", G_CALLBACK(_autoplugQuery), videoSink);
     } while(0);
 
     return decoder;
@@ -1020,7 +1016,7 @@ GstVideoReceiver::_addDecoder(GstElement* src)
     gst_object_unref(srcpad);
     srcpad = nullptr;
 
-    if ((_decoder = _makeDecoder(caps, _videoSink)) == nullptr) {
+    if ((_decoder = _makeDecoder()) == nullptr) {
         qCCritical(VideoReceiverLog) << "_makeDecoder() failed";
         gst_caps_unref(caps);
         caps = nullptr;
@@ -1449,137 +1445,6 @@ GstVideoReceiver::_padProbe(GstElement* element, GstPad* pad, gpointer user_data
     return TRUE;
 }
 
-gboolean
-GstVideoReceiver::_filterParserCaps(GstElement* bin, GstPad* pad, GstElement* element, GstQuery* query, gpointer data)
-{
-    Q_UNUSED(bin)
-    Q_UNUSED(pad)
-    Q_UNUSED(element)
-    Q_UNUSED(data)
-
-    if (GST_QUERY_TYPE(query) != GST_QUERY_CAPS) {
-        return FALSE;
-    }
-
-    GstCaps* srcCaps;
-
-    gst_query_parse_caps(query, &srcCaps);
-
-    if (srcCaps == nullptr || gst_caps_is_any(srcCaps)) {
-        return FALSE;
-    }
-
-    GstCaps* sinkCaps = nullptr;
-
-    GstCaps* filter;
-
-    if (sinkCaps == nullptr && (filter = gst_caps_from_string("video/x-h264")) != nullptr) {
-        if (gst_caps_can_intersect(srcCaps, filter)) {
-            sinkCaps = gst_caps_from_string("video/x-h264,stream-format=avc");
-        }
-
-        gst_caps_unref(filter);
-        filter = nullptr;
-    } else if (sinkCaps == nullptr && (filter = gst_caps_from_string("video/x-h265")) != nullptr) {
-        if (gst_caps_can_intersect(srcCaps, filter)) {
-            sinkCaps = gst_caps_from_string("video/x-h265,stream-format=hvc1");
-        }
-
-        gst_caps_unref(filter);
-        filter = nullptr;
-    }
-
-    if (sinkCaps == nullptr) {
-        return FALSE;
-    }
-
-    gst_query_set_caps_result(query, sinkCaps);
-
-    gst_caps_unref(sinkCaps);
-    sinkCaps = nullptr;
-
-    return TRUE;
-}
-
-gboolean
-GstVideoReceiver::_autoplugQueryCaps(GstElement* bin, GstPad* pad, GstElement* element, GstQuery* query, gpointer data)
-{
-    Q_UNUSED(bin)
-    Q_UNUSED(pad)
-    Q_UNUSED(element)
-
-    GstElement* glupload = (GstElement* )data;
-
-    GstPad* sinkpad;
-
-    if ((sinkpad = gst_element_get_static_pad(glupload, "sink")) == nullptr) {
-        qCCritical(VideoReceiverLog) << "No sink pad found";
-        return FALSE;
-    }
-
-    GstCaps* filter;
-
-    gst_query_parse_caps(query, &filter);
-
-    GstCaps* sinkcaps = gst_pad_query_caps(sinkpad, filter);
-
-    gst_query_set_caps_result(query, sinkcaps);
-
-    const gboolean ret = !gst_caps_is_empty(sinkcaps);
-
-    gst_caps_unref(sinkcaps);
-    sinkcaps = nullptr;
-
-    gst_object_unref(sinkpad);
-    sinkpad = nullptr;
-
-    return ret;
-}
-
-gboolean
-GstVideoReceiver::_autoplugQueryContext(GstElement* bin, GstPad* pad, GstElement* element, GstQuery* query, gpointer data)
-{
-    Q_UNUSED(bin)
-    Q_UNUSED(pad)
-    Q_UNUSED(element)
-
-    GstElement* glsink = (GstElement* )data;
-
-    GstPad* sinkpad;
-
-    if ((sinkpad = gst_element_get_static_pad(glsink, "sink")) == nullptr){
-        qCCritical(VideoReceiverLog) << "No sink pad found";
-        return FALSE;
-    }
-
-    const gboolean ret = gst_pad_query(sinkpad, query);
-
-    gst_object_unref(sinkpad);
-    sinkpad = nullptr;
-
-    return ret;
-}
-
-gboolean
-GstVideoReceiver::_autoplugQuery(GstElement* bin, GstPad* pad, GstElement* element, GstQuery* query, gpointer data)
-{
-    gboolean ret;
-
-    switch (GST_QUERY_TYPE(query)) {
-    case GST_QUERY_CAPS:
-        ret = _autoplugQueryCaps(bin, pad, element, query, data);
-        break;
-    case GST_QUERY_CONTEXT:
-        ret = _autoplugQueryContext(bin, pad, element, query, data);
-        break;
-    default:
-        ret = FALSE;
-        break;
-    }
-
-    return ret;
-}
-
 GstPadProbeReturn
 GstVideoReceiver::_teeProbe(GstPad* pad, GstPadProbeInfo* info, gpointer user_data)
 {

src/VideoReceiver/GstVideoReceiver.h

@@ -103,7 +103,7 @@ protected slots:
 
 protected:
     virtual GstElement* _makeSource(const QString& uri);
-    virtual GstElement* _makeDecoder(GstCaps* caps, GstElement* videoSink);
+    virtual GstElement* _makeDecoder(GstCaps* caps = nullptr, GstElement* videoSink = nullptr);
     virtual GstElement* _makeFileSink(const QString& videoFile, FILE_FORMAT format);
 
     virtual void _onNewSourcePad(GstPad* pad);
@@ -125,10 +125,6 @@ protected:
     static void _wrapWithGhostPad(GstElement* element, GstPad* pad, gpointer data);
     static void _linkPad(GstElement* element, GstPad* pad, gpointer data);
     static gboolean _padProbe(GstElement* element, GstPad* pad, gpointer user_data);
-    static gboolean _filterParserCaps(GstElement* bin, GstPad* pad, GstElement* element, GstQuery* query, gpointer data);
-    static gboolean _autoplugQueryCaps(GstElement* bin, GstPad* pad, GstElement* element, GstQuery* query, gpointer data);
-    static gboolean _autoplugQueryContext(GstElement* bin, GstPad* pad, GstElement* element, GstQuery* query, gpointer data);
-    static gboolean _autoplugQuery(GstElement* bin, GstPad* pad, GstElement* element, GstQuery* query, gpointer data);
     static GstPadProbeReturn _teeProbe(GstPad* pad, GstPadProbeInfo* info, gpointer user_data);
     static GstPadProbeReturn _videoSinkProbe(GstPad* pad, GstPadProbeInfo* info, gpointer user_data);
     static GstPadProbeReturn _eosProbe(GstPad* pad, GstPadProbeInfo* info, gpointer user_data);

src/ui/preferences/GeneralSettings.qml

@@ -313,6 +313,19 @@ Rectangle {
                                     visible:                _showSaveVideoSettings && _videoSettings.enableStorageLimit.value && maxSavedVideoStorageLabel.visible
                                 }
 
+                                QGCLabel {
+                                    id:         videoDecodeLabel
+                                    text:       qsTr("Video decode priority")
+                                    visible:    forceVideoDecoderComboBox.visible
+                                }
+                                FactComboBox {
+                                    id:                     forceVideoDecoderComboBox
+                                    Layout.preferredWidth:  _comboFieldWidth
+                                    fact:                   _videoSettings.forceVideoDecoder
+                                    visible:                fact.visible
+                                    indexModel:             false
+                                }
+
                                 Item { width: 1; height: 1}
                                 FactCheckBox {
                                     text:       qsTr("Disable When Disarmed")