User hardware video decoding on Android

android/src/org/mavlink/qgroundcontrol/QGCActivity.java

@@ -184,6 +184,8 @@ public class QGCActivity extends QtActivity
     public static native void qgcLogDebug(String message);
     public static native void qgcLogWarning(String message);
 
+    private static native void nativeInit();
+
     // QGCActivity singleton
     public QGCActivity()
     {
@@ -745,5 +747,10 @@ public class QGCActivity extends QtActivity
             }
         }).start();
     }
+
+    public void jniOnLoad()
+    {
+        nativeInit();
+    }
 }
 

src/VideoStreaming/VideoReceiver.cc

@@ -71,6 +71,8 @@ VideoReceiver::VideoReceiver(QObject* parent)
     , _videoRunning(false)
     , _showFullScreen(false)
     , _videoSettings(nullptr)
+    , _hwDecoderName(nullptr)
+    , _swDecoderName("avdec_h264")
 {
     _videoSurface = new VideoSurface;
     _videoSettings = qgcApp()->toolbox()->settingsManager()->videoSettings();    
@@ -159,6 +161,10 @@ VideoReceiver::_restart_timeout()
 void
 VideoReceiver::start()
 {
+    if (_uri.isEmpty()) {
+        return;
+    }
+    qCDebug(VideoReceiverLog) << "start():" << _uri;
     if(qgcApp()->runningUnitTests()) {
         return;
     }
@@ -170,7 +176,6 @@ VideoReceiver::start()
 
 #if defined(QGC_GST_STREAMING)
     _stop = false;
-    qCDebug(VideoReceiverLog) << "start():" << _uri;
 
 #if defined(QGC_GST_TAISYNC_ENABLED) && (defined(__android__) || defined(__ios__))
     //-- Taisync on iOS or Android sends a raw h.264 stream
@@ -280,9 +285,12 @@ VideoReceiver::start()
             break;
         }
 
-        if ((decoder = gst_element_factory_make(_decoderName, "decoder")) == nullptr) {
-            qCritical() << "VideoReceiver::start() failed. Error with gst_element_factory_make('" << _decoderName << "')";
-            break;
+        if (!_hwDecoderName || (decoder = gst_element_factory_make(_hwDecoderName, "decoder")) == nullptr) {
+            qWarning() << "VideoReceiver::start() hardware decoding not available " << ((_hwDecoderName) ? _hwDecoderName : "");
+            if ((decoder = gst_element_factory_make(_swDecoderName, "decoder")) == nullptr) {
+                qCritical() << "VideoReceiver::start() failed. Error with gst_element_factory_make('" << _swDecoderName << "')";
+                break;
+            }
         }
 
         if ((queue1 = gst_element_factory_make("queue", nullptr)) == nullptr) {
@@ -470,6 +478,8 @@ VideoReceiver::_shutdownPipeline() {
 void
 VideoReceiver::_handleError() {
     qCDebug(VideoReceiverLog) << "Gstreamer error!";
+    // If there was an error we switch to software decoding only
+    _tryWithHardwareDecoding = false;
     stop();
     _restart_timer.start(_restart_time_ms);
 }
@@ -580,17 +590,34 @@ VideoReceiver::_cleanupOldVideos()
 void
 VideoReceiver::setVideoDecoder(VideoEncoding encoding)
 {
+    /*
+    #if defined(Q_OS_MAC)
+        _hwDecoderName = "vtdec";
+    #else
+        _hwDecoderName = "vaapidecode";
+    #endif
+    */
+
     if (encoding == H265_HW || encoding == H265_SW) {
         _depayName = "rtph265depay";
         _parserName = "h265parse";
-        _decoderName = "avdec_h265";
+#if defined(__android__)
+        _hwDecoderName = "amcviddec-omxgooglehevcdecoder";
+#endif
+        _swDecoderName = "avdec_h265";
     } else {
         _depayName = "rtph264depay";
         _parserName = "h264parse";
-        _decoderName = "avdec_h264";
+#if defined(__android__)
+        _hwDecoderName = "amcviddec-omxgoogleh264decoder";
+#endif
+        _swDecoderName = "avdec_h264";
     }
-}
 
+    if (!_tryWithHardwareDecoding) {
+        _hwDecoderName = nullptr;
+    }
+}
 //-----------------------------------------------------------------------------
 // When we finish our pipeline will look like this:
 //

src/VideoStreaming/VideoReceiver.h

@@ -152,6 +152,8 @@ protected:
     VideoSettings*  _videoSettings;
     const char*     _depayName;
     const char*     _parserName;
-    const char*     _decoderName;
+    bool            _tryWithHardwareDecoding = true;
+    const char*     _hwDecoderName;
+    const char*     _swDecoderName;
 };
 

src/VideoStreaming/VideoStreaming.cc

@@ -48,6 +48,9 @@
     GST_PLUGIN_STATIC_DECLARE(isomp4);
     GST_PLUGIN_STATIC_DECLARE(matroska);
 #endif
+#if defined(__android__)
+    GST_PLUGIN_STATIC_DECLARE(androidmedia);
+#endif
     G_END_DECLS
 #endif
 
@@ -159,6 +162,7 @@ void initializeVideoStreaming(int &argc, char* argv[], char* logpath, char* debu
         GST_PLUGIN_STATIC_REGISTER(rtpmanager);
         GST_PLUGIN_STATIC_REGISTER(isomp4);
         GST_PLUGIN_STATIC_REGISTER(matroska);
+        GST_PLUGIN_STATIC_REGISTER(androidmedia);
     #endif
 #else
     Q_UNUSED(argc);

src/VideoStreaming/VideoStreaming.pri

@@ -97,11 +97,13 @@ LinuxBuild {
             -lgstrtpmanager \
             -lgstisomp4 \
             -lgstmatroska \
+            -lgstandroidmedia
 
         # Rest of GStreamer dependencies
         LIBS += -L$$GST_ROOT/lib \
             -lgstfft-1.0 -lm  \
             -lgstnet-1.0 -lgio-2.0 \
+            -lgstphotography-1.0 -lgstgl-1.0 -lEGL \
             -lgstaudio-1.0 -lgstcodecparsers-1.0 -lgstbase-1.0 \
             -lgstreamer-1.0 -lgstrtp-1.0 -lgstpbutils-1.0 -lgstrtsp-1.0 -lgsttag-1.0 \
             -lgstvideo-1.0 -lavformat -lavcodec -lavutil -lx264 -lavfilter -lswresample \

src/main.cc

@@ -1,6 +1,6 @@
 /****************************************************************************
  *
- *   (c) 2009-2016 QGROUNDCONTROL PROJECT <http://www.qgroundcontrol.org>
+ *   (c) 2009-2019 QGROUNDCONTROL PROJECT <http://www.qgroundcontrol.org>
  *
  * QGroundControl is licensed according to the terms in the file
  * COPYING.md in the root of the source code directory.
@@ -85,6 +85,84 @@ int WindowsCrtReportHook(int reportType, char* message, int* returnValue)
 #include "qserialport.h"
 #endif
 
+static jobject _class_loader = nullptr;
+static jobject _context = nullptr;
+
+//-----------------------------------------------------------------------------
+extern "C" {
+    void gst_amc_jni_set_java_vm(JavaVM *java_vm);
+
+    jobject gst_android_get_application_class_loader(void)
+    {
+        return _class_loader;
+    }
+}
+
+//-----------------------------------------------------------------------------
+static void
+gst_android_init(JNIEnv* env, jobject context)
+{
+    jobject class_loader = nullptr;
+
+    jclass context_cls = env->GetObjectClass(context);
+    if (!context_cls) {
+        return;
+    }
+
+    jmethodID get_class_loader_id = env->GetMethodID(context_cls, "getClassLoader", "()Ljava/lang/ClassLoader;");
+    if (env->ExceptionCheck()) {
+        env->ExceptionDescribe();
+        env->ExceptionClear();
+        return;
+    }
+
+    class_loader = env->CallObjectMethod(context, get_class_loader_id);
+    if (env->ExceptionCheck()) {
+        env->ExceptionDescribe();
+        env->ExceptionClear();
+        return;
+    }
+
+    _context = env->NewGlobalRef(context);
+    _class_loader = env->NewGlobalRef (class_loader);
+}
+
+//-----------------------------------------------------------------------------
+static const char kJniClassName[] {"org/mavlink/qgroundcontrol/QGCActivity"};
+
+void setNativeMethods(void)
+{
+    JNINativeMethod javaMethods[] {
+        {"nativeInit", "(Landroid/content/Context;)V", reinterpret_cast<void *>(gst_android_init)}
+    };
+
+    QAndroidJniEnvironment jniEnv;
+    if (jniEnv->ExceptionCheck()) {
+        jniEnv->ExceptionDescribe();
+        jniEnv->ExceptionClear();
+    }
+
+    jclass objectClass = jniEnv->FindClass(kJniClassName);
+    if(!objectClass) {
+        qWarning() << "Couldn't find class:" << kJniClassName;
+        return;
+    }
+
+    jint val = jniEnv->RegisterNatives(objectClass, javaMethods, sizeof(javaMethods) / sizeof(javaMethods[0]));
+
+    if (val < 0) {
+        qWarning() << "Error registering methods: " << val;
+    } else {
+        qDebug() << "Main Native Functions Registered";
+    }
+
+    if (jniEnv->ExceptionCheck()) {
+        jniEnv->ExceptionDescribe();
+        jniEnv->ExceptionClear();
+    }
+}
+
+//-----------------------------------------------------------------------------
 jint JNI_OnLoad(JavaVM* vm, void* reserved)
 {
     Q_UNUSED(reserved);
@@ -93,6 +171,18 @@ jint JNI_OnLoad(JavaVM* vm, void* reserved)
     if (vm->GetEnv(reinterpret_cast<void**>(&env), JNI_VERSION_1_6) != JNI_OK) {
         return -1;
     }
+    setNativeMethods();
+
+    QAndroidJniObject resultL = QAndroidJniObject::callStaticObjectMethod(
+        kJniClassName,
+        "jniOnLoad",
+        "();");
+
+#if defined(QGC_GST_STREAMING)
+    // Tell the androidmedia plugin about the Java VM
+    gst_amc_jni_set_java_vm(vm);
+#endif
+
  #if !defined(NO_SERIAL_LINK)
     QSerialPort::setNativeMethods();
  #endif
@@ -102,6 +192,7 @@ jint JNI_OnLoad(JavaVM* vm, void* reserved)
 }
 #endif
 
+//-----------------------------------------------------------------------------
 #ifdef __android__
 #include <QtAndroid>
 bool checkAndroidWritePermission() {
@@ -117,6 +208,7 @@ bool checkAndroidWritePermission() {
 }
 #endif
 
+//-----------------------------------------------------------------------------
 /**
  * @brief Starts the application
  *