From ef6ef06c07f69ddfcbe2a9258015a15aad17aca3 Mon Sep 17 00:00:00 2001 From: Gus Grubba Date: Wed, 26 Jun 2019 10:17:29 -0300 Subject: [PATCH 1/4] UDP RTP h.265 stream WIP --- ChangeLog.md | 1 + src/FlightDisplay/VideoManager.cc | 7 +++++-- src/Settings/VideoSettings.cc | 8 +++++--- src/Settings/VideoSettings.h | 9 ++++++--- src/Taisync/TaisyncManager.cc | 2 +- src/Vehicle/Vehicle.cc | 2 +- src/VideoStreaming/VideoReceiver.cc | 17 ++++++++++++----- src/ui/preferences/GeneralSettings.qml | 6 +++--- 8 files changed, 34 insertions(+), 18 deletions(-) diff --git a/ChangeLog.md b/ChangeLog.md index 3d58591..4123756 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -6,6 +6,7 @@ Note: This file only contains high level features or important fixes. ### 3.6.0 - Daily Build +* Adding support for UDP RTP h.265 video streams * For text to speech engine on Linux to English (all messages are in English) * Automated the ingestion of localization from Crowdin * Automated the generation of language resources into the application diff --git a/src/FlightDisplay/VideoManager.cc b/src/FlightDisplay/VideoManager.cc index 9773642..1bacebb 100644 --- a/src/FlightDisplay/VideoManager.cc +++ b/src/FlightDisplay/VideoManager.cc @@ -249,7 +249,8 @@ VideoManager::isGStreamer() #if defined(QGC_GST_STREAMING) QString videoSource = _videoSettings->videoSource()->rawValue().toString(); return - videoSource == VideoSettings::videoSourceUDP || + videoSource == VideoSettings::videoSourceUDPH264 || + videoSource == VideoSettings::videoSourceUDPH265 || videoSource == VideoSettings::videoSourceRTSP || videoSource == VideoSettings::videoSourceTCP || videoSource == VideoSettings::videoSourceMPEGTS || @@ -325,8 +326,10 @@ VideoManager::_updateSettings() } } QString source = _videoSettings->videoSource()->rawValue().toString(); - if (source == VideoSettings::videoSourceUDP) + if (source == VideoSettings::videoSourceUDPH264) _videoReceiver->setUri(QStringLiteral("udp://0.0.0.0:%1").arg(_videoSettings->udpPort()->rawValue().toInt())); + else if (source == VideoSettings::videoSourceUDPH265) + _videoReceiver->setUri(QStringLiteral("udp265://0.0.0.0:%1").arg(_videoSettings->udpPort()->rawValue().toInt())); else if (source == VideoSettings::videoSourceMPEGTS) _videoReceiver->setUri(QStringLiteral("mpegts://0.0.0.0:%1").arg(_videoSettings->udpPort()->rawValue().toInt())); else if (source == VideoSettings::videoSourceRTSP) diff --git a/src/Settings/VideoSettings.cc b/src/Settings/VideoSettings.cc index 78b0c6f..a0d0184 100644 --- a/src/Settings/VideoSettings.cc +++ b/src/Settings/VideoSettings.cc @@ -22,7 +22,8 @@ const char* VideoSettings::videoSourceNoVideo = "No Video Available"; const char* VideoSettings::videoDisabled = "Video Stream Disabled"; const char* VideoSettings::videoSourceRTSP = "RTSP Video Stream"; -const char* VideoSettings::videoSourceUDP = "UDP Video Stream"; +const char* VideoSettings::videoSourceUDPH264 = "UDP h.264 Video Stream"; +const char* VideoSettings::videoSourceUDPH265 = "UDP h.265 Video Stream"; const char* VideoSettings::videoSourceTCP = "TCP-MPEG2 Video Stream"; const char* VideoSettings::videoSourceMPEGTS = "MPEG-TS (h.264) Video Stream"; @@ -35,7 +36,8 @@ DECLARE_SETTINGGROUP(Video, "Video") #ifdef QGC_GST_STREAMING videoSourceList.append(videoSourceRTSP); #ifndef NO_UDP_VIDEO - videoSourceList.append(videoSourceUDP); + videoSourceList.append(videoSourceUDPH264); + videoSourceList.append(videoSourceUDPH265); #endif videoSourceList.append(videoSourceTCP); videoSourceList.append(videoSourceMPEGTS); @@ -141,7 +143,7 @@ bool VideoSettings::streamConfigured(void) return false; } //-- If UDP, check if port is set - if(vSource == videoSourceUDP) { + if(vSource == videoSourceUDPH264 || vSource == videoSourceUDPH265) { qCDebug(VideoManagerLog) << "Testing configuration for UDP Stream:" << udpPort()->rawValue().toInt(); return udpPort()->rawValue().toInt() != 0; } diff --git a/src/Settings/VideoSettings.h b/src/Settings/VideoSettings.h index a419b6a..431e5a3 100644 --- a/src/Settings/VideoSettings.h +++ b/src/Settings/VideoSettings.h @@ -37,19 +37,22 @@ public: Q_PROPERTY(bool streamConfigured READ streamConfigured NOTIFY streamConfiguredChanged) Q_PROPERTY(QString rtspVideoSource READ rtspVideoSource CONSTANT) - Q_PROPERTY(QString udpVideoSource READ udpVideoSource CONSTANT) + Q_PROPERTY(QString udp264VideoSource READ udp264VideoSource CONSTANT) + Q_PROPERTY(QString udp265VideoSource READ udp265VideoSource CONSTANT) Q_PROPERTY(QString tcpVideoSource READ tcpVideoSource CONSTANT) Q_PROPERTY(QString mpegtsVideoSource READ mpegtsVideoSource CONSTANT) bool streamConfigured (); QString rtspVideoSource () { return videoSourceRTSP; } - QString udpVideoSource () { return videoSourceUDP; } + QString udp264VideoSource () { return videoSourceUDPH264; } + QString udp265VideoSource () { return videoSourceUDPH265; } QString tcpVideoSource () { return videoSourceTCP; } QString mpegtsVideoSource () { return videoSourceMPEGTS; } static const char* videoSourceNoVideo; static const char* videoDisabled; - static const char* videoSourceUDP; + static const char* videoSourceUDPH264; + static const char* videoSourceUDPH265; static const char* videoSourceRTSP; static const char* videoSourceTCP; static const char* videoSourceMPEGTS; diff --git a/src/Taisync/TaisyncManager.cc b/src/Taisync/TaisyncManager.cc index cdc0ead..86033f9 100644 --- a/src/Taisync/TaisyncManager.cc +++ b/src/Taisync/TaisyncManager.cc @@ -337,7 +337,7 @@ TaisyncManager::_setVideoEnabled() pVSettings->udpPort()->setRawValue(5600); //-- TODO: this AR must come from somewhere pVSettings->aspectRatio()->setRawValue(1024.0 / 768.0); - pVSettings->videoSource()->setRawValue(QString(VideoSettings::videoSourceUDP)); + pVSettings->videoSource()->setRawValue(QString(VideoSettings::videoSourceUDPH264)); #if defined(__ios__) || defined(__android__) if(!_taiVideo) { //-- iOS and Android receive raw h.264 and need a different pipeline diff --git a/src/Vehicle/Vehicle.cc b/src/Vehicle/Vehicle.cc index 690c2a8..fb1e76b 100644 --- a/src/Vehicle/Vehicle.cc +++ b/src/Vehicle/Vehicle.cc @@ -511,7 +511,7 @@ void Vehicle::_commonInit(void) // Set video stream to udp if running ArduSub and Video is disabled if (sub() && _settingsManager->videoSettings()->videoSource()->rawValue() == VideoSettings::videoDisabled) { - _settingsManager->videoSettings()->videoSource()->setRawValue(VideoSettings::videoSourceUDP); + _settingsManager->videoSettings()->videoSource()->setRawValue(VideoSettings::videoSourceUDPH264); } //-- Airspace Management diff --git a/src/VideoStreaming/VideoReceiver.cc b/src/VideoStreaming/VideoReceiver.cc index 6fc9c31..07a836e 100644 --- a/src/VideoStreaming/VideoReceiver.cc +++ b/src/VideoStreaming/VideoReceiver.cc @@ -75,7 +75,7 @@ VideoReceiver::VideoReceiver(QObject* parent) , _swDecoderName("avdec_h264") { _videoSurface = new VideoSurface; - _videoSettings = qgcApp()->toolbox()->settingsManager()->videoSettings(); + _videoSettings = qgcApp()->toolbox()->settingsManager()->videoSettings(); #if defined(QGC_GST_STREAMING) setVideoDecoder(H264_SW); _setVideoSink(_videoSurface->videoSink()); @@ -183,7 +183,8 @@ VideoReceiver::start() #else bool isTaisyncUSB = false; #endif - bool isUdp = _uri.contains("udp://") && !isTaisyncUSB; + bool isUdp264 = _uri.contains("udp://") && !isTaisyncUSB; + bool isUdp265 = _uri.contains("udp265://") && !isTaisyncUSB; bool isTCP = _uri.contains("tcp://") && !isTaisyncUSB; bool isMPEGTS = _uri.contains("mpegts://") && !isTaisyncUSB; @@ -219,7 +220,7 @@ VideoReceiver::start() break; } - if(isUdp || isMPEGTS || isTaisyncUSB) { + if(isUdp264 || isUdp265 || isMPEGTS || isTaisyncUSB) { dataSource = gst_element_factory_make("udpsrc", "udp-source"); } else if(isTCP) { dataSource = gst_element_factory_make("tcpclientsrc", "tcpclient-source"); @@ -232,12 +233,18 @@ VideoReceiver::start() break; } - if(isUdp) { + if(isUdp264) { if ((caps = gst_caps_from_string("application/x-rtp, media=(string)video, clock-rate=(int)90000, encoding-name=(string)H264")) == nullptr) { qCritical() << "VideoReceiver::start() failed. Error with gst_caps_from_string()"; break; } g_object_set(static_cast(dataSource), "uri", qPrintable(_uri), "caps", caps, nullptr); + } else if(isUdp265) { + + + //-- TODO + + #if defined(QGC_GST_TAISYNC_ENABLED) && (defined(__android__) || defined(__ios__)) } else if(isTaisyncUSB) { QString uri = QString("0.0.0.0:%1").arg(TAISYNC_VIDEO_UDP_PORT); @@ -305,7 +312,7 @@ VideoReceiver::start() } pipelineUp = true; - if(isUdp) { + if(isUdp264 || isUdp265) { // Link the pipeline in front of the tee if(!gst_element_link_many(dataSource, demux, parser, _tee, queue, decoder, queue1, _videoSink, nullptr)) { qCritical() << "Unable to link UDP elements."; diff --git a/src/ui/preferences/GeneralSettings.qml b/src/ui/preferences/GeneralSettings.qml index 809807b..fb077f2 100644 --- a/src/ui/preferences/GeneralSettings.qml +++ b/src/ui/preferences/GeneralSettings.qml @@ -46,7 +46,7 @@ Rectangle { property string _videoSource: QGroundControl.settingsManager.videoSettings.videoSource.value property bool _isGst: QGroundControl.videoManager.isGStreamer - property bool _isUDP: _isGst && _videoSource === QGroundControl.settingsManager.videoSettings.udpVideoSource + property bool _isUDP264: _isGst && _videoSource === QGroundControl.settingsManager.videoSettings.udp264VideoSource property bool _isRTSP: _isGst && _videoSource === QGroundControl.settingsManager.videoSettings.rtspVideoSource property bool _isTCP: _isGst && _videoSource === QGroundControl.settingsManager.videoSettings.tcpVideoSource property bool _isMPEGTS: _isGst && _videoSource === QGroundControl.settingsManager.videoSettings.mpegtsVideoSource @@ -801,12 +801,12 @@ Rectangle { QGCLabel { text: qsTr("UDP Port") - visible: (_isUDP || _isMPEGTS) && QGroundControl.settingsManager.videoSettings.udpPort.visible + visible: (_isUDP264 || _isMPEGTS) && QGroundControl.settingsManager.videoSettings.udpPort.visible } FactTextField { Layout.preferredWidth: _comboFieldWidth fact: QGroundControl.settingsManager.videoSettings.udpPort - visible: (_isUDP || _isMPEGTS) && QGroundControl.settingsManager.videoSettings.udpPort.visible + visible: (_isUDP264 || _isMPEGTS) && QGroundControl.settingsManager.videoSettings.udpPort.visible } QGCLabel { From e997db34f2154cb8913a51b58edfad020029958c Mon Sep 17 00:00:00 2001 From: Gus Grubba Date: Wed, 26 Jun 2019 10:33:22 -0300 Subject: [PATCH 2/4] Handle UDP h.265 port in general settings (same as the one used for h.264) --- src/ui/preferences/GeneralSettings.qml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ui/preferences/GeneralSettings.qml b/src/ui/preferences/GeneralSettings.qml index fb077f2..b1e697c 100644 --- a/src/ui/preferences/GeneralSettings.qml +++ b/src/ui/preferences/GeneralSettings.qml @@ -801,12 +801,12 @@ Rectangle { QGCLabel { text: qsTr("UDP Port") - visible: (_isUDP264 || _isMPEGTS) && QGroundControl.settingsManager.videoSettings.udpPort.visible + visible: (_isUDP264 || _isUDP265 || _isMPEGTS) && QGroundControl.settingsManager.videoSettings.udpPort.visible } FactTextField { Layout.preferredWidth: _comboFieldWidth fact: QGroundControl.settingsManager.videoSettings.udpPort - visible: (_isUDP264 || _isMPEGTS) && QGroundControl.settingsManager.videoSettings.udpPort.visible + visible: (_isUDP264 || _isUDP265 || _isMPEGTS) && QGroundControl.settingsManager.videoSettings.udpPort.visible } QGCLabel { From 99070b9fd0251521927fb3b3d5ef18e60ea566eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Fran=C4=8De=C5=A1kin?= Date: Wed, 26 Jun 2019 21:33:50 +0200 Subject: [PATCH 3/4] Added udp265 gstreamer pipeline --- src/VideoStreaming/VideoReceiver.cc | 15 ++++++++++----- src/ui/preferences/GeneralSettings.qml | 1 + 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/VideoStreaming/VideoReceiver.cc b/src/VideoStreaming/VideoReceiver.cc index 07a836e..cd62f8d 100644 --- a/src/VideoStreaming/VideoReceiver.cc +++ b/src/VideoStreaming/VideoReceiver.cc @@ -200,6 +200,11 @@ VideoReceiver::start() qCDebug(VideoReceiverLog) << "Already running!"; return; } + if (isUdp264) { + setVideoDecoder(H264_HW); + } else if (isUdp265) { + setVideoDecoder(H265_HW); + } _starting = true; @@ -240,11 +245,11 @@ VideoReceiver::start() } g_object_set(static_cast(dataSource), "uri", qPrintable(_uri), "caps", caps, nullptr); } else if(isUdp265) { - - - //-- TODO - - + if ((caps = gst_caps_from_string("application/x-rtp, media=(string)video, clock-rate=(int)90000, encoding-name=(string)H265")) == nullptr) { + qCritical() << "VideoReceiver::start() failed. Error with gst_caps_from_string()"; + break; + } + g_object_set(static_cast(dataSource), "uri", qPrintable(_uri.replace("udp265", "udp")), "caps", caps, nullptr); #if defined(QGC_GST_TAISYNC_ENABLED) && (defined(__android__) || defined(__ios__)) } else if(isTaisyncUSB) { QString uri = QString("0.0.0.0:%1").arg(TAISYNC_VIDEO_UDP_PORT); diff --git a/src/ui/preferences/GeneralSettings.qml b/src/ui/preferences/GeneralSettings.qml index b1e697c..add7963 100644 --- a/src/ui/preferences/GeneralSettings.qml +++ b/src/ui/preferences/GeneralSettings.qml @@ -47,6 +47,7 @@ Rectangle { property string _videoSource: QGroundControl.settingsManager.videoSettings.videoSource.value property bool _isGst: QGroundControl.videoManager.isGStreamer property bool _isUDP264: _isGst && _videoSource === QGroundControl.settingsManager.videoSettings.udp264VideoSource + property bool _isUDP265: _isGst && _videoSource === QGroundControl.settingsManager.videoSettings.udp265VideoSource property bool _isRTSP: _isGst && _videoSource === QGroundControl.settingsManager.videoSettings.rtspVideoSource property bool _isTCP: _isGst && _videoSource === QGroundControl.settingsManager.videoSettings.tcpVideoSource property bool _isMPEGTS: _isGst && _videoSource === QGroundControl.settingsManager.videoSettings.mpegtsVideoSource From e1536993cbd961c482678357dc0e87d7161e854b Mon Sep 17 00:00:00 2001 From: Gus Grubba Date: Wed, 26 Jun 2019 16:45:02 -0300 Subject: [PATCH 4/4] Documentation --- src/VideoStreaming/README.md | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/VideoStreaming/README.md b/src/VideoStreaming/README.md index feb2916..4e851fe 100644 --- a/src/VideoStreaming/README.md +++ b/src/VideoStreaming/README.md @@ -7,15 +7,22 @@ To build video streaming support, you will need to install the GStreamer develop If you do have the proper GStreamer development libraries installed where QGC looks for it, the QGC build system will automatically use it and build video streaming support. If you would like to disable video streaming support, you can add **DISABLE_VIDEOSTREAMING** to the **DEFINES** build variable. -### Pipeline +### UDP Pipeline -For the time being, the pipeline is somewhat hardcoded, using h.264. It's best to use a camera capable of hardware encoding h.264, such as the Logitech C920. On the sender end, for RTP (UDP Streaming) you would run something like this: +For the time being, the RTP UDP pipeline is somewhat hardcoded, using h.264 or h.265. It's best to use a camera capable of hardware encoding either h.264 (such as the Logitech C920) or h.265. On the sender end, for RTP (UDP Streaming) you would run something like this: +h.264 ``` gst-launch-1.0 uvch264src initial-bitrate=1000000 average-bitrate=1000000 iframe-period=1000 device=/dev/video0 name=src auto-start=true src.vidsrc ! video/x-h264,width=1920,height=1080,framerate=24/1 ! h264parse ! rtph264pay ! udpsink host=xxx.xxx.xxx.xxx port=5600 ``` -Where xxx.xxx.xxx.xxx is the IP address where QGC is running. You may tweak the bitrate, the resolution and the FPS based on your needs and/or available bandwidth. +h.265 +``` +ffmpeg -f v4l2 -i /dev/video1 -pix_fmt yuv420p -c:v libx265 -preset ultrafast -x265-params crf=23 -strict experimental -f rtp udp://xxx.xxx.xxx.xxx:5600 +``` + +Where xxx.xxx.xxx.xxx is the IP address where QGC is running. + To test using a test source on localhost, you can run this command: ``` @@ -31,6 +38,10 @@ On the receiving end, if you want to test it from the command line, you can use gst-launch-1.0 udpsrc port=5600 caps='application/x-rtp, media=(string)video, clock-rate=(int)90000, encoding-name=(string)H264' ! rtph264depay ! avdec_h264 ! autovideosink fps-update-interval=1000 sync=false ``` +### Additional Protocols + +QGC also supports RTSP, TCP-MPEG2 and MPEG-TS (h.264) pipelines. + ### Linux Use apt-get to install GStreamer 1.0