Significantly improved logging precision, added hole filling for missing data points (user-preferences for this still pending)

src/LogCompressor.cc

@@ -45,7 +45,8 @@ LogCompressor::LogCompressor(QString logFileName, QString outFileName, int uasid
     running(true),
     currentDataLine(0),
     dataLines(1),
-    uasid(uasid)
+    uasid(uasid),
+    holeFillingEnabled(true)
 {
 }
 
@@ -59,7 +60,7 @@ void LogCompressor::run()
     QList<quint64> times;// = new QList<quint64>();
     QList<quint64> finalTimes;
 
-    qDebug() << "LOG COMPRESSOR: Starting" << fileName;
+    //qDebug() << "LOG COMPRESSOR: Starting" << fileName;
 
     if (!file.exists() || !file.open(QIODevice::ReadOnly | QIODevice::Text)) {
         //qDebug() << "LOG COMPRESSOR: INPUT FILE DOES NOT EXIST";
@@ -89,7 +90,10 @@ void LogCompressor::run()
         // Accumulate map of keys
         // Data field name is at position 2
         QString key = line.split(separator).at(2);
-        if (!keys->contains(key)) keys->append(key);
+        if (!keys->contains(key))
+        {
+            keys->append(key);
+        }
         keyCounter++;
     }
     keys->sort();
@@ -98,15 +102,11 @@ void LogCompressor::run()
     QString spacer = "";
     for (int i = 0; i < keys->length(); i++) {
         header += keys->at(i) + separator;
-        spacer += " " + separator;
+        spacer += separator;
     }
 
     emit logProcessingStatusChanged(tr("Log compressor: Dataset contains dimension: ") + header);
 
-    //qDebug() << header;
-
-    //qDebug() << "NOW READING TIMES";
-
     // Find all times
     //in.reset();
     file.reset();
@@ -158,16 +158,18 @@ void LogCompressor::run()
         // Get time
         quint64 time = static_cast<QString>(parts.first()).toLongLong(&ok);
         QString field = parts.at(2);
+        int fieldIndex = keys->indexOf(field);
         QString value = parts.at(3);
-        // Enforce NaN if no value is present
+//        // Enforce NaN if no value is present
-        if (value.length() == 0 || value == "" || value == " " || value == "\t" || value == "\n") {
+//        if (value.length() == 0 || value == "" || value == " " || value == "\t" || value == "\n") {
-            value = "NaN";
+//            // Hole filling disabled, fill with NaN
-        }
+//            value = "NaN";
+//        }
         // Get matching output line
 
         // Constraining the search area might result in not finding a key,
         // but it significantly reduces the time needed for the search
-        // setting a window of 1000 entries means that a 1 Hz data point
+        // setting a window of 100 entries means that a 1 Hz data point
         // can still be located
         quint64 offsetLimit = 100;
         quint64 offset;
@@ -205,12 +207,54 @@ void LogCompressor::run()
             QString outLine = outLines->at(index);
             QStringList outParts = outLine.split(separator);
             // Replace measurement placeholder with current value
-            outParts.replace(keys->indexOf(field)+1, value);
+            outParts.replace(fieldIndex+1, value);
             outLine = outParts.join(separator);
             outLines->replace(index, outLine);
         }
     }
 
+    ///////////////////////////
+    // HOLE FILLING
+
+    // If hole filling is enabled, run again through the whole file and replace holes
+    if (holeFillingEnabled)
+    {
+        // Build up the fill values - initialize to NaN
+        QStringList fillValues;
+        int fillCount = keys->count();
+        for (int i = 0; i< fillCount; ++i)
+        {
+            fillValues.append("NaN");
+        }
+
+        // Run through all lines and replace with fill values
+        for (int index = 0; index < outLines->count(); ++index)
+        {
+            QString line = outLines->at(index);
+            QStringList fields = line.split(separator, QString::SkipEmptyParts);
+            // The fields line contains the timestamp
+            // index of the data fields therefore runs from 1 to n-1
+            int fieldCount = fields.count();
+            for (int i = 1; i < fillCount+1; ++i)
+            {
+                if (fieldCount < fillCount) fields.append("");
+
+                // Check if this is NaN
+                if (fields[i] == 0 || fields[i] == "" || fields[i] == "\t" || fields[i] == " " || fields[i] == "\n")
+                {
+                    // Value was empty, replace it
+                    fields.replace(i, fillValues[i-1]);
+                }
+                else
+                {
+                    // Value was not NaN, use it as
+                    // new fill value
+                    fillValues.replace(i-1, fields[i]);
+                }
+            }
+            outLines->replace(index, fields.join(separator));
+        }
+    }
 
     // Add header, write out file
     file.close();
@@ -222,9 +266,8 @@ void LogCompressor::run()
     }
     if (!outfile.open(QIODevice::WriteOnly | QIODevice::Text))
         return;
-    outfile.write(QString(QString("unix_timestamp") + separator + header.replace(" ", "_") + QString("\n")).toLatin1());
+    outfile.write(QString(QString("timestamp_ms") + separator + header.replace(" ", "_") + QString("\n")).toLatin1());
     emit logProcessingStatusChanged(tr("Log Compressor: Writing output to file %1").arg(QFileInfo(outFileName).absoluteFilePath()));
-    //QString fileHeader = QString("unix_timestamp") + header.replace(" ", "_") + QString("\n");
 
     // File output
     for (int i = 0; i < outLines->length(); i++) {

src/LogCompressor.h

@@ -22,6 +22,7 @@ protected:
     int currentDataLine;
     int dataLines;
     int uasid;
+    bool holeFillingEnabled;       ///< Enables the filling of holes in the dataset with the previous value (or NaN if none exists)
 
 signals:
     /** @brief This signal is emitted once a logfile has been finished writing

src/ui/linechart/LinechartWidget.cc

@@ -310,7 +310,7 @@ void LinechartWidget::appendData(int uasId, QString curve, double value, quint64
             qint64 time = usec - logStartTime;
             if (time < 0) time = 0;
 
-            logFile->write(QString(QString::number(time) + "\t" + QString::number(uasId) + "\t" + curve + "\t" + QString::number(value,'g',10) + "\n").toLatin1());
+            logFile->write(QString(QString::number(time) + "\t" + QString::number(uasId) + "\t" + curve + "\t" + QString::number(value,'g',18) + "\n").toLatin1());
             logFile->flush();
         }
     }
@@ -338,7 +338,7 @@ void LinechartWidget::appendData(int uasId, const QString& curve, const QString&
             qint64 time = usec - logStartTime;
             if (time < 0) time = 0;
 
-            logFile->write(QString(QString::number(time) + "\t" + QString::number(uasId) + "\t" + curve + "\t" + QString::number(value,'g',10) + "\n").toLatin1());
+            logFile->write(QString(QString::number(time) + "\t" + QString::number(uasId) + "\t" + curve + "\t" + QString::number(value,'g',18) + "\n").toLatin1());
             logFile->flush();
         }
     }

src/ui/map/QGCMapToolBar.cc

@@ -18,6 +18,7 @@ void QGCMapToolBar::setMap(QGCMapWidget* map)
     {
         connect(ui->goToButton, SIGNAL(clicked()), map, SLOT(showGoToDialog()));
         connect(ui->goHomeButton, SIGNAL(clicked()), map, SLOT(goHome()));
+        connect(ui->lastPosButton, SIGNAL(clicked()), map, SLOT(loadSettings()));
         connect(map, SIGNAL(OnTileLoadStart()), this, SLOT(tileLoadStart()));
         connect(map, SIGNAL(OnTileLoadComplete()), this, SLOT(tileLoadEnd()));
         connect(map, SIGNAL(OnTilesStillToLoad(int)), this, SLOT(tileLoadProgress(int)));

src/ui/map/QGCMapToolBar.ui

@@ -42,6 +42,13 @@
     </widget>
    </item>
    <item>
+    <widget class="QPushButton" name="lastPosButton">
+     <property name="text">
+      <string>Last Pos</string>
+     </property>
+    </widget>
+   </item>
+   <item>
     <widget class="QPushButton" name="ripMapButton">
      <property name="text">
       <string>Cache</string>

src/ui/map/QGCMapWidget.cc

@@ -112,7 +112,7 @@ void QGCMapWidget::showEvent(QShowEvent* event)
     connect(&updateTimer, SIGNAL(timeout()), this, SLOT(updateGlobalPosition()));
     updateTimer.start(maxUpdateInterval*1000);
     updateGlobalPosition();
-    QTimer::singleShot(1, this, SLOT(loadSettings()));
+    //QTimer::singleShot(800, this, SLOT(loadSettings()));
 }
 
 void QGCMapWidget::hideEvent(QHideEvent* event)