zhaohuwei
/
QG4
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

Fixed logging of large datasets

QGC4.4
lm 15 years ago
parent
835279e9a6
commit
0e044f6c63
5 changed files with 108 additions and 60 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 127
      src/LogCompressor.cc
  2. 1
      src/LogCompressor.h
  3. 29
      src/ui/MainWindow.cc
  4. 6
      src/ui/MainWindow.h
  5. 5
      src/ui/linechart/LinechartWidget.cc

127
src/LogCompressor.cc
Unescape View File

@ -1,5 +1,4 @@
/*===================================================================== /*===================================================================
QGroundControl Open Source Ground Control Station QGroundControl Open Source Ground Control Station
(c) 2009, 2010 QGROUNDCONTROL PROJECT <http://www.qgroundcontrol.org> (c) 2009, 2010 QGROUNDCONTROL PROJECT <http://www.qgroundcontrol.org>
@ -10,15 +9,15 @@ This file is part of the QGROUNDCONTROL project
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or the Free Software Foundation, either version 3 of the License, or
(at your option) any later version. (at your option) any later version.
QGROUNDCONTROL is distributed in the hope that it will be useful, QGROUNDCONTROL is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details. GNU General Public License for more details.
You should have received a copy of the GNU General Public License You should have received a copy of the GNU General Public License
along with QGROUNDCONTROL. If not, see <http://www.gnu.org/licenses/>. along with QGROUNDCONTROL. If not, see <http://www.gnu.org/licenses/>.
======================================================================*/ ======================================================================*/
/** /**
@ -32,6 +31,7 @@ This file is part of the QGROUNDCONTROL project
#include <QTextStream> #include <QTextStream>
#include <QStringList> #include <QStringList>
#include <QFileInfo> #include <QFileInfo>
#include <QList>
#include "LogCompressor.h" #include "LogCompressor.h"
#include <QDebug> #include <QDebug>
@ -56,12 +56,13 @@ void LogCompressor::run()
QFile file(fileName); QFile file(fileName);
QFile outfile(outFileName); QFile outfile(outFileName);
QStringList* keys = new QStringList(); QStringList* keys = new QStringList();
QStringList* times = new QStringList(); QList<quint64> times;// = new QList<quint64>();
QList<quint64> finalTimes;
if (!file.exists()) return; if (!file.exists()) return;
if (!file.open(QIODevice::ReadOnly | QIODevice::Text)) if (!file.open(QIODevice::ReadOnly | QIODevice::Text))
return; return;
if (outFileName != "") if (outFileName != "")
{ {
// Check if file is writeable // Check if file is writeable
@ -70,18 +71,27 @@ void LogCompressor::run()
return; return;
} }
} }
// Find all keys // Find all keys
QTextStream in(&file); QTextStream in(&file);
while (!in.atEnd()) {
// Search only a certain region, assuming that not more
// than N dimensions at H Hertz can be send
const unsigned int keySearchLimit = 15000;
// e.g. 500 Hz * 30 values or
// e.g. 100 Hz * 150 values
unsigned int keyCounter = 0;
while (!in.atEnd() && keyCounter < keySearchLimit) {
QString line = in.readLine(); QString line = in.readLine();
// Accumulate map of keys // Accumulate map of keys
// Data field name is at position 2 // Data field name is at position 2
QString key = line.split(separator).at(2); QString key = line.split(separator).at(2);
if (!keys->contains(key)) keys->append(key); if (!keys->contains(key)) keys->append(key);
keyCounter++;
} }
keys->sort(); keys->sort();
QString header = ""; QString header = "";
QString spacer = ""; QString spacer = "";
for (int i = 0; i < keys->length(); i++) for (int i = 0; i < keys->length(); i++)
@ -90,42 +100,58 @@ void LogCompressor::run()
spacer += " " + separator; spacer += " " + separator;
} }
emit logProcessingStatusChanged(tr("Log compressor: Dataset contains dimension: ") + header);
//qDebug() << header; //qDebug() << header;
//qDebug() << "NOW READING TIMES"; //qDebug() << "NOW READING TIMES";
// Find all times // Find all times
//in.reset(); //in.reset();
file.reset(); file.reset();
in.reset(); in.reset();
in.resetStatus(); in.resetStatus();
while (!in.atEnd()) { bool ok;
while (!in.atEnd())
{
QString line = in.readLine(); QString line = in.readLine();
// Accumulate map of keys // Accumulate map of keys
// Data field name is at position 2 // Data field name is at position 2b
QString time = line.split(separator).at(0); quint64 time = static_cast<QString>(line.split(separator).at(0)).toLongLong(&ok);
if (!times->contains(time)) if (ok)
{ {
times->append(time); times.append(time);
} }
} }
dataLines = times->length(); qSort(times);
times->sort(); qint64 lastTime = -1;
// Create lines // Create lines
QStringList* outLines = new QStringList(); QStringList* outLines = new QStringList();
for (int i = 0; i < times->length(); i++) for (int i = 0; i < times.length(); i++)
{ {
outLines->append(times->at(i) + separator + spacer); if (times.at(i) != lastTime)
{
outLines->append(QString("%1").arg(times.at(i)) + separator + spacer);
lastTime = static_cast<qint64>(times.at(i));
finalTimes.append(times.at(i));
//qDebug() << "ADDED:" << outLines->last();
}
} }
dataLines = finalTimes.length();
emit logProcessingStatusChanged(tr("Log compressor: Now processing %1 log lines").arg(finalTimes.length()));
// Fill in the values for all keys // Fill in the values for all keys
file.reset(); file.reset();
QTextStream data(&file); QTextStream data(&file);
int linecounter = 0; int linecounter = 0;
quint64 lastTimeIndex = 0; quint64 lastTimeIndex = 0;
bool failed = false;
while (!data.atEnd()) while (!data.atEnd())
{ {
linecounter++; linecounter++;
@ -133,7 +159,7 @@ void LogCompressor::run()
QString line = data.readLine(); QString line = data.readLine();
QStringList parts = line.split(separator); QStringList parts = line.split(separator);
// Get time // Get time
QString time = parts.first(); quint64 time = static_cast<QString>(parts.first()).toLongLong(&ok);
QString field = parts.at(2); QString field = parts.at(2);
QString value = parts.at(3); QString value = parts.at(3);
// Enforce NaN if no value is present // Enforce NaN if no value is present
@ -142,15 +168,19 @@ void LogCompressor::run()
value = "NaN"; value = "NaN";
} }
// Get matching output line // Get matching output line
// Constraining the search area might result in not finding a key, // Constraining the search area might result in not finding a key,
// but it significantly reduces the time needed for the search // but it significantly reduces the time needed for the search
// setting a window of 1000 entries means that a 1 Hz data point // setting a window of 1000 entries means that a 1 Hz data point
// can still be located // can still be located
int offsetLimit = 200; quint64 offsetLimit = 100;
quint64 offset; quint64 offset;
quint64 index = -1; qint64 index = -1;
while (index == -1) failed = false;
// Search the index until it is valid (!= -1)
// or the start of the list has been reached (failed)
while (index == -1 && !failed)
{ {
if (lastTimeIndex > offsetLimit) if (lastTimeIndex > offsetLimit)
{ {
@ -160,14 +190,27 @@ void LogCompressor::run()
{ {
offset = 0; offset = 0;
} }
quint64 index = times->indexOf(time, offset);
index = finalTimes.indexOf(time, offset);
if (index == -1) if (index == -1)
{ {
qDebug() << "INDEX NOT FOUND DURING LOGFILE PROCESSING, RESTARTING SEARCH"; if (offset == 0)
// FIXME Reset and start without offset heuristic again {
offsetLimit+=1000; emit logProcessingStatusChanged(tr("Log compressor: Timestamp %1 not found in dataset, ignoring log line %2").arg(time).arg(linecounter));
//continue;
failed = true;
}
else
{
emit logProcessingStatusChanged(tr("Log compressor: Timestamp %1 not found in dataset, restarting search.").arg(time));
offsetLimit*=2;
}
} }
} }
if (index % (dataLines/10) == 0) emit logProcessingStatusChanged(tr("Log compressor: Processed %1%% of %2 lines").arg(index/(float)dataLines).arg(dataLines));
// When the algorithm reaches here the correct index was found
lastTimeIndex = index; lastTimeIndex = index;
QString outLine = outLines->at(index); QString outLine = outLines->at(index);
QStringList outParts = outLine.split(separator); QStringList outParts = outLine.split(separator);
@ -176,12 +219,11 @@ void LogCompressor::run()
outLine = outParts.join(separator); outLine = outParts.join(separator);
outLines->replace(index, outLine); outLines->replace(index, outLine);
} }
// Add header, write out file // Add header, write out file
file.close(); file.close();
if (outFileName == "") if (outFileName == "")
{ {
QFile::remove(file.fileName()); QFile::remove(file.fileName());
@ -191,19 +233,20 @@ void LogCompressor::run()
return; return;
outfile.write(QString(QString("unix_timestamp") + separator + header.replace(" ", "_") + QString("\n")).toLatin1()); outfile.write(QString(QString("unix_timestamp") + separator + header.replace(" ", "_") + QString("\n")).toLatin1());
//QString fileHeader = QString("unix_timestamp") + header.replace(" ", "_") + QString("\n"); //QString fileHeader = QString("unix_timestamp") + header.replace(" ", "_") + QString("\n");
// File output // File output
for (int i = 0; i < outLines->length(); i++) for (int i = 0; i < outLines->length(); i++)
{ {
//qDebug() << outLines->at(i); //qDebug() << outLines->at(i);
outfile.write(QString(outLines->at(i) + "\n").toLatin1()); outfile.write(QString(outLines->at(i) + "\n").toLatin1());
} }
currentDataLine = 0; currentDataLine = 0;
dataLines = 1; dataLines = 1;
delete keys; delete keys;
qDebug() << "Done with logfile processing"; emit logProcessingStatusChanged(tr("Log compressor: Finished processing file: %1").arg(outfile.fileName()));
//qDebug() << "Done with logfile processing";
emit finishedFile(outfile.fileName()); emit finishedFile(outfile.fileName());
running = false; running = false;
} }

1
src/LogCompressor.h
Unescape View File

@ -27,6 +27,7 @@ signals:
/** @brief This signal is emitted once a logfile has been finished writing /** @brief This signal is emitted once a logfile has been finished writing
* @param fileName The name out the output (CSV) file * @param fileName The name out the output (CSV) file
*/ */
void logProcessingStatusChanged(QString);
void finishedFile(QString fileName); void finishedFile(QString fileName);
}; };

29
src/ui/MainWindow.cc
Unescape View File

@ -120,6 +120,8 @@ MainWindow::MainWindow(QWidget *parent):
// Load mavlink view as default widget set // Load mavlink view as default widget set
//loadMAVLinkView(); //loadMAVLinkView();
statusBar()->setSizeGripEnabled(true);
if (settings.contains("geometry")) if (settings.contains("geometry"))
{ {
// Restore the window geometry // Restore the window geometry
@ -144,8 +146,7 @@ MainWindow::MainWindow(QWidget *parent):
MainWindow::~MainWindow() MainWindow::~MainWindow()
{ {
delete statusBar;
statusBar = NULL;
} }
void MainWindow::buildCommonWidgets() void MainWindow::buildCommonWidgets()
@ -811,15 +812,6 @@ void MainWindow::configureWindowName()
#endif #endif
} }
QStatusBar* MainWindow::createStatusBar()
{
QStatusBar* bar = new QStatusBar();
/* Add status fields and messages */
/* Enable resize grip in the bottom right corner */
bar->setSizeGripEnabled(true);
return bar;
}
void MainWindow::startVideoCapture() void MainWindow::startVideoCapture()
{ {
QString format = "bmp"; QString format = "bmp";
@ -890,7 +882,7 @@ void MainWindow::reloadStylesheet()
*/ */
void MainWindow::showStatusMessage(const QString& status, int timeout) void MainWindow::showStatusMessage(const QString& status, int timeout)
{ {
statusBar->showMessage(status, timeout); statusBar()->showMessage(status, timeout);
} }
/** /**
@ -901,7 +893,7 @@ void MainWindow::showStatusMessage(const QString& status, int timeout)
*/ */
void MainWindow::showStatusMessage(const QString& status) void MainWindow::showStatusMessage(const QString& status)
{ {
statusBar->showMessage(status, 5); statusBar()->showMessage(status, 20000);
} }
void MainWindow::showCriticalMessage(const QString& title, const QString& message) void MainWindow::showCriticalMessage(const QString& title, const QString& message)
@ -915,6 +907,17 @@ void MainWindow::showCriticalMessage(const QString& title, const QString& messag
msgBox.exec(); msgBox.exec();
} }
void MainWindow::showInfoMessage(const QString& title, const QString& message)
{
QMessageBox msgBox(MainWindow::instance());
msgBox.setIcon(QMessageBox::Information);
msgBox.setText(title);
msgBox.setInformativeText(message);
msgBox.setStandardButtons(QMessageBox::Ok);
msgBox.setDefaultButton(QMessageBox::Ok);
msgBox.exec();
}
/** /**
* @brief Create all actions associated to the main window * @brief Create all actions associated to the main window
* *

6
src/ui/MainWindow.h
Unescape View File

@ -98,6 +98,8 @@ public slots:
void showStatusMessage(const QString& status); void showStatusMessage(const QString& status);
/** @brief Shows a critical message as popup or as widget */ /** @brief Shows a critical message as popup or as widget */
void showCriticalMessage(const QString& title, const QString& message); void showCriticalMessage(const QString& title, const QString& message);
/** @brief Shows an info message as popup or as widget */
void showInfoMessage(const QString& title, const QString& message);
void addLink(); void addLink();
void addLink(LinkInterface* link); void addLink(LinkInterface* link);
@ -296,10 +298,6 @@ protected:
VIEW_SECTIONS currentView; VIEW_SECTIONS currentView;
bool aboutToCloseFlag; bool aboutToCloseFlag;
QStatusBar* statusBar;
QStatusBar* createStatusBar();
void clearView(); void clearView();
void buildCommonWidgets(); void buildCommonWidgets();

5
src/ui/linechart/LinechartWidget.cc
Unescape View File

@ -48,6 +48,7 @@ This file is part of the PIXHAWK project
#include "LinechartWidget.h" #include "LinechartWidget.h"
#include "LinechartPlot.h" #include "LinechartPlot.h"
#include "LogCompressor.h" #include "LogCompressor.h"
#include "MainWindow.h"
#include "QGC.h" #include "QGC.h"
#include "MG.h" #include "MG.h"
@ -348,7 +349,7 @@ void LinechartWidget::startLogging()
// Let user select the log file name // Let user select the log file name
QDate date(QDate::currentDate()); QDate date(QDate::currentDate());
// QString("./pixhawk-log-" + date.toString("yyyy-MM-dd") + "-" + QString::number(logindex) + ".log") // QString("./pixhawk-log-" + date.toString("yyyy-MM-dd") + "-" + QString::number(logindex) + ".log")
QString fileName = QFileDialog::getSaveFileName(this, tr("Specify log file name"), QDesktopServices::storageLocation(QDesktopServices::DesktopLocation), tr("Logfile (*.csv, *.txt);;")); QString fileName = QFileDialog::getSaveFileName(this, tr("Specify log file name"), QDesktopServices::storageLocation(QDesktopServices::DesktopLocation), tr("Logfile (*.csv *.txt);;"));
if (!fileName.contains(".")) if (!fileName.contains("."))
{ {
@ -399,6 +400,8 @@ void LinechartWidget::stopLogging()
// Postprocess log file // Postprocess log file
compressor = new LogCompressor(logFile->fileName()); compressor = new LogCompressor(logFile->fileName());
connect(compressor, SIGNAL(finishedFile(QString)), this, SIGNAL(logfileWritten(QString))); connect(compressor, SIGNAL(finishedFile(QString)), this, SIGNAL(logfileWritten(QString)));
connect(compressor, SIGNAL(logProcessingStatusChanged(QString)), MainWindow::instance(), SLOT(showStatusMessage(QString)));
MainWindow::instance()->showInfoMessage("Logging ended", "QGroundControl is now compressing the logfile in a consistent CVS file. This may take a while, you can continue to use QGroundControl. Status updates appear at the bottom of the window.");
compressor->startCompression(); compressor->startCompression();
} }
logButton->setText(tr("Start logging")); logButton->setText(tr("Start logging"));

Write Preview
Loading…
Cancel
Save