diff --git a/qgroundcontrol.pro b/qgroundcontrol.pro index 7899463..d3fc44a 100644 --- a/qgroundcontrol.pro +++ b/qgroundcontrol.pro @@ -489,6 +489,7 @@ DebugBuild { PX4FirmwarePlugin { PX4FirmwarePluginFactory { APMFirmwarePlugin { src/MissionManager/TransectStyleComplexItemTest.h \ src/MissionManager/TransectStyleComplexItemTestBase.h \ src/MissionManager/VisualMissionItemTest.h \ + src/qgcunittest/ComponentInformationCacheTest.h \ src/qgcunittest/GeoTest.h \ src/qgcunittest/MavlinkLogTest.h \ src/qgcunittest/MultiSignalSpy.h \ @@ -535,6 +536,7 @@ DebugBuild { PX4FirmwarePlugin { PX4FirmwarePluginFactory { APMFirmwarePlugin { src/MissionManager/TransectStyleComplexItemTest.cc \ src/MissionManager/TransectStyleComplexItemTestBase.cc \ src/MissionManager/VisualMissionItemTest.cc \ + src/qgcunittest/ComponentInformationCacheTest.cc \ src/qgcunittest/GeoTest.cc \ src/qgcunittest/MavlinkLogTest.cc \ src/qgcunittest/MultiSignalSpy.cc \ @@ -682,6 +684,7 @@ HEADERS += \ src/Vehicle/CompInfo.h \ src/Vehicle/CompInfoParam.h \ src/Vehicle/CompInfoGeneral.h \ + src/Vehicle/ComponentInformationCache.h \ src/Vehicle/ComponentInformationManager.h \ src/Vehicle/FTPManager.h \ src/Vehicle/GPSRTKFactGroup.h \ @@ -916,6 +919,7 @@ SOURCES += \ src/Vehicle/CompInfo.cc \ src/Vehicle/CompInfoParam.cc \ src/Vehicle/CompInfoGeneral.cc \ + src/Vehicle/ComponentInformationCache.cc \ src/Vehicle/ComponentInformationManager.cc \ src/Vehicle/FTPManager.cc \ src/Vehicle/GPSRTKFactGroup.cc \ diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index fdbf220..627c07f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -49,6 +49,7 @@ if(BUILD_TESTING) add_subdirectory(qgcunittest) + add_qgc_test(ComponentInformationCacheTest) add_qgc_test(CameraCalcTest) add_qgc_test(CameraSectionTest) add_qgc_test(CorridorScanComplexItemTest) diff --git a/src/Vehicle/CMakeLists.txt b/src/Vehicle/CMakeLists.txt index a652879..bef84fe 100644 --- a/src/Vehicle/CMakeLists.txt +++ b/src/Vehicle/CMakeLists.txt @@ -22,6 +22,8 @@ add_library(Vehicle CompInfoParam.h CompInfoGeneral.cc CompInfoGeneral.h + ComponentInformationCache.cc + ComponentInformationCache.h ComponentInformationManager.cc ComponentInformationManager.h FTPManager.cc diff --git a/src/Vehicle/ComponentInformationCache.cc b/src/Vehicle/ComponentInformationCache.cc new file mode 100644 index 0000000..d1cab49 --- /dev/null +++ b/src/Vehicle/ComponentInformationCache.cc @@ -0,0 +1,192 @@ +/**************************************************************************** + * + * (c) 2021 QGROUNDCONTROL PROJECT + * + * QGroundControl is licensed according to the terms in the file + * COPYING.md in the root of the source code directory. + * + ****************************************************************************/ + +#include "ComponentInformationCache.h" + +#include +#include +#include + +QGC_LOGGING_CATEGORY(ComponentInformationCacheLog, "ComponentInformationCacheLog") + +ComponentInformationCache::ComponentInformationCache(const QDir& path, int maxNumFiles) + : _path(path), _maxNumFiles(maxNumFiles) +{ + initializeDirectory(); +} + +ComponentInformationCache& ComponentInformationCache::defaultInstance() +{ + QString cacheDir = QStandardPaths::writableLocation(QStandardPaths::CacheLocation) + QLatin1String("/QGCCompInfoCache"); + static ComponentInformationCache instance(cacheDir, 50); + return instance; +} + +QString ComponentInformationCache::metaFileName(const QString& fileTag) +{ + return _path.filePath(fileTag+_metaExtension); +} + +QString ComponentInformationCache::dataFileName(const QString& fileTag) +{ + return _path.filePath(fileTag+_cacheExtension); +} + +QString ComponentInformationCache::access(const QString &fileTag) +{ + QFile meta(metaFileName(fileTag)); + QFile data(dataFileName(fileTag)); + if (!meta.exists() || !data.exists()) { + qCDebug(ComponentInformationCacheLog) << "Cache miss for" << fileTag; + return ""; + } + + qCDebug(ComponentInformationCacheLog) << "Cache hit for" << fileTag; + + // mark access + Meta m{}; + AccessCounterType previousCounter = -1; + if (meta.open(QIODevice::ReadWrite)) { + if (meta.read((char*)&m, sizeof(m)) == sizeof(m)) { + previousCounter = m.accessCounter; + m.accessCounter = _nextAccessCounter; + meta.seek(0); + if (meta.write((const char*)&m, sizeof(m)) != sizeof(m)) { + qCWarning(ComponentInformationCacheLog) << "Meta write failed" << meta.fileName() << meta.errorString(); + } + } else { + qCWarning(ComponentInformationCacheLog) << "Meta read failed" << meta.fileName() << meta.errorString(); + } + meta.close(); + } else { + qCWarning(ComponentInformationCacheLog) << "Failed to open" << meta.fileName() << meta.errorString(); + } + + _cachedFiles.remove(previousCounter); + _cachedFiles[_nextAccessCounter] = fileTag; + ++_nextAccessCounter; + + return data.fileName(); +} + +QString ComponentInformationCache::insert(const QString &fileTag, const QString &fileName) +{ + QFile meta(metaFileName(fileTag)); + QFile data(dataFileName(fileTag)); + QFile fileToCache(fileName); + if (meta.exists() || data.exists()) { + qCDebug(ComponentInformationCacheLog) << "Not inserting, entry already exists" << fileTag; + fileToCache.remove(); + return data.fileName(); + } + + // move the file to the cache location + if (!fileToCache.rename(data.fileName())) { + qCWarning(ComponentInformationCacheLog) << "File rename failed from:to" << fileName << data.fileName(); + return ""; + } + + // write meta data + Meta m{}; + m.accessCounter = _nextAccessCounter; + if (meta.open(QIODevice::WriteOnly)) { + if (meta.write((const char*)&m, sizeof(m)) != sizeof(m)) { + qCWarning(ComponentInformationCacheLog) << "Meta write failed" << meta.fileName() << meta.errorString(); + } + meta.close(); + } else { + qCWarning(ComponentInformationCacheLog) << "Failed to open" << meta.fileName() << meta.errorString(); + } + + // update internal data + _cachedFiles[_nextAccessCounter++] = fileTag; + ++_numFiles; + + removeOldEntries(); + return data.fileName(); +} + +void ComponentInformationCache::initializeDirectory() +{ + if (!_path.exists()) { + QDir d; + if (!d.mkdir(_path.path())) { + qCWarning(ComponentInformationCacheLog) << "Failed to create dir" << _path.path(); + } + } + + QDir::Filters filters = QDir::Files | QDir::NoDotAndDotDot; + QDirIterator it(_path.path(), filters, QDirIterator::NoIteratorFlags); + while (it.hasNext()) { + QString path = it.next(); + + if (path.endsWith(_metaExtension)) { + QFile meta(path); + QFile data(path.mid(0, path.length()-strlen(_metaExtension))+_cacheExtension); + bool validationFailed = false; + if (!data.exists()) { + validationFailed = true; + } + + // read meta + validate + Meta m{}; + const uint32_t expectedMagic = m.magic; + const uint32_t expectedVersion = m.version; + if (meta.open(QIODevice::ReadOnly)) { + if (meta.read((char*)&m, sizeof(m)) == sizeof(m)) { + if (m.magic != expectedMagic || m.version != expectedVersion) { + validationFailed = true; + } + } else { + validationFailed = true; + } + meta.close(); + } else { + validationFailed = true; + } + + if (validationFailed) { + qCWarning(ComponentInformationCacheLog) << "Validation failed, removing cache files" << path; + meta.remove(); + data.remove(); + } else { + // extract the tag + QString tag = it.fileName(); + tag = tag.mid(0, tag.length()-strlen(_metaExtension)); + _cachedFiles[m.accessCounter] = tag; + + qCDebug(ComponentInformationCacheLog) << "Found cached file:counter" << meta.fileName() << m.accessCounter; + + if (m.accessCounter >= _nextAccessCounter) { + _nextAccessCounter = m.accessCounter + 1; + } + } + + } else if (!path.endsWith(_cacheExtension)) { + QFile::remove(path); + } + } + _numFiles = _cachedFiles.size(); + removeOldEntries(); +} + +void ComponentInformationCache::removeOldEntries() +{ + while (_numFiles > _maxNumFiles) { + auto iter = _cachedFiles.begin(); + QFile meta(metaFileName(iter.value())); + QFile data(dataFileName(iter.value())); + qCDebug(ComponentInformationCacheLog) << "Removing cache entry num:counter:file" << _numFiles << iter.key() << iter.value(); + meta.remove(); + data.remove(); + + _cachedFiles.erase(iter); + --_numFiles; + } +} diff --git a/src/Vehicle/ComponentInformationCache.h b/src/Vehicle/ComponentInformationCache.h new file mode 100644 index 0000000..6d83c73 --- /dev/null +++ b/src/Vehicle/ComponentInformationCache.h @@ -0,0 +1,77 @@ +/**************************************************************************** + * + * (c) 2021 QGROUNDCONTROL PROJECT + * + * QGroundControl is licensed according to the terms in the file + * COPYING.md in the root of the source code directory. + * + ****************************************************************************/ + +#pragma once + +#include "QGCLoggingCategory.h" + +#include +#include +#include + +#include + +Q_DECLARE_LOGGING_CATEGORY(ComponentInformationCacheLog) + +/** + * Simple file cache with a maximum number of files and LRU retention policy based on last access + * Notes: + * - fileTag defines the cache keys and the format is up to the user + * - only one instance per directory must exist + * - not thread-safe + */ +class ComponentInformationCache : public QObject +{ + Q_OBJECT +public: + ComponentInformationCache(const QDir& path, int maxNumFiles); + + static ComponentInformationCache& defaultInstance(); + + /** + * Try to access a file and set the access counter + * @param fileTag + * @return empty string if not found, or file path + */ + QString access(const QString& fileTag); + + /** + * Insert a file into the cache & remove old files if there's too many. + * @param fileTag + * @param fileName file to insert, will be moved (or deleted if already exists) + * @return cached file name if inserted or already exists, "" on error + */ + QString insert(const QString &fileTag, const QString& fileName); + +private: + + static constexpr const char* _metaExtension = ".meta"; + static constexpr const char* _cacheExtension = ".cache"; + + using AccessCounterType = uint64_t; + + struct Meta { + uint32_t magic{0x9a9cad0e}; + uint32_t version{0}; + AccessCounterType accessCounter{0}; + }; + + void initializeDirectory(); + void removeOldEntries(); + + QString metaFileName(const QString& fileTag); + QString dataFileName(const QString& fileTag); + + const QDir _path; + const int _maxNumFiles; + + AccessCounterType _nextAccessCounter{0}; + int _numFiles{0}; + QMap _cachedFiles; +}; diff --git a/src/qgcunittest/CMakeLists.txt b/src/qgcunittest/CMakeLists.txt index 5f65805..c1228f2 100644 --- a/src/qgcunittest/CMakeLists.txt +++ b/src/qgcunittest/CMakeLists.txt @@ -4,6 +4,8 @@ add_library(qgcunittest #FileDialogTest.h #FileManagerTest.cc #FileManagerTest.h + ComponentInformationCacheTest.cc + ComponentInformationCacheTest.h GeoTest.cc GeoTest.h #MainWindowTest.cc diff --git a/src/qgcunittest/ComponentInformationCacheTest.cc b/src/qgcunittest/ComponentInformationCacheTest.cc new file mode 100644 index 0000000..cbeda0a --- /dev/null +++ b/src/qgcunittest/ComponentInformationCacheTest.cc @@ -0,0 +1,154 @@ +/**************************************************************************** + * + * (c) 2021 QGROUNDCONTROL PROJECT + * + * QGroundControl is licensed according to the terms in the file + * COPYING.md in the root of the source code directory. + * + ****************************************************************************/ + + +#include "ComponentInformationCacheTest.h" + + +ComponentInformationCacheTest::ComponentInformationCacheTest() +{ + _cacheDir = QStandardPaths::writableLocation(QStandardPaths::TempLocation) + QLatin1String("/QGCCacheTest"); + _tmpFilesDir = QStandardPaths::writableLocation(QStandardPaths::TempLocation) + QLatin1String("/QGCTestFiles"); + _cleanup(); +} + +void ComponentInformationCacheTest::_setup() +{ + QDir d(_tmpFilesDir); + d.mkdir(_tmpFilesDir); + + _tmpFiles.clear(); + + for (int i = 0; i < 30; ++i) { + TmpFile t; + t.content = QString::asprintf("%i", i); + t.path = _tmpFilesDir + "/" + t.content + ".txt"; + t.cacheTag = QString::asprintf("_tag_%08i_xy", i); + QFile f(t.path); + f.open(QIODevice::WriteOnly); + f.write(t.content.toUtf8().constData(), t.content.toUtf8().size()); + f.close(); + _tmpFiles.push_back(t); + } +} + +void ComponentInformationCacheTest::_cleanup() +{ + QDir t(_tmpFilesDir); + t.removeRecursively(); + QDir d(_cacheDir); + d.removeRecursively(); +} + +void ComponentInformationCacheTest::_basic_test() +{ + _setup(); + ComponentInformationCache cache(_cacheDir, 10); + + QDir cacheDir(_cacheDir); + QVERIFY(cacheDir.exists()); + + _tmpFiles[0].cachedPath = cache.insert(_tmpFiles[0].cacheTag, _tmpFiles[0].path); + QVERIFY(!_tmpFiles[0].cachedPath.isEmpty()); + QVERIFY(QFile(_tmpFiles[0].cachedPath).exists()); + QVERIFY(!QFile(_tmpFiles[0].path).exists()); + + QVERIFY(cache.access(_tmpFiles[0].cacheTag) == _tmpFiles[0].cachedPath); + + QFile f(_tmpFiles[0].cachedPath); + QVERIFY(f.open(QFile::ReadOnly | QFile::Text)); + QTextStream in(&f); + QVERIFY(in.readAll() == _tmpFiles[0].content); + + _cleanup(); +} + + +void ComponentInformationCacheTest::_lru_test() +{ + _setup(); + ComponentInformationCache cache(_cacheDir, 3); + + auto insert = [&](int idx) { + _tmpFiles[idx].cachedPath = cache.insert(_tmpFiles[idx].cacheTag, _tmpFiles[idx].path); + QVERIFY(!_tmpFiles[idx].cachedPath.isEmpty()); + }; + insert(1); + insert(3); + insert(0); + + QVERIFY(cache.access(_tmpFiles[0].cacheTag) == _tmpFiles[0].cachedPath); + QVERIFY(cache.access(_tmpFiles[1].cacheTag) == _tmpFiles[1].cachedPath); + QVERIFY(cache.access(_tmpFiles[3].cacheTag) == _tmpFiles[3].cachedPath); + + insert(4); + + QVERIFY(cache.access(_tmpFiles[0].cacheTag) == ""); + QVERIFY(cache.access(_tmpFiles[1].cacheTag) == _tmpFiles[1].cachedPath); + QVERIFY(cache.access(_tmpFiles[3].cacheTag) == _tmpFiles[3].cachedPath); + QVERIFY(cache.access(_tmpFiles[4].cacheTag) == _tmpFiles[4].cachedPath); + + QVERIFY(cache.access(_tmpFiles[3].cacheTag) == _tmpFiles[3].cachedPath); + QVERIFY(cache.access(_tmpFiles[1].cacheTag) == _tmpFiles[1].cachedPath); + QVERIFY(cache.access(_tmpFiles[3].cacheTag) == _tmpFiles[3].cachedPath); + + insert(5); + + QVERIFY(cache.access(_tmpFiles[4].cacheTag) == ""); + QVERIFY(cache.access(_tmpFiles[1].cacheTag) == _tmpFiles[1].cachedPath); + QVERIFY(cache.access(_tmpFiles[3].cacheTag) == _tmpFiles[3].cachedPath); + QVERIFY(cache.access(_tmpFiles[5].cacheTag) == _tmpFiles[5].cachedPath); + + QVERIFY(cache.access(_tmpFiles[3].cacheTag) == _tmpFiles[3].cachedPath); + insert(6); + insert(7); + + QVERIFY(cache.access(_tmpFiles[4].cacheTag) == ""); + QVERIFY(cache.access(_tmpFiles[1].cacheTag) == ""); + QVERIFY(cache.access(_tmpFiles[5].cacheTag) == ""); + QVERIFY(cache.access(_tmpFiles[3].cacheTag) == _tmpFiles[3].cachedPath); + QVERIFY(cache.access(_tmpFiles[6].cacheTag) == _tmpFiles[6].cachedPath); + QVERIFY(cache.access(_tmpFiles[7].cacheTag) == _tmpFiles[7].cachedPath); + + _cleanup(); +} + +void ComponentInformationCacheTest::_multi_test() +{ + _setup(); + + auto insert = [&](ComponentInformationCache& cache, int idx) { + _tmpFiles[idx].cachedPath = cache.insert(_tmpFiles[idx].cacheTag, _tmpFiles[idx].path); + QVERIFY(!_tmpFiles[idx].cachedPath.isEmpty()); + }; + + { + ComponentInformationCache cache(_cacheDir, 5); + for (int i = 0; i < 5; ++i) { + insert(cache, i); + QVERIFY(cache.access(_tmpFiles[i].cacheTag) == _tmpFiles[i].cachedPath); + } + QVERIFY(cache.access(_tmpFiles[1].cacheTag) == _tmpFiles[1].cachedPath); + } + { + // reduce cache size and ensure oldest entries are evicted + ComponentInformationCache cache(_cacheDir, 3); + QVERIFY(cache.access(_tmpFiles[0].cacheTag) == ""); + QVERIFY(cache.access(_tmpFiles[1].cacheTag) == _tmpFiles[1].cachedPath); + QVERIFY(cache.access(_tmpFiles[2].cacheTag) == ""); + QVERIFY(cache.access(_tmpFiles[3].cacheTag) == _tmpFiles[3].cachedPath); + QVERIFY(cache.access(_tmpFiles[4].cacheTag) == _tmpFiles[4].cachedPath); + + insert(cache, 10); + QVERIFY(cache.access(_tmpFiles[1].cacheTag) == ""); + QVERIFY(cache.access(_tmpFiles[10].cacheTag) == _tmpFiles[10].cachedPath); + } + + _cleanup(); +} diff --git a/src/qgcunittest/ComponentInformationCacheTest.h b/src/qgcunittest/ComponentInformationCacheTest.h new file mode 100644 index 0000000..cd34444 --- /dev/null +++ b/src/qgcunittest/ComponentInformationCacheTest.h @@ -0,0 +1,47 @@ +/**************************************************************************** + * + * (c) 2021 QGROUNDCONTROL PROJECT + * + * QGroundControl is licensed according to the terms in the file + * COPYING.md in the root of the source code directory. + * + ****************************************************************************/ + + +#pragma once + +#include "ComponentInformationCache.h" + +#include "UnitTest.h" + +#include + +class ComponentInformationCacheTest : public UnitTest +{ + Q_OBJECT + +public: + ComponentInformationCacheTest(); + virtual ~ComponentInformationCacheTest() = default; + +private slots: + void _basic_test(); + void _lru_test(); + void _multi_test(); +private: + void _setup(); + void _cleanup(); + + struct TmpFile { + QString path; + QString cacheTag; + QString content; + QString cachedPath; + }; + + QVector _tmpFiles; + + QString _cacheDir; + QString _tmpFilesDir; +}; + diff --git a/src/qgcunittest/UnitTestList.cc b/src/qgcunittest/UnitTestList.cc index 98c43aa..e6e8411 100644 --- a/src/qgcunittest/UnitTestList.cc +++ b/src/qgcunittest/UnitTestList.cc @@ -11,6 +11,7 @@ // We keep the list of all unit tests in a global location so it's easier to see which // ones are enabled/disabled +#include "ComponentInformationCacheTest.h" #include "FactSystemTestGeneric.h" #include "FactSystemTestPX4.h" //#include "FileDialogTest.h" @@ -49,6 +50,7 @@ #include "VehicleLinkManagerTest.h" #include "LandingComplexItemTest.h" +UT_REGISTER_TEST(ComponentInformationCacheTest) UT_REGISTER_TEST(FactSystemTestGeneric) UT_REGISTER_TEST(FactSystemTestPX4) //UT_REGISTER_TEST(FileDialogTest)