Beat Küng
4 years ago
committed by
Don Gagne
9 changed files with 481 additions and 0 deletions
@ -0,0 +1,192 @@
@@ -0,0 +1,192 @@
|
||||
/****************************************************************************
|
||||
* |
||||
* (c) 2021 QGROUNDCONTROL PROJECT <http://www.qgroundcontrol.org>
|
||||
* |
||||
* QGroundControl is licensed according to the terms in the file |
||||
* COPYING.md in the root of the source code directory. |
||||
* |
||||
****************************************************************************/ |
||||
|
||||
#include "ComponentInformationCache.h" |
||||
|
||||
#include <QFile> |
||||
#include <QDirIterator> |
||||
#include <QStandardPaths> |
||||
|
||||
QGC_LOGGING_CATEGORY(ComponentInformationCacheLog, "ComponentInformationCacheLog") |
||||
|
||||
ComponentInformationCache::ComponentInformationCache(const QDir& path, int maxNumFiles) |
||||
: _path(path), _maxNumFiles(maxNumFiles) |
||||
{ |
||||
initializeDirectory(); |
||||
} |
||||
|
||||
ComponentInformationCache& ComponentInformationCache::defaultInstance() |
||||
{ |
||||
QString cacheDir = QStandardPaths::writableLocation(QStandardPaths::CacheLocation) + QLatin1String("/QGCCompInfoCache"); |
||||
static ComponentInformationCache instance(cacheDir, 50); |
||||
return instance; |
||||
} |
||||
|
||||
QString ComponentInformationCache::metaFileName(const QString& fileTag) |
||||
{ |
||||
return _path.filePath(fileTag+_metaExtension); |
||||
} |
||||
|
||||
QString ComponentInformationCache::dataFileName(const QString& fileTag) |
||||
{ |
||||
return _path.filePath(fileTag+_cacheExtension); |
||||
} |
||||
|
||||
QString ComponentInformationCache::access(const QString &fileTag) |
||||
{ |
||||
QFile meta(metaFileName(fileTag)); |
||||
QFile data(dataFileName(fileTag)); |
||||
if (!meta.exists() || !data.exists()) { |
||||
qCDebug(ComponentInformationCacheLog) << "Cache miss for" << fileTag; |
||||
return ""; |
||||
} |
||||
|
||||
qCDebug(ComponentInformationCacheLog) << "Cache hit for" << fileTag; |
||||
|
||||
// mark access
|
||||
Meta m{}; |
||||
AccessCounterType previousCounter = -1; |
||||
if (meta.open(QIODevice::ReadWrite)) { |
||||
if (meta.read((char*)&m, sizeof(m)) == sizeof(m)) { |
||||
previousCounter = m.accessCounter; |
||||
m.accessCounter = _nextAccessCounter; |
||||
meta.seek(0); |
||||
if (meta.write((const char*)&m, sizeof(m)) != sizeof(m)) { |
||||
qCWarning(ComponentInformationCacheLog) << "Meta write failed" << meta.fileName() << meta.errorString(); |
||||
} |
||||
} else { |
||||
qCWarning(ComponentInformationCacheLog) << "Meta read failed" << meta.fileName() << meta.errorString(); |
||||
} |
||||
meta.close(); |
||||
} else { |
||||
qCWarning(ComponentInformationCacheLog) << "Failed to open" << meta.fileName() << meta.errorString(); |
||||
} |
||||
|
||||
_cachedFiles.remove(previousCounter); |
||||
_cachedFiles[_nextAccessCounter] = fileTag; |
||||
++_nextAccessCounter; |
||||
|
||||
return data.fileName(); |
||||
} |
||||
|
||||
QString ComponentInformationCache::insert(const QString &fileTag, const QString &fileName) |
||||
{ |
||||
QFile meta(metaFileName(fileTag)); |
||||
QFile data(dataFileName(fileTag)); |
||||
QFile fileToCache(fileName); |
||||
if (meta.exists() || data.exists()) { |
||||
qCDebug(ComponentInformationCacheLog) << "Not inserting, entry already exists" << fileTag; |
||||
fileToCache.remove(); |
||||
return data.fileName(); |
||||
} |
||||
|
||||
// move the file to the cache location
|
||||
if (!fileToCache.rename(data.fileName())) { |
||||
qCWarning(ComponentInformationCacheLog) << "File rename failed from:to" << fileName << data.fileName(); |
||||
return ""; |
||||
} |
||||
|
||||
// write meta data
|
||||
Meta m{}; |
||||
m.accessCounter = _nextAccessCounter; |
||||
if (meta.open(QIODevice::WriteOnly)) { |
||||
if (meta.write((const char*)&m, sizeof(m)) != sizeof(m)) { |
||||
qCWarning(ComponentInformationCacheLog) << "Meta write failed" << meta.fileName() << meta.errorString(); |
||||
} |
||||
meta.close(); |
||||
} else { |
||||
qCWarning(ComponentInformationCacheLog) << "Failed to open" << meta.fileName() << meta.errorString(); |
||||
} |
||||
|
||||
// update internal data
|
||||
_cachedFiles[_nextAccessCounter++] = fileTag; |
||||
++_numFiles; |
||||
|
||||
removeOldEntries(); |
||||
return data.fileName(); |
||||
} |
||||
|
||||
void ComponentInformationCache::initializeDirectory() |
||||
{ |
||||
if (!_path.exists()) { |
||||
QDir d; |
||||
if (!d.mkdir(_path.path())) { |
||||
qCWarning(ComponentInformationCacheLog) << "Failed to create dir" << _path.path(); |
||||
} |
||||
} |
||||
|
||||
QDir::Filters filters = QDir::Files | QDir::NoDotAndDotDot; |
||||
QDirIterator it(_path.path(), filters, QDirIterator::NoIteratorFlags); |
||||
while (it.hasNext()) { |
||||
QString path = it.next(); |
||||
|
||||
if (path.endsWith(_metaExtension)) { |
||||
QFile meta(path); |
||||
QFile data(path.mid(0, path.length()-strlen(_metaExtension))+_cacheExtension); |
||||
bool validationFailed = false; |
||||
if (!data.exists()) { |
||||
validationFailed = true; |
||||
} |
||||
|
||||
// read meta + validate
|
||||
Meta m{}; |
||||
const uint32_t expectedMagic = m.magic; |
||||
const uint32_t expectedVersion = m.version; |
||||
if (meta.open(QIODevice::ReadOnly)) { |
||||
if (meta.read((char*)&m, sizeof(m)) == sizeof(m)) { |
||||
if (m.magic != expectedMagic || m.version != expectedVersion) { |
||||
validationFailed = true; |
||||
} |
||||
} else { |
||||
validationFailed = true; |
||||
} |
||||
meta.close(); |
||||
} else { |
||||
validationFailed = true; |
||||
} |
||||
|
||||
if (validationFailed) { |
||||
qCWarning(ComponentInformationCacheLog) << "Validation failed, removing cache files" << path; |
||||
meta.remove(); |
||||
data.remove(); |
||||
} else { |
||||
// extract the tag
|
||||
QString tag = it.fileName(); |
||||
tag = tag.mid(0, tag.length()-strlen(_metaExtension)); |
||||
_cachedFiles[m.accessCounter] = tag; |
||||
|
||||
qCDebug(ComponentInformationCacheLog) << "Found cached file:counter" << meta.fileName() << m.accessCounter; |
||||
|
||||
if (m.accessCounter >= _nextAccessCounter) { |
||||
_nextAccessCounter = m.accessCounter + 1; |
||||
} |
||||
} |
||||
|
||||
} else if (!path.endsWith(_cacheExtension)) { |
||||
QFile::remove(path); |
||||
} |
||||
} |
||||
_numFiles = _cachedFiles.size(); |
||||
removeOldEntries(); |
||||
} |
||||
|
||||
void ComponentInformationCache::removeOldEntries() |
||||
{ |
||||
while (_numFiles > _maxNumFiles) { |
||||
auto iter = _cachedFiles.begin(); |
||||
QFile meta(metaFileName(iter.value())); |
||||
QFile data(dataFileName(iter.value())); |
||||
qCDebug(ComponentInformationCacheLog) << "Removing cache entry num:counter:file" << _numFiles << iter.key() << iter.value(); |
||||
meta.remove(); |
||||
data.remove(); |
||||
|
||||
_cachedFiles.erase(iter); |
||||
--_numFiles; |
||||
} |
||||
} |
||||
@ -0,0 +1,77 @@
@@ -0,0 +1,77 @@
|
||||
/****************************************************************************
|
||||
* |
||||
* (c) 2021 QGROUNDCONTROL PROJECT <http://www.qgroundcontrol.org>
|
||||
* |
||||
* QGroundControl is licensed according to the terms in the file |
||||
* COPYING.md in the root of the source code directory. |
||||
* |
||||
****************************************************************************/ |
||||
|
||||
#pragma once |
||||
|
||||
#include "QGCLoggingCategory.h" |
||||
|
||||
#include <QString> |
||||
#include <QDir> |
||||
#include <QMap> |
||||
|
||||
#include <cstdint> |
||||
|
||||
Q_DECLARE_LOGGING_CATEGORY(ComponentInformationCacheLog) |
||||
|
||||
/**
|
||||
* Simple file cache with a maximum number of files and LRU retention policy based on last access |
||||
* Notes: |
||||
* - fileTag defines the cache keys and the format is up to the user |
||||
* - only one instance per directory must exist |
||||
* - not thread-safe |
||||
*/ |
||||
class ComponentInformationCache : public QObject |
||||
{ |
||||
Q_OBJECT |
||||
public: |
||||
ComponentInformationCache(const QDir& path, int maxNumFiles); |
||||
|
||||
static ComponentInformationCache& defaultInstance(); |
||||
|
||||
/**
|
||||
* Try to access a file and set the access counter |
||||
* @param fileTag |
||||
* @return empty string if not found, or file path |
||||
*/ |
||||
QString access(const QString& fileTag); |
||||
|
||||
/**
|
||||
* Insert a file into the cache & remove old files if there's too many. |
||||
* @param fileTag |
||||
* @param fileName file to insert, will be moved (or deleted if already exists) |
||||
* @return cached file name if inserted or already exists, "" on error |
||||
*/ |
||||
QString insert(const QString &fileTag, const QString& fileName); |
||||
|
||||
private: |
||||
|
||||
static constexpr const char* _metaExtension = ".meta"; |
||||
static constexpr const char* _cacheExtension = ".cache"; |
||||
|
||||
using AccessCounterType = uint64_t; |
||||
|
||||
struct Meta { |
||||
uint32_t magic{0x9a9cad0e}; |
||||
uint32_t version{0}; |
||||
AccessCounterType accessCounter{0}; |
||||
}; |
||||
|
||||
void initializeDirectory(); |
||||
void removeOldEntries(); |
||||
|
||||
QString metaFileName(const QString& fileTag); |
||||
QString dataFileName(const QString& fileTag); |
||||
|
||||
const QDir _path; |
||||
const int _maxNumFiles; |
||||
|
||||
AccessCounterType _nextAccessCounter{0}; |
||||
int _numFiles{0}; |
||||
QMap<AccessCounterType, QString> _cachedFiles; |
||||
}; |
||||
@ -0,0 +1,154 @@
@@ -0,0 +1,154 @@
|
||||
/****************************************************************************
|
||||
* |
||||
* (c) 2021 QGROUNDCONTROL PROJECT <http://www.qgroundcontrol.org>
|
||||
* |
||||
* QGroundControl is licensed according to the terms in the file |
||||
* COPYING.md in the root of the source code directory. |
||||
* |
||||
****************************************************************************/ |
||||
|
||||
|
||||
#include "ComponentInformationCacheTest.h" |
||||
|
||||
|
||||
ComponentInformationCacheTest::ComponentInformationCacheTest() |
||||
{ |
||||
_cacheDir = QStandardPaths::writableLocation(QStandardPaths::TempLocation) + QLatin1String("/QGCCacheTest"); |
||||
_tmpFilesDir = QStandardPaths::writableLocation(QStandardPaths::TempLocation) + QLatin1String("/QGCTestFiles"); |
||||
_cleanup(); |
||||
} |
||||
|
||||
void ComponentInformationCacheTest::_setup() |
||||
{ |
||||
QDir d(_tmpFilesDir); |
||||
d.mkdir(_tmpFilesDir); |
||||
|
||||
_tmpFiles.clear(); |
||||
|
||||
for (int i = 0; i < 30; ++i) { |
||||
TmpFile t; |
||||
t.content = QString::asprintf("%i", i); |
||||
t.path = _tmpFilesDir + "/" + t.content + ".txt"; |
||||
t.cacheTag = QString::asprintf("_tag_%08i_xy", i); |
||||
QFile f(t.path); |
||||
f.open(QIODevice::WriteOnly); |
||||
f.write(t.content.toUtf8().constData(), t.content.toUtf8().size()); |
||||
f.close(); |
||||
_tmpFiles.push_back(t); |
||||
} |
||||
} |
||||
|
||||
void ComponentInformationCacheTest::_cleanup() |
||||
{ |
||||
QDir t(_tmpFilesDir); |
||||
t.removeRecursively(); |
||||
QDir d(_cacheDir); |
||||
d.removeRecursively(); |
||||
} |
||||
|
||||
void ComponentInformationCacheTest::_basic_test() |
||||
{ |
||||
_setup(); |
||||
ComponentInformationCache cache(_cacheDir, 10); |
||||
|
||||
QDir cacheDir(_cacheDir); |
||||
QVERIFY(cacheDir.exists()); |
||||
|
||||
_tmpFiles[0].cachedPath = cache.insert(_tmpFiles[0].cacheTag, _tmpFiles[0].path); |
||||
QVERIFY(!_tmpFiles[0].cachedPath.isEmpty()); |
||||
QVERIFY(QFile(_tmpFiles[0].cachedPath).exists()); |
||||
QVERIFY(!QFile(_tmpFiles[0].path).exists()); |
||||
|
||||
QVERIFY(cache.access(_tmpFiles[0].cacheTag) == _tmpFiles[0].cachedPath); |
||||
|
||||
QFile f(_tmpFiles[0].cachedPath); |
||||
QVERIFY(f.open(QFile::ReadOnly | QFile::Text)); |
||||
QTextStream in(&f); |
||||
QVERIFY(in.readAll() == _tmpFiles[0].content); |
||||
|
||||
_cleanup(); |
||||
} |
||||
|
||||
|
||||
void ComponentInformationCacheTest::_lru_test() |
||||
{ |
||||
_setup(); |
||||
ComponentInformationCache cache(_cacheDir, 3); |
||||
|
||||
auto insert = [&](int idx) { |
||||
_tmpFiles[idx].cachedPath = cache.insert(_tmpFiles[idx].cacheTag, _tmpFiles[idx].path); |
||||
QVERIFY(!_tmpFiles[idx].cachedPath.isEmpty()); |
||||
}; |
||||
insert(1); |
||||
insert(3); |
||||
insert(0); |
||||
|
||||
QVERIFY(cache.access(_tmpFiles[0].cacheTag) == _tmpFiles[0].cachedPath); |
||||
QVERIFY(cache.access(_tmpFiles[1].cacheTag) == _tmpFiles[1].cachedPath); |
||||
QVERIFY(cache.access(_tmpFiles[3].cacheTag) == _tmpFiles[3].cachedPath); |
||||
|
||||
insert(4); |
||||
|
||||
QVERIFY(cache.access(_tmpFiles[0].cacheTag) == ""); |
||||
QVERIFY(cache.access(_tmpFiles[1].cacheTag) == _tmpFiles[1].cachedPath); |
||||
QVERIFY(cache.access(_tmpFiles[3].cacheTag) == _tmpFiles[3].cachedPath); |
||||
QVERIFY(cache.access(_tmpFiles[4].cacheTag) == _tmpFiles[4].cachedPath); |
||||
|
||||
QVERIFY(cache.access(_tmpFiles[3].cacheTag) == _tmpFiles[3].cachedPath); |
||||
QVERIFY(cache.access(_tmpFiles[1].cacheTag) == _tmpFiles[1].cachedPath); |
||||
QVERIFY(cache.access(_tmpFiles[3].cacheTag) == _tmpFiles[3].cachedPath); |
||||
|
||||
insert(5); |
||||
|
||||
QVERIFY(cache.access(_tmpFiles[4].cacheTag) == ""); |
||||
QVERIFY(cache.access(_tmpFiles[1].cacheTag) == _tmpFiles[1].cachedPath); |
||||
QVERIFY(cache.access(_tmpFiles[3].cacheTag) == _tmpFiles[3].cachedPath); |
||||
QVERIFY(cache.access(_tmpFiles[5].cacheTag) == _tmpFiles[5].cachedPath); |
||||
|
||||
QVERIFY(cache.access(_tmpFiles[3].cacheTag) == _tmpFiles[3].cachedPath); |
||||
insert(6); |
||||
insert(7); |
||||
|
||||
QVERIFY(cache.access(_tmpFiles[4].cacheTag) == ""); |
||||
QVERIFY(cache.access(_tmpFiles[1].cacheTag) == ""); |
||||
QVERIFY(cache.access(_tmpFiles[5].cacheTag) == ""); |
||||
QVERIFY(cache.access(_tmpFiles[3].cacheTag) == _tmpFiles[3].cachedPath); |
||||
QVERIFY(cache.access(_tmpFiles[6].cacheTag) == _tmpFiles[6].cachedPath); |
||||
QVERIFY(cache.access(_tmpFiles[7].cacheTag) == _tmpFiles[7].cachedPath); |
||||
|
||||
_cleanup(); |
||||
} |
||||
|
||||
void ComponentInformationCacheTest::_multi_test() |
||||
{ |
||||
_setup(); |
||||
|
||||
auto insert = [&](ComponentInformationCache& cache, int idx) { |
||||
_tmpFiles[idx].cachedPath = cache.insert(_tmpFiles[idx].cacheTag, _tmpFiles[idx].path); |
||||
QVERIFY(!_tmpFiles[idx].cachedPath.isEmpty()); |
||||
}; |
||||
|
||||
{ |
||||
ComponentInformationCache cache(_cacheDir, 5); |
||||
for (int i = 0; i < 5; ++i) { |
||||
insert(cache, i); |
||||
QVERIFY(cache.access(_tmpFiles[i].cacheTag) == _tmpFiles[i].cachedPath); |
||||
} |
||||
QVERIFY(cache.access(_tmpFiles[1].cacheTag) == _tmpFiles[1].cachedPath); |
||||
} |
||||
{ |
||||
// reduce cache size and ensure oldest entries are evicted
|
||||
ComponentInformationCache cache(_cacheDir, 3); |
||||
QVERIFY(cache.access(_tmpFiles[0].cacheTag) == ""); |
||||
QVERIFY(cache.access(_tmpFiles[1].cacheTag) == _tmpFiles[1].cachedPath); |
||||
QVERIFY(cache.access(_tmpFiles[2].cacheTag) == ""); |
||||
QVERIFY(cache.access(_tmpFiles[3].cacheTag) == _tmpFiles[3].cachedPath); |
||||
QVERIFY(cache.access(_tmpFiles[4].cacheTag) == _tmpFiles[4].cachedPath); |
||||
|
||||
insert(cache, 10); |
||||
QVERIFY(cache.access(_tmpFiles[1].cacheTag) == ""); |
||||
QVERIFY(cache.access(_tmpFiles[10].cacheTag) == _tmpFiles[10].cachedPath); |
||||
} |
||||
|
||||
_cleanup(); |
||||
} |
||||
@ -0,0 +1,47 @@
@@ -0,0 +1,47 @@
|
||||
/****************************************************************************
|
||||
* |
||||
* (c) 2021 QGROUNDCONTROL PROJECT <http://www.qgroundcontrol.org>
|
||||
* |
||||
* QGroundControl is licensed according to the terms in the file |
||||
* COPYING.md in the root of the source code directory. |
||||
* |
||||
****************************************************************************/ |
||||
|
||||
|
||||
#pragma once |
||||
|
||||
#include "ComponentInformationCache.h" |
||||
|
||||
#include "UnitTest.h" |
||||
|
||||
#include <QString> |
||||
|
||||
class ComponentInformationCacheTest : public UnitTest |
||||
{ |
||||
Q_OBJECT |
||||
|
||||
public: |
||||
ComponentInformationCacheTest(); |
||||
virtual ~ComponentInformationCacheTest() = default; |
||||
|
||||
private slots: |
||||
void _basic_test(); |
||||
void _lru_test(); |
||||
void _multi_test(); |
||||
private: |
||||
void _setup(); |
||||
void _cleanup(); |
||||
|
||||
struct TmpFile { |
||||
QString path; |
||||
QString cacheTag; |
||||
QString content; |
||||
QString cachedPath; |
||||
}; |
||||
|
||||
QVector<TmpFile> _tmpFiles; |
||||
|
||||
QString _cacheDir; |
||||
QString _tmpFilesDir; |
||||
}; |
||||
|
||||
Loading…
Reference in new issue