Mercurial > dedupe
diff DBCache.hpp @ 16:06166d6c083b
Add configuration processing.
Cache DB values
Add a custom RBTree to save space.
Track multiple DB connections properly.
More testing.
Add ValueExistsException.
| author | Tom Fredrik Blenning Klaussen <bfg@blenning.no> |
|---|---|
| date | Tue, 28 Aug 2012 18:58:02 +0200 |
| parents | |
| children | 5d14d8c2c299 |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/DBCache.hpp Tue Aug 28 18:58:02 2012 +0200 @@ -0,0 +1,360 @@ +#ifndef DBCACHE_HPP +#define DBCACHE_HPP + +#include <QtSql/QSqlDatabase> +#include <QtSql/QSqlQuery> +#include <QtSql/QSqlError> +#include <QtSql/QSqlRecord> + +#include <QtCore/QDebug> +#include <QtCore/QStringList> +#include <cassert> +#include "OrderedPair.hpp" + +#include <boost/optional.hpp> + +#include "ThreadSafeLookup.hpp" +#include "UniqueString.hpp" + + +template<typename T> +struct SQLGenerator +{ +}; + +template<> +struct SQLGenerator<int> +{ + static QString fieldName(const QString &prefix = QString()) + { + return QString("%1_Int").arg(prefix); + } + + static QString createFields(const QString &prefix = QString()) + { + return QString("%1 INTEGER").arg(fieldName(prefix)); + } + + static QString valueString(const QString &prefix = QString()) + { + return QString(":") + fieldName(prefix); + } + + static void bindValue(QSqlQuery& query, int value, const QString& prefix = QString()) + { + query.bindValue(valueString(prefix), value); + } + + static void bindValues(QSqlQuery& query, const QVariantList& values, const QString& prefix = QString()) + { + query.bindValue(valueString(prefix), values); + } + + static void bindValues(QSqlQuery& query, const QList<int>& values, const QString& prefix = QString()) + { + QVariantList list; + foreach(int value, values) { + list << value; + } + bindValues(query, list, prefix); + } + + static boost::optional<int> extract(QSqlQuery& query, const QString& prefix = QString()) + { + int fieldNo = query.record().indexOf(fieldName(prefix)); + if (query.at() >= 0) + return query.value(fieldNo).toInt(); + else + return boost::optional<int>(); + } + +}; + +template<> +struct SQLGenerator<QString> +{ + static QString fieldName(const QString &prefix = QString()) + { + return QString("%1_QString").arg(prefix); + } + + static QString createFields(const QString &prefix = QString()) + { + return QString("%1 TEXT").arg(fieldName(prefix)); + } + + static QString valueString(const QString &prefix = QString()) + { + return QString(":") + fieldName(prefix); + } + + static QString restrict(const QString &prefix = QString()) + { + return QString("%1 = %2").arg(fieldName(prefix)).arg(valueString(prefix)); + } + + static void bindValue(QSqlQuery& query, const QString& value, const QString& prefix = QString()) + { + query.bindValue(valueString(prefix), value); + } + + static void bindValues(QSqlQuery& query, const QVariantList& value, const QString& prefix = QString()) + { + query.bindValue(valueString(prefix), value); + } + + static void bindValues(QSqlQuery& query, const QList<QString>& values, const QString& prefix = QString()) + { + QVariantList list; + foreach(const QString& value, values) { + list << value; + } + bindValues(query, list, prefix); + } + + static boost::optional<QString> extract(QSqlQuery& query, const QString& prefix = QString()) + { + int fieldNo = query.record().indexOf(fieldName(prefix)); + if (query.at() >= 0) + return query.value(fieldNo).toString(); + else + return boost::optional<QString>(); + } + + +}; + +template<> +struct SQLGenerator<UniqueString> +{ + static QString fieldName(const QString &prefix = QString()) + { + return QString("%1_QString").arg(prefix); + } + + static QString createFields(const QString &prefix = QString()) + { + return QString("%1 TEXT").arg(fieldName(prefix)); + } + + static QString valueString(const QString &prefix = QString()) + { + return QString(":") + fieldName(prefix); + } + + static QString restrict(const QString &prefix = QString()) + { + return QString("%1 = %2").arg(fieldName(prefix)).arg(valueString(prefix)); + } + + static void bindValue(QSqlQuery& query, const QString& value, const QString& prefix = QString()) + { + query.bindValue(valueString(prefix), value); + } + + static void bindValues(QSqlQuery& query, const QVariantList& value, const QString& prefix = QString()) + { + query.bindValue(valueString(prefix), value); + } + + static void bindValues(QSqlQuery& query, const QList<UniqueString>& values, const QString& prefix = QString()) + { + QVariantList list; + foreach(const QString& value, values) { + list << value; + } + bindValues(query, list, prefix); + } + + static boost::optional<QString> extract(QSqlQuery& query, const QString& prefix = QString()) + { + int fieldNo = query.record().indexOf(fieldName(prefix)); + if (query.at() >= 0) + return query.value(fieldNo).toString(); + else + return boost::optional<QString>(); + } + + +}; + +template<typename T> +struct SQLGenerator<OrderedPair<T> > +{ + static QString fieldName(const QString &prefix = QString()) + { + return SQLGenerator<T>::fieldName(prefix + "_1") + ", " + + SQLGenerator<T>::fieldName(prefix + "_2"); + } + + static QString createFields(const QString &prefix = QString()) + { + return SQLGenerator<T>::createFields(prefix + "_1") + ", " + + SQLGenerator<T>::createFields(prefix + "_2"); + } + + static QString restriction(const QString& prefix = QString()) + { + return SQLGenerator<T>::restrict(prefix + "_1") + " AND " + + SQLGenerator<T>::restrict(prefix + "_2"); + } + + static QString valueString(const QString &prefix = QString()) + { + return SQLGenerator<T>::valueString(prefix + "_1") + ", " + + SQLGenerator<T>::valueString(prefix + "_2"); + } + + static void bindValue(QSqlQuery& query, const OrderedPair<T>& value, const QString& prefix = QString()) + { + SQLGenerator<T>::bindValue(query, value.first, prefix + "_1"); + SQLGenerator<T>::bindValue(query, value.second, prefix + "_2"); + } + + static void bindValues(QSqlQuery& query, const QList<OrderedPair<T> >& values, const QString& prefix = QString()) + { + QList<T> first; + QList<T> second; + foreach(OrderedPair<T> value, values) { + first << value.first; + second << value.second; + } + SQLGenerator<T>::bindValues(query, first, prefix + "_1"); + SQLGenerator<T>::bindValues(query, second, prefix + "_2"); + } + + static boost::optional<OrderedPair<T> > extract(QSqlQuery& query, const QString& prefix = QString()) + { + if (query.at() >= 0) + return OrderedPair<T> (*SQLGenerator<T>::extract(query, prefix + "_1"), + *SQLGenerator<T>::extract(query, prefix + "_2")); + else + return boost::optional<OrderedPair<T> >(); + } + + +}; + +template<typename Key, typename Value, bool memoryMapped = false> +class DBCache +{ +private: + ThreadSafeLookup<Key, Value> memoryMap; + QList<Key> unsyncedKeys; + + void setup(const QSqlDatabase& db, const QString& dictName) + { + this->dictName = dictName; + if (!db.tables().contains(dictName)) { + QString keyFields = SQLGenerator<Key>::createFields("key"); + QString valueFields = SQLGenerator<Value>::createFields("value"); + QString createQuery = QString("CREATE TABLE %1(%2, %3);").arg(dictName).arg(keyFields).arg(valueFields); + QSqlQuery query(db); + query.exec(createQuery); + } + if (!db.tables().contains(dictName)) { + qDebug()<<"No database"; + exit(1); + } + if (memoryMapped) { + QString keyFields = SQLGenerator<Key>::fieldName("key"); + QString valueFields = SQLGenerator<Value>::fieldName("value"); + QString repopulateQuery = QString("SELECT %1, %2 FROM %3;").arg(keyFields).arg(valueFields).arg(dictName); + QSqlQuery query(db); + if (!query.exec(repopulateQuery)) { + qDebug() << query.lastError() << repopulateQuery; + } + while (query.next()) { + Key key = *SQLGenerator<Key>::extract(query, "key"); + Value value = *SQLGenerator<Value>::extract(query, "value"); + memoryMap.insert(key, value); + } + } + QString queryString = QString("INSERT into %1 (%2, %3) VALUES(%4, %5);") + .arg(dictName) + .arg(SQLGenerator<Key>::fieldName("key")) + .arg(SQLGenerator<Value>::fieldName("value")) + .arg(SQLGenerator<Key>::valueString("key")) + .arg(SQLGenerator<Value>::valueString("value")); + insertQuery = QSqlQuery(db); + insertQuery.prepare(queryString); + } + + void syncInsert() + { + SQLGenerator<Key>::bindValues(insertQuery, unsyncedKeys, "key"); + QList<Value> values; + foreach(Key key, unsyncedKeys) { + values << *memoryMap.value(key); + } + SQLGenerator<Value>::bindValues(insertQuery, values, "value"); + if (!insertQuery.exec()) { + qDebug() << insertQuery.lastError() << insertQuery.lastQuery(); + } + insertQuery.finish(); + unsyncedKeys.clear(); + } + +public: + DBCache(const QString& dbName, const QString& dictName) + { + db = QSqlDatabase::addDatabase("QSQLITE", "dictName"); + db.setDatabaseName(dbName); + bool ok = db.open(); + assert(ok); + setup(db, dictName); + } + + DBCache(const QSqlDatabase& db, const QString& dictName) + { + setup(db, dictName); + } + + boost::optional<Value> value(const Key& key) const + { + if (memoryMapped) { + return memoryMap.value(key); + } + else { + QString queryString = QString("SELECT %1 FROM %2 WHERE %3") + .arg(SQLGenerator<Value>::fieldName("value")) + .arg(dictName) + .arg(SQLGenerator<Key>::restriction("key")); + QSqlQuery query(db); + query.prepare(queryString); + SQLGenerator<Key>::bindValue(query, key, "key"); + if (!query.exec()) { + qDebug() << query.lastError() << queryString; + } + query.next(); + return SQLGenerator<Value>::extract(query, "value"); + } + } + + void insert(const Key& key, const Value& value) + { + if (memoryMapped) { + memoryMap.insert(key, value); + unsyncedKeys.append(key); + if (unsyncedKeys.size() > 1024) { + syncInsert(); + } + } + else { + SQLGenerator<Key>::bindValue(insertQuery, key, "key"); + SQLGenerator<Value>::bindValue(insertQuery, value, "value"); + if (!insertQuery.exec()) { + qDebug() << insertQuery.lastError() << insertQuery.lastQuery(); + } + insertQuery.finish(); + } + + } + +private: + QSqlDatabase db; + QString dictName; + QSqlQuery insertQuery; +}; + +#endif //DBCACHE_HPP
