Mercurial > dedupe
view DBCache.hpp @ 58:7b7e84356b39
Introduce API for fine tuning inserts.
Reduce the amount of rebuilds.
| author | Tom Fredrik Blenning Klaussen <bfg@blenning.no> |
|---|---|
| date | Fri, 14 Sep 2012 00:04:24 +0200 |
| parents | f711ddb56ae7 |
| children | 74be5a2f49db |
line wrap: on
line source
#ifndef DBCACHE_HPP #define DBCACHE_HPP #include "OrderedPair.hpp" #include "ThreadSafeLookup.hpp" #include "UniqueString.hpp" #include "Exception/IOException.hpp" #include "Exception/SQLException.hpp" #include <QtCore/QStringList> #include <QtCore/QVariant> #include <QtSql/QSqlDatabase> #include <QtSql/QSqlError> #include <QtSql/QSqlQuery> #include <QtSql/QSqlRecord> #include <boost/optional.hpp> #include <QtCore/QString> template<typename Key, typename Value> struct InsertRegulator { static void start() {} static void finish() {} static void next() {} }; template<typename T> struct SQLGenerator { }; template<> struct SQLGenerator<int> { static QString fieldName(const QString &prefix = QString()) { return QString("%1_Int").arg(prefix); } static QString createFields(const QString &prefix = QString()) { return QString("%1 INTEGER").arg(fieldName(prefix)); } static QString valueString(const QString &prefix = QString()) { return QString(":") + fieldName(prefix); } static void bindValue(QSqlQuery& query, int value, const QString& prefix = QString()) { query.bindValue(valueString(prefix), value); } static void bindValues(QSqlQuery& query, const QVariantList& values, const QString& prefix = QString()) { query.bindValue(valueString(prefix), values); } static void bindValues(QSqlQuery& query, const QList<int>& values, const QString& prefix = QString()) { QVariantList list; foreach(int value, values) { list << value; } bindValues(query, list, prefix); } static boost::optional<int> extract(QSqlQuery& query, const QString& prefix = QString()) { int fieldNo = query.record().indexOf(fieldName(prefix)); if (query.at() >= 0) return query.value(fieldNo).toInt(); else return boost::optional<int>(); } }; template<> struct SQLGenerator<QString> { static QString fieldName(const QString &prefix = QString()) { return QString("%1_QString").arg(prefix); } static QString createFields(const QString &prefix = QString()) { return QString("%1 TEXT").arg(fieldName(prefix)); } static QString valueString(const QString &prefix = QString()) { return QString(":") + fieldName(prefix); } static QString restrict(const QString &prefix = QString()) { return QString("%1 = %2").arg(fieldName(prefix)).arg(valueString(prefix)); } static void bindValue(QSqlQuery& query, const QString& value, const QString& prefix = QString()) { query.bindValue(valueString(prefix), value); } static void bindValues(QSqlQuery& query, const QVariantList& value, const QString& prefix = QString()) { query.bindValue(valueString(prefix), value); } static void bindValues(QSqlQuery& query, const QList<QString>& values, const QString& prefix = QString()) { QVariantList list; foreach(const QString& value, values) { list << value; } bindValues(query, list, prefix); } static boost::optional<QString> extract(QSqlQuery& query, const QString& prefix = QString()) { int fieldNo = query.record().indexOf(fieldName(prefix)); if (query.at() >= 0) return query.value(fieldNo).toString(); else return boost::optional<QString>(); } }; template<> struct SQLGenerator<UniqueString> { static QString fieldName(const QString &prefix = QString()) { return QString("%1_QString").arg(prefix); } static QString createFields(const QString &prefix = QString()) { return QString("%1 TEXT").arg(fieldName(prefix)); } static QString valueString(const QString &prefix = QString()) { return QString(":") + fieldName(prefix); } static QString restrict(const QString &prefix = QString()) { return QString("%1 = %2").arg(fieldName(prefix)).arg(valueString(prefix)); } static void bindValue(QSqlQuery& query, const QString& value, const QString& prefix = QString()) { query.bindValue(valueString(prefix), value); } static void bindValues(QSqlQuery& query, const QVariantList& values, const QString& prefix = QString()) { query.bindValue(valueString(prefix), values); } static void bindValues(QSqlQuery& query, const QList<UniqueString>& values, const QString& prefix = QString()) { QVariantList list; foreach(const UniqueString& value, values) { list << static_cast<const QString>(value); } bindValues(query, list, prefix); } static boost::optional<QString> extract(QSqlQuery& query, const QString& prefix = QString()) { int fieldNo = query.record().indexOf(fieldName(prefix)); if (query.at() >= 0) return query.value(fieldNo).toString(); else return boost::optional<QString>(); } }; template<typename T> struct SQLGenerator<OrderedPair<T> > { static QString fieldName(const QString &prefix = QString()) { return SQLGenerator<T>::fieldName(prefix + "_1") + ", " + SQLGenerator<T>::fieldName(prefix + "_2"); } static QString createFields(const QString &prefix = QString()) { return SQLGenerator<T>::createFields(prefix + "_1") + ", " + SQLGenerator<T>::createFields(prefix + "_2"); } static QString restriction(const QString& prefix = QString()) { return SQLGenerator<T>::restrict(prefix + "_1") + " AND " + SQLGenerator<T>::restrict(prefix + "_2"); } static QString valueString(const QString &prefix = QString()) { return SQLGenerator<T>::valueString(prefix + "_1") + ", " + SQLGenerator<T>::valueString(prefix + "_2"); } static void bindValue(QSqlQuery& query, const OrderedPair<T>& value, const QString& prefix = QString()) { SQLGenerator<T>::bindValue(query, value.first, prefix + "_1"); SQLGenerator<T>::bindValue(query, value.second, prefix + "_2"); } static void bindValues(QSqlQuery& query, const QList<OrderedPair<T> >& values, const QString& prefix = QString()) { QList<T> first; QList<T> second; foreach(OrderedPair<T> value, values) { first << value.first; second << value.second; } SQLGenerator<T>::bindValues(query, first, prefix + "_1"); SQLGenerator<T>::bindValues(query, second, prefix + "_2"); } static boost::optional<OrderedPair<T> > extract(QSqlQuery& query, const QString& prefix = QString()) { if (query.at() >= 0) return OrderedPair<T> (*SQLGenerator<T>::extract(query, prefix + "_1"), *SQLGenerator<T>::extract(query, prefix + "_2")); else return boost::optional<OrderedPair<T> >(); } }; template<typename Key, typename Value, bool memoryMapped = false> class DBCache { private: ThreadSafeLookup<Key, Value> memoryMap; QList<Key> unsyncedKeys; void setup(const QSqlDatabase& db, const QString& dictName) { this->dictName = dictName; if (!db.tables().contains(dictName)) { QString keyFields = SQLGenerator<Key>::createFields("key"); QString valueFields = SQLGenerator<Value>::createFields("value"); QString createQuery = QString("CREATE TABLE %1(%2, %3);").arg(dictName).arg(keyFields).arg(valueFields); QSqlQuery query(db); query.exec(createQuery); } if (!db.tables().contains(dictName)) { throw SQLException("No databaase"); } if (memoryMapped) { QString keyFields = SQLGenerator<Key>::fieldName("key"); QString valueFields = SQLGenerator<Value>::fieldName("value"); QString repopulateQuery = QString("SELECT %1, %2 FROM %3;").arg(keyFields).arg(valueFields).arg(dictName); QSqlQuery query(db); if (!query.exec(repopulateQuery)) { throw SQLException(query); } InsertRegulator<Key, Value> regulator; regulator.start(); while (query.next()) { Key key = *SQLGenerator<Key>::extract(query, "key"); Value value = *SQLGenerator<Value>::extract(query, "value"); memoryMap.insert(key, value); regulator.next(); } regulator.finish(); } QString queryString = QString("INSERT into %1 (%2, %3) VALUES(%4, %5);") .arg(dictName) .arg(SQLGenerator<Key>::fieldName("key")) .arg(SQLGenerator<Value>::fieldName("value")) .arg(SQLGenerator<Key>::valueString("key")) .arg(SQLGenerator<Value>::valueString("value")); insertQuery = QSqlQuery(db); insertQuery.prepare(queryString); } void syncInsert() { SQLGenerator<Key>::bindValues(insertQuery, unsyncedKeys, "key"); QList<Value> values; foreach(Key key, unsyncedKeys) { values << *memoryMap.value(key); } SQLGenerator<Value>::bindValues(insertQuery, values, "value"); if (!insertQuery.execBatch()) { throw SQLException(insertQuery); } insertQuery.finish(); unsyncedKeys.clear(); } public: DBCache(const QString& dbName, const QString& dictName) { db = QSqlDatabase::addDatabase("QSQLITE", "dictName"); db.setDatabaseName(dbName); if (!db.open()) throw IOException(QString("Unable to open SQLite database with path '%1'").arg(dictName)); setup(db, dictName); } DBCache(const QSqlDatabase& db, const QString& dictName) { setup(db, dictName); } boost::optional<Value> value(const Key& key) const { if (memoryMapped) { return memoryMap.value(key); } else { QString queryString = QString("SELECT %1 FROM %2 WHERE %3") .arg(SQLGenerator<Value>::fieldName("value")) .arg(dictName) .arg(SQLGenerator<Key>::restriction("key")); QSqlQuery query(db); query.prepare(queryString); SQLGenerator<Key>::bindValue(query, key, "key"); if (!query.exec()) { throw SQLException(query); } query.next(); return SQLGenerator<Value>::extract(query, "value"); } } void insert(const Key& key, const Value& value) { if (memoryMapped) { memoryMap.insert(key, value); unsyncedKeys.append(key); if (unsyncedKeys.size() >= 1024) { syncInsert(); } } else { SQLGenerator<Key>::bindValue(insertQuery, key, "key"); SQLGenerator<Value>::bindValue(insertQuery, value, "value"); if (!insertQuery.exec()) { throw SQLException(insertQuery); } insertQuery.finish(); } } private: QSqlDatabase db; QString dictName; QSqlQuery insertQuery; }; #endif //DBCACHE_HPP
