view DBCache.hpp @ 58:7b7e84356b39

Introduce API for fine tuning inserts. Reduce the amount of rebuilds.
author Tom Fredrik Blenning Klaussen <bfg@blenning.no>
date Fri, 14 Sep 2012 00:04:24 +0200
parents f711ddb56ae7
children 74be5a2f49db
line wrap: on
line source

#ifndef DBCACHE_HPP
#define DBCACHE_HPP

#include "OrderedPair.hpp"
#include "ThreadSafeLookup.hpp"
#include "UniqueString.hpp"

#include "Exception/IOException.hpp"
#include "Exception/SQLException.hpp"

#include <QtCore/QStringList>
#include <QtCore/QVariant>

#include <QtSql/QSqlDatabase>
#include <QtSql/QSqlError>
#include <QtSql/QSqlQuery>
#include <QtSql/QSqlRecord>

#include <boost/optional.hpp>

#include <QtCore/QString>

template<typename Key, typename Value>
struct InsertRegulator {
  static void start() {}
  static void finish() {}
  static void next() {}
};

template<typename T>
struct SQLGenerator
{
};

template<>
struct SQLGenerator<int>
{
  static QString fieldName(const QString &prefix = QString())
  {
    return QString("%1_Int").arg(prefix);
  }

  static QString createFields(const QString &prefix = QString())
  {
    return QString("%1 INTEGER").arg(fieldName(prefix));
  }

  static QString valueString(const QString &prefix = QString())
  {
    return QString(":") + fieldName(prefix);
  }

  static void bindValue(QSqlQuery& query, int value, const QString& prefix = QString())
  {
    query.bindValue(valueString(prefix), value);
  }

  static void bindValues(QSqlQuery& query, const QVariantList& values, const QString& prefix = QString())
  {
    query.bindValue(valueString(prefix), values);
  }

  static void bindValues(QSqlQuery& query, const QList<int>& values, const QString& prefix = QString())
  {
    QVariantList list;
    foreach(int value, values) {
      list << value;
    }
    bindValues(query, list, prefix);
  }

  static boost::optional<int> extract(QSqlQuery& query, const QString& prefix = QString())
  {
    int fieldNo = query.record().indexOf(fieldName(prefix));
    if (query.at() >= 0)
      return query.value(fieldNo).toInt();
    else
      return boost::optional<int>();
  }

};

template<>
struct SQLGenerator<QString>
{
  static QString fieldName(const QString &prefix = QString())
  {
    return QString("%1_QString").arg(prefix);
  }

  static QString createFields(const QString &prefix = QString())
  {
    return QString("%1 TEXT").arg(fieldName(prefix));
  }

  static QString valueString(const QString &prefix = QString())
  {
    return QString(":") + fieldName(prefix);
  }

  static QString restrict(const QString &prefix = QString())
  {
    return QString("%1 = %2").arg(fieldName(prefix)).arg(valueString(prefix));
  }

  static void bindValue(QSqlQuery& query, const QString& value, const QString& prefix = QString())
  {
    query.bindValue(valueString(prefix), value);
  }

  static void bindValues(QSqlQuery& query, const QVariantList& value, const QString& prefix = QString())
  {
    query.bindValue(valueString(prefix), value);
  }

  static void bindValues(QSqlQuery& query, const QList<QString>& values, const QString& prefix = QString())
  {
    QVariantList list;
    foreach(const QString& value, values) {
      list << value;
    }
    bindValues(query, list, prefix);
  }

  static boost::optional<QString> extract(QSqlQuery& query, const QString& prefix = QString())
  {
    int fieldNo = query.record().indexOf(fieldName(prefix));
    if (query.at() >= 0)
      return query.value(fieldNo).toString();
    else
      return boost::optional<QString>();
  }


};

template<>
struct SQLGenerator<UniqueString>
{
  static QString fieldName(const QString &prefix = QString())
  {
    return QString("%1_QString").arg(prefix);
  }

  static QString createFields(const QString &prefix = QString())
  {
    return QString("%1 TEXT").arg(fieldName(prefix));
  }

  static QString valueString(const QString &prefix = QString())
  {
    return QString(":") + fieldName(prefix);
  }

  static QString restrict(const QString &prefix = QString())
  {
    return QString("%1 = %2").arg(fieldName(prefix)).arg(valueString(prefix));
  }

  static void bindValue(QSqlQuery& query, const QString& value, const QString& prefix = QString())
  {
    query.bindValue(valueString(prefix), value);
  }

  static void bindValues(QSqlQuery& query, const QVariantList& values, const QString& prefix = QString())
  {
    query.bindValue(valueString(prefix), values);
  }

  static void bindValues(QSqlQuery& query, const QList<UniqueString>& values, const QString& prefix = QString())
  {
    QVariantList list;
    foreach(const UniqueString& value, values) {
      list << static_cast<const QString>(value);
    }
    bindValues(query, list, prefix);
  }

  static boost::optional<QString> extract(QSqlQuery& query, const QString& prefix = QString())
  {
    int fieldNo = query.record().indexOf(fieldName(prefix));
    if (query.at() >= 0)
      return query.value(fieldNo).toString();
    else
      return boost::optional<QString>();
  }


};

template<typename T>
struct SQLGenerator<OrderedPair<T> >
{
  static QString fieldName(const QString &prefix = QString())
  {
    return SQLGenerator<T>::fieldName(prefix + "_1") + ", " +
      SQLGenerator<T>::fieldName(prefix + "_2");
  }

  static QString createFields(const QString &prefix = QString())
  {
    return SQLGenerator<T>::createFields(prefix + "_1") + ", " +
      SQLGenerator<T>::createFields(prefix + "_2");
  }

  static QString restriction(const QString& prefix = QString())
  {
    return SQLGenerator<T>::restrict(prefix + "_1") + " AND " +
      SQLGenerator<T>::restrict(prefix + "_2");
  }

  static QString valueString(const QString &prefix = QString())
  {
    return SQLGenerator<T>::valueString(prefix + "_1") + ", " +
      SQLGenerator<T>::valueString(prefix + "_2");
  }

  static void bindValue(QSqlQuery& query, const OrderedPair<T>& value, const QString& prefix = QString())
  {
    SQLGenerator<T>::bindValue(query, value.first, prefix + "_1");
    SQLGenerator<T>::bindValue(query, value.second, prefix + "_2");
  }

  static void bindValues(QSqlQuery& query, const QList<OrderedPair<T> >& values, const QString& prefix = QString())
  {
    QList<T> first;
    QList<T> second;
    foreach(OrderedPair<T> value, values) {
      first << value.first;
      second << value.second;
    }
    SQLGenerator<T>::bindValues(query, first, prefix + "_1");
    SQLGenerator<T>::bindValues(query, second, prefix + "_2");
  }

  static boost::optional<OrderedPair<T> > extract(QSqlQuery& query, const QString& prefix = QString())
  {
    if (query.at() >= 0)
      return OrderedPair<T> (*SQLGenerator<T>::extract(query, prefix + "_1"),
			     *SQLGenerator<T>::extract(query, prefix + "_2"));
    else
      return boost::optional<OrderedPair<T> >();
  }


};

template<typename Key, typename Value, bool memoryMapped = false>
class DBCache
{
private:
  ThreadSafeLookup<Key, Value> memoryMap;
  QList<Key> unsyncedKeys;

  void setup(const QSqlDatabase& db, const QString& dictName)
  {
    this->dictName = dictName;
    if (!db.tables().contains(dictName)) {
      QString keyFields = SQLGenerator<Key>::createFields("key");
      QString valueFields = SQLGenerator<Value>::createFields("value");
      QString createQuery = QString("CREATE TABLE %1(%2, %3);").arg(dictName).arg(keyFields).arg(valueFields);
      QSqlQuery query(db);
      query.exec(createQuery);
    }
    if (!db.tables().contains(dictName)) {
      throw SQLException("No databaase");
    }
    if (memoryMapped) {
      QString keyFields = SQLGenerator<Key>::fieldName("key");
      QString valueFields = SQLGenerator<Value>::fieldName("value");
      QString repopulateQuery = QString("SELECT %1, %2 FROM %3;").arg(keyFields).arg(valueFields).arg(dictName);
      QSqlQuery query(db);
      if (!query.exec(repopulateQuery)) {
	throw SQLException(query);
      }
      InsertRegulator<Key, Value> regulator;
      regulator.start();
      while (query.next()) {
	Key key = *SQLGenerator<Key>::extract(query, "key");
	Value value = *SQLGenerator<Value>::extract(query, "value");
	memoryMap.insert(key, value);
	regulator.next();
      }
      regulator.finish();
    }
    QString queryString = QString("INSERT into %1 (%2, %3) VALUES(%4, %5);")
      .arg(dictName)
      .arg(SQLGenerator<Key>::fieldName("key"))
      .arg(SQLGenerator<Value>::fieldName("value"))
      .arg(SQLGenerator<Key>::valueString("key"))
      .arg(SQLGenerator<Value>::valueString("value"));
    insertQuery = QSqlQuery(db);
    insertQuery.prepare(queryString);
  }

  void syncInsert()
  {
    SQLGenerator<Key>::bindValues(insertQuery, unsyncedKeys, "key");
    QList<Value> values;
    foreach(Key key, unsyncedKeys) {
      values << *memoryMap.value(key);
    }
    SQLGenerator<Value>::bindValues(insertQuery, values, "value");
    if (!insertQuery.execBatch()) {
      throw SQLException(insertQuery);
    }
    insertQuery.finish();
    unsyncedKeys.clear();
  }

public:
  DBCache(const QString& dbName, const QString& dictName)
  {
    db = QSqlDatabase::addDatabase("QSQLITE", "dictName");
    db.setDatabaseName(dbName);
    if (!db.open())
      throw IOException(QString("Unable to open SQLite database with path '%1'").arg(dictName));
    setup(db, dictName);
  }

  DBCache(const QSqlDatabase& db, const QString& dictName)
  {
    setup(db, dictName);
  }

  boost::optional<Value> value(const Key& key) const
  {
    if (memoryMapped) {
      return memoryMap.value(key);
    }
    else {
      QString queryString = QString("SELECT %1 FROM %2 WHERE %3")
	.arg(SQLGenerator<Value>::fieldName("value"))
	.arg(dictName)
	.arg(SQLGenerator<Key>::restriction("key"));
      QSqlQuery query(db);
      query.prepare(queryString);
      SQLGenerator<Key>::bindValue(query, key, "key");
      if (!query.exec()) {
	throw SQLException(query);
      }
      query.next();
      return SQLGenerator<Value>::extract(query, "value");
    }
  }

  void insert(const Key& key, const Value& value)
  {
    if (memoryMapped) {
      memoryMap.insert(key, value);
      unsyncedKeys.append(key);
      if (unsyncedKeys.size() >= 1024) {
	syncInsert();
      }
    }
    else {
      SQLGenerator<Key>::bindValue(insertQuery, key, "key");
      SQLGenerator<Value>::bindValue(insertQuery, value, "value");
      if (!insertQuery.exec()) {
	throw SQLException(insertQuery);
      }
      insertQuery.finish();
    }

  }

private:
  QSqlDatabase db;
  QString dictName;
  QSqlQuery insertQuery;
};

#endif //DBCACHE_HPP