view SqliteDBLink.cpp @ 101:6c6f3a5f96ea

Better support for batch operations.
author Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
date Thu, 13 Feb 2014 15:19:37 +0100
parents c7da835ea912
children 6bc013d5788b
line wrap: on
line source



#include "SqliteDBLink.hpp"

#include <QtCore/QStringList>
#include <QtCore/QVariant>


#include <QtSql/QSqlQuery>
#include <QtSql/QSqlRecord>

#include "Exception/SQLException.hpp"
#include "Exception/IOException.hpp"

#include <cassert>

const QString SqliteDBLink::connectionName("SqliteDBLink");

#include <QtCore/QDebug>


SqliteDBLink::SqliteDBLink(const QString& dbPath)
{
  if (!QSqlDatabase::contains(connectionName))
    db = QSqlDatabase::addDatabase("QSQLITE", connectionName);
  else {
    db = QSqlDatabase::database(connectionName);
  }
  db.setDatabaseName(dbPath);
  if (!db.open())
    throw
      IOException(QString("Unable to open SQLite database with path '%1'")
		  .arg(dbPath));
  QSqlQuery query(db);
  if (!query.exec(QString("SELECT * FROM files;"))) {
    query.exec("CREATE TABLE files(path VARCHAR PRIMARY KEY ASC,"
	       " size INTEGER, mtime TEXT, checksum TEXT);");
  }
  if (!query.exec(QString("SELECT * FROM files;"))) {
    throw SQLException("No database");
  }

  if (!query.exec(QString("SELECT * FROM human_files;"))) {
    query.exec("CREATE VIEW human_files AS SELECT path, size, date(mtime) as mdate, time(mtime) as mtime, hex(checksum) as checksum FROM files;");
  }
  if (!query.exec(QString("SELECT * FROM human_files;"))) {
    throw SQLException("No view");
  }


  preparedSizePrefixQuery = new QSqlQuery(db);
  preparedSizePrefixQuery->prepare("SELECT * FROM files WHERE path LIKE :prefix AND size = :size");

  preparedSizeQuery = new QSqlQuery(db);
  preparedSizeQuery->prepare("SELECT * FROM files WHERE size = :size");

  preparedTryAddQuery = new QSqlQuery(db);
  preparedTryAddQuery->prepare("INSERT INTO files (path, size, mtime, checksum)"
			       " VALUES (:path, :size, :mtime, :checksum)");

  preparedUpdateQuery = new QSqlQuery(db);
  preparedUpdateQuery->prepare(
			 "UPDATE files "
			 "SET size=:size, mtime=:mtime, checksum=:checksum "
			 "WHERE path=:path");

  preparedDeleteQuery = new QSqlQuery(db);
  preparedDeleteQuery->prepare("DELETE FROM files WHERE path = :path");
}

SqliteDBLink::~SqliteDBLink()
{
  delete preparedSizePrefixQuery;
  delete preparedSizeQuery;
  delete preparedTryAddQuery;
  delete preparedUpdateQuery;
  db.close();
}

bool SqliteDBLink::exists(const QString& path)
{
  QSqlQuery query(db);
  query.prepare("SELECT path FROM files WHERE path = :path;");
  query.bindValue(":path", path);
  if (!query.exec()) {
    throw SQLException(query);
  }
  return query.last();
}

FileDBLink::DBStatus SqliteDBLink::existsWithMtime(const QString& path,
						   const QDateTime& mtime)
{
  QSqlQuery query(db);
  query.prepare("SELECT mtime FROM files WHERE path = :path;");
  query.bindValue(":path", path);
  if (!query.exec()) {
    throw SQLException(query);
  }
  if (query.next()) {
    int dateIndex = query.record().indexOf("mtime");
    QDateTime mtimeEntry = query.value(dateIndex).toDateTime();
    if (mtimeEntry == mtime)
      return SAME;
    return MTIME_DIFFERENT;
  }
  return NONE;
}

void SqliteDBLink::addFile(const QString& path, quint64 size,
			   const QDateTime& dtime, const QByteArray& hash,
			   bool lazy)
{
  addFile(DBInfo(path, size, dtime, hash), lazy);
}

bool SqliteDBLink::tryAddFile(const DBInfo& dbinfo)
{
  if (exists(dbinfo.path()))
    return false;
  preparedTryAddQuery->bindValue(":path", dbinfo.path());
  preparedTryAddQuery->bindValue(":size", dbinfo.size());
  preparedTryAddQuery->bindValue(":mtime", dbinfo.mtime());
  preparedTryAddQuery->bindValue(":checksum", dbinfo.checksum());
  if (!preparedTryAddQuery->exec()) {
    throw SQLException(*preparedTryAddQuery);
  }
  return true;
}

void SqliteDBLink::updateFile(const QString& path,
			      quint64 size, const QDateTime& dtime,
			      const QByteArray& hash, bool lazy)
{
  updateFile(DBInfo(path, size, dtime, hash), lazy);
}

void SqliteDBLink::updateFile(const DBInfo& dbinfo, bool lazy)
{
  if (lazy) {
    operations.push_back(new UpdateOperation(dbinfo));
  }
  else {
    preparedUpdateQuery->bindValue(":path", dbinfo.path());
    preparedUpdateQuery->bindValue(":size", dbinfo.size());
    preparedUpdateQuery->bindValue(":mtime", dbinfo.mtime());
    preparedUpdateQuery->bindValue(":checksum", dbinfo.checksum());
    if (!preparedUpdateQuery->exec()) {
      throw SQLException(*preparedUpdateQuery);
    }
  }
}

void SqliteDBLink::addFile(const DBInfo& dbinfo, bool lazy)
{
  if (lazy) {
    operations.push_back(new AddOperation(dbinfo));
  }
  else {
    if (!tryAddFile(dbinfo)) {
      abort(); //Should throw exception
    }
  }
}


QStringList SqliteDBLink::toStringList()
{
  abort();
  QStringList list;
  /*
  foreach(dbinf_ptr_t info, entries) {
    list << info->serialize();
  }
  */
  return list;
}

const QList<FileDBLink::dbinf_ptr_t >
SqliteDBLink::values(const QString& prefix) const
{
  QList<FileDBLink::dbinf_ptr_t > values;

  QSqlQuery query(db);

  if (prefix.size() > 0) {
    query.prepare("SELECT * FROM files WHERE path LIKE :prefix");
    query.bindValue(":prefix", QString("%1%").arg(prefix));
  }
  else {
    query.prepare("SELECT * FROM files");
  }

  if (!query.exec()) {
    throw SQLException(query);
  }

  int pathIndex = query.record().indexOf("path");
  int sizeIndex = query.record().indexOf("size");
  int dateIndex = query.record().indexOf("mtime");
  int checksumIndex = query.record().indexOf("checksum");
  while (query.next()) {
    QString path = query.value(pathIndex).toString();
    quint64 size = query.value(sizeIndex).toInt();
    QDateTime mtime = query.value(dateIndex).toDateTime();
    QByteArray checksum = query.value(checksumIndex).toByteArray();

    values <<
      FileDBLink::dbinf_ptr_t(new FileDBLink::DBInfo(path,
						     size, mtime, checksum));
  }

  return values;
}

void SqliteDBLink::deleteFileFromDB(const QString& path, bool lazy)
{
  if (lazy) {
    operations.push_back(new DeleteOperation(path));
  }
  else {
    QSqlQuery query(db);
    query.prepare("DELETE FROM files WHERE path = :path");
    query.bindValue(":path", path);
    if (!query.exec()) {
      throw SQLException(query);
    }
  }
}


void SqliteDBLink::keepOnlyFromPrefix(const QString& prefix,
				      const QStringList& files,
				      bool lazy)
{
  QStringList list;
  foreach(dbinf_ptr_t info, values(prefix)) {
    if (!files.contains(info->path())) {
      list << info->path();
    }
  }
  foreach(QString path, list) {
    deleteFileFromDB(path, lazy);
  }
}

const QList<FileDBLink::dbinf_ptr_t>
SqliteDBLink::filesWithSize(quint64 size, const QString& prefix) const
{
  QList<dbinf_ptr_t > values;

  QSqlQuery & query = (prefix.size() > 0) ? *preparedSizePrefixQuery : *preparedSizeQuery;

  if (prefix.size() > 0) {
    query.bindValue(":prefix", QString("%1%").arg(prefix));
  }
  query.bindValue(":size", QString("%1%").arg(size));


  if (!query.exec()) {
    throw SQLException(query);
  }

  int pathIndex = query.record().indexOf("path");
  int sizeIndex = query.record().indexOf("size");
  int dateIndex = query.record().indexOf("mtime");
  int checksumIndex = query.record().indexOf("checksum");
  while (query.next()) {
    QString path = query.value(pathIndex).toString();
    quint64 size = query.value(sizeIndex).toInt();
    QDateTime mtime = query.value(dateIndex).toDateTime();
    QByteArray checksum = query.value(checksumIndex).toByteArray();

    values <<
      FileDBLink::dbinf_ptr_t(new FileDBLink::DBInfo(path,
						     size, mtime, checksum));
  }

  return values;
}

void SqliteDBLink::executeOperation(QVariantList& paths,
				    QVariantList& sizes,
				    QVariantList& mtimes,
				    QVariantList& checksums,
				    OperationType operation)
{
  assert(sizes.size() == 0 || paths.size() == sizes.size());
  assert(mtimes.size() == 0 || paths.size() == mtimes.size());
  assert(checksums.size() == 0 || paths.size() == checksums.size());
  QSqlQuery* query;
  bool infoQuery;
  switch (operation) {
  case Add:
    query = preparedTryAddQuery;
    infoQuery = true;
    break;
  case Update:
    query = preparedUpdateQuery;
    infoQuery = true;
    break;
  case Delete:
    query = preparedDeleteQuery;
    infoQuery = false;
    break;
  case None:
    assert(paths.size() == 0);
    return;
  }
  query->bindValue("path", paths);
  if (infoQuery) {
    query->bindValue("size", sizes);
    query->bindValue("mtime", mtimes);
    query->bindValue("checksum", checksums);
  }

  if (!query->execBatch())
    throw SQLException(*query);

  paths.clear();
  sizes.clear();
  mtimes.clear();
  checksums.clear();
}

const char* SqliteDBLink::typeString(OperationType type)
{
  switch(type) {
  case Add:
    return "Add";
  case Update:
    return "Update";
  case Delete:
    return "Delete";
  case None:
    return "None";
  }

  return "";
}


bool SqliteDBLink::commit(const QString& prefix)
{
  OperationType last = None;
  QVariantList paths, sizes, mtimes, hashes;

  foreach(const Operation* operation, operations) {
    if (operation->type() != last) {
      executeOperation(paths, sizes, mtimes, hashes, last);
    }

    switch (operation->type()) {
    case Add:
    case Update: {
      const InfoOperation* iOperation = dynamic_cast<const InfoOperation*>(operation);
      paths.push_back(iOperation->info().path());
      sizes.push_back(iOperation->info().size());
      mtimes.push_back(iOperation->info().mtime());
      hashes.push_back(iOperation->info().checksum());
      break;
    }
    case Delete: {
      const DeleteOperation* dOperation = dynamic_cast<const DeleteOperation*>(operation);
      paths.push_back(dOperation->path());
    }
    case None:
      break;
    }
    last = operation->type();
  }
  if (last != None) {
    qDebug() << "Execute Operation" << typeString(last);
    qDebug() << paths;
    executeOperation(paths, sizes, mtimes, hashes, last);
    qDebug() << "Execute Operation Done";
  }

  QSqlQuery whatToUpdate(db);
  QString whatToUpdateQuery =
    "SELECT path FROM files WHERE checksum is NULL AND path in "
    "(SELECT path FROM files WHERE size <> 0 %1 "
    "GROUP BY size HAVING count(*) > 1) ORDER BY size";
  if (prefix.isEmpty()) {
    whatToUpdateQuery = whatToUpdateQuery.arg("");
  }
  else {
    whatToUpdateQuery = whatToUpdateQuery.arg("AND path LIKE :prefix");
  }
  whatToUpdate.prepare(whatToUpdateQuery);
  if (!prefix.isEmpty()) {
    whatToUpdate.bindValue("prefix", QString("%1%").arg(prefix));
  }

  qDebug() << "Before whatToUpdate";
  if (!whatToUpdate.exec()) {
    throw SQLException(whatToUpdate);
  }
  qDebug() << "After whatToUpdate";

  int pathIndex = whatToUpdate.record().indexOf("path");
  QStringList updatePaths;
  while (whatToUpdate.next()) {
    updatePaths << whatToUpdate.value(pathIndex).toString();
  }
  int n = 0;
  int max = updatePaths.size();
  emit progressUpdate(0, max);
  QSqlQuery updateChecksum(db);
  updateChecksum.prepare("UPDATE files "
			 "SET checksum=:checksum "
			 "WHERE path=:path");

  foreach (const QString& path, updatePaths) {
    qDebug() << path;
    QByteArray ohash = computeHash(path);
    emit progressUpdate(++n, max);
    updateChecksum.bindValue("checksum", ohash);
    updateChecksum.bindValue("path", path);
    if (!updateChecksum.exec())
      throw SQLException(updateChecksum);

  }
  return true;
}