view SqliteDBLink.cpp @ 115:404795616b1e default tip

Added a lot of common files to ignore
author Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
date Sat, 25 Mar 2017 17:43:57 +0100
parents d4e337567960
children
line wrap: on
line source



#include "SqliteDBLink.hpp"

#include <QtCore/QStringList>
#include <QtCore/QVariant>


#include <QtSql/QSqlQuery>
#include <QtSql/QSqlRecord>

#include "Exception/SQLException.hpp"
#include "Exception/IOException.hpp"

#include <cassert>

const QString SqliteDBLink::connectionName("SqliteDBLink");

#include <QtCore/QDebug>


SqliteDBLink::SqliteDBLink(const QString& dbPath)
{
  if (!QSqlDatabase::contains(connectionName))
    db = QSqlDatabase::addDatabase("QSQLITE", connectionName);
  else {
    db = QSqlDatabase::database(connectionName);
  }
  db.setDatabaseName(dbPath);
  if (!db.open())
    throw
      IOException(QString("Unable to open SQLite database with path '%1'")
		  .arg(dbPath));
  QSqlQuery query(db);
  if (!query.exec(QString("SELECT * FROM files;"))) {
    query.exec("CREATE TABLE files(path VARCHAR PRIMARY KEY ASC,"
	       " size INTEGER, mtime TEXT, checksum TEXT);");
  }
  if (!query.exec(QString("SELECT * FROM files;"))) {
    throw SQLException("No database");
  }

  if (!query.exec(QString("SELECT * FROM human_files;"))) {
    query.exec("CREATE VIEW human_files AS SELECT path, size, date(mtime) as mdate, time(mtime) as mtime, hex(checksum) as checksum FROM files;");
  }
  if (!query.exec(QString("SELECT * FROM human_files;"))) {
    throw SQLException("No view");
  }


  preparedBeginQuery = new QSqlQuery(db);
  preparedBeginQuery->prepare("BEGIN TRANSACTION");

  preparedEndQuery = new QSqlQuery(db);
  preparedEndQuery->prepare("END TRANSACTION");

  preparedSizePrefixQuery = new QSqlQuery(db);
  preparedSizePrefixQuery->prepare("SELECT * FROM files WHERE path LIKE :prefix AND size = :size");

  preparedSizeQuery = new QSqlQuery(db);
  preparedSizeQuery->prepare("SELECT * FROM files WHERE size = :size");

  preparedTryAddQuery = new QSqlQuery(db);
  preparedTryAddQuery->prepare("INSERT INTO files (path, size, mtime, checksum)"
			       " VALUES (:path, :size, :mtime, :checksum)");

  preparedUpdateQuery = new QSqlQuery(db);
  preparedUpdateQuery->prepare(
			 "UPDATE files "
			 "SET size=:size, mtime=:mtime, checksum=:checksum "
			 "WHERE path=:path");

  preparedDeleteQuery = new QSqlQuery(db);
  preparedDeleteQuery->prepare("DELETE FROM files WHERE path = :path");
}

SqliteDBLink::~SqliteDBLink()
{
  delete preparedSizePrefixQuery;
  delete preparedSizeQuery;
  delete preparedTryAddQuery;
  delete preparedUpdateQuery;
  db.close();
}

bool SqliteDBLink::exists(const QString& path)
{
  QSqlQuery query(db);
  query.prepare("SELECT path FROM files WHERE path = :path;");
  query.bindValue(":path", path);
  if (!query.exec()) {
    throw SQLException(query);
  }
  return query.last();
}

FileDBLink::DBStatus SqliteDBLink::existsWithMtime(const QString& path,
						   const QDateTime& mtime)
{
  QSqlQuery query(db);
  query.prepare("SELECT mtime FROM files WHERE path = :path;");
  query.bindValue(":path", path);
  if (!query.exec()) {
    throw SQLException(query);
  }
  if (query.next()) {
    int dateIndex = query.record().indexOf("mtime");
    QDateTime mtimeEntry = query.value(dateIndex).toDateTime();
    if (mtimeEntry == mtime)
      return SAME;
    return MTIME_DIFFERENT;
  }
  return NONE;
}

void SqliteDBLink::addFile(const QString& path, quint64 size,
			   const QDateTime& dtime, const QByteArray& hash,
			   bool lazy)
{
  addFile(DBInfo(path, size, dtime, hash), lazy);
}

bool SqliteDBLink::tryAddFile(const DBInfo& dbinfo)
{
  if (exists(dbinfo.path()))
    return false;
  preparedTryAddQuery->bindValue(":path", dbinfo.path());
  preparedTryAddQuery->bindValue(":size", dbinfo.size());
  preparedTryAddQuery->bindValue(":mtime", dbinfo.mtime());
  preparedTryAddQuery->bindValue(":checksum", dbinfo.checksum());
  if (!preparedTryAddQuery->exec()) {
    throw SQLException(*preparedTryAddQuery);
  }
  return true;
}

void SqliteDBLink::updateFile(const QString& path,
			      quint64 size, const QDateTime& dtime,
			      const QByteArray& hash, bool lazy)
{
  updateFile(DBInfo(path, size, dtime, hash), lazy);
}

void SqliteDBLink::updateFile(const DBInfo& dbinfo, bool lazy)
{
  if (lazy) {
    operations.push_back(new UpdateOperation(dbinfo));
  }
  else {
    preparedUpdateQuery->bindValue(":path", dbinfo.path());
    preparedUpdateQuery->bindValue(":size", dbinfo.size());
    preparedUpdateQuery->bindValue(":mtime", dbinfo.mtime());
    preparedUpdateQuery->bindValue(":checksum", dbinfo.checksum());
    if (!preparedUpdateQuery->exec()) {
      throw SQLException(*preparedUpdateQuery);
    }
  }
}

void SqliteDBLink::addFile(const DBInfo& dbinfo, bool lazy)
{
  if (lazy) {
    operations.push_back(new AddOperation(dbinfo));
  }
  else {
    if (!tryAddFile(dbinfo)) {
      abort(); //Should throw exception
    }
  }
}


QStringList SqliteDBLink::toStringList()
{
  abort();
  QStringList list;
  /*
  foreach(dbinf_ptr_t info, entries) {
    list << info->serialize();
  }
  */
  return list;
}

const QList<FileDBLink::dbinf_ptr_t >
SqliteDBLink::values(const QString& prefix) const
{
  QList<FileDBLink::dbinf_ptr_t > values;

  QSqlQuery query(db);

  if (prefix.size() > 0) {
    query.prepare("SELECT * FROM files WHERE path LIKE :prefix");
    query.bindValue(":prefix", QString("%1%").arg(prefix));
  }
  else {
    query.prepare("SELECT * FROM files");
  }

  if (!query.exec()) {
    throw SQLException(query);
  }

  int pathIndex = query.record().indexOf("path");
  int sizeIndex = query.record().indexOf("size");
  int dateIndex = query.record().indexOf("mtime");
  int checksumIndex = query.record().indexOf("checksum");
  while (query.next()) {
    QString path = query.value(pathIndex).toString();
    quint64 size = query.value(sizeIndex).toInt();
    QDateTime mtime = query.value(dateIndex).toDateTime();
    QByteArray checksum = query.value(checksumIndex).toByteArray();

    values <<
      FileDBLink::dbinf_ptr_t(new FileDBLink::DBInfo(path,
						     size, mtime, checksum));
  }

  return values;
}

void SqliteDBLink::deleteFileFromDB(const QString& path, bool lazy)
{
  if (lazy) {
    operations.push_back(new DeleteOperation(path));
  }
  else {
    QSqlQuery query(db);
    query.prepare("DELETE FROM files WHERE path = :path");
    query.bindValue(":path", path);
    if (!query.exec()) {
      throw SQLException(query);
    }
  }
}


void SqliteDBLink::keepOnlyFromPrefix(const QString& prefix,
				      const QStringList& files,
				      bool lazy)
{
  QString keepQuerySQL =
    "DELETE FROM files WHERE %1 path NOT IN ( %2 )";

  bool first = true;
  QString inList = "";
  foreach (const QString& file, files) {
    if (first)
      first = false;
    else
      inList+=", ";

    inList+="'"+QString(file).replace("'","''")+"'";
  }

  if (prefix.isEmpty()) {
    keepQuerySQL = keepQuerySQL.arg("");
  }
  else {
    keepQuerySQL = keepQuerySQL.arg("path LIKE :prefix1 AND ");
  }
  keepQuerySQL = keepQuerySQL.arg(inList);
  QSqlQuery keepQuery(db);
  keepQuery.prepare(keepQuerySQL);
  if (prefix.size() > 0) {
    keepQuery.bindValue(":prefix", QString("%1%").arg(prefix));
  }
  if (!keepQuery.exec()) {
    throw SQLException(keepQuery);
  }
}

const QList<FileDBLink::dbinf_ptr_t>
SqliteDBLink::filesWithSize(quint64 size, const QString& prefix) const
{
  QList<dbinf_ptr_t > values;

  QSqlQuery & query = (prefix.size() > 0) ? *preparedSizePrefixQuery : *preparedSizeQuery;

  if (prefix.size() > 0) {
    query.bindValue(":prefix", QString("%1%").arg(prefix));
  }
  query.bindValue(":size", QString("%1%").arg(size));


  if (!query.exec()) {
    throw SQLException(query);
  }

  int pathIndex = query.record().indexOf("path");
  int sizeIndex = query.record().indexOf("size");
  int dateIndex = query.record().indexOf("mtime");
  int checksumIndex = query.record().indexOf("checksum");
  while (query.next()) {
    QString path = query.value(pathIndex).toString();
    quint64 size = query.value(sizeIndex).toInt();
    QDateTime mtime = query.value(dateIndex).toDateTime();
    QByteArray checksum = query.value(checksumIndex).toByteArray();

    values <<
      FileDBLink::dbinf_ptr_t(new FileDBLink::DBInfo(path,
						     size, mtime, checksum));
  }

  return values;
}

void SqliteDBLink::executeOperation(QVariantList& paths,
				    QVariantList& sizes,
				    QVariantList& mtimes,
				    QVariantList& checksums,
				    OperationType operation)
{
  assert(sizes.size() == 0 || paths.size() == sizes.size());
  assert(mtimes.size() == 0 || paths.size() == mtimes.size());
  assert(checksums.size() == 0 || paths.size() == checksums.size());
  QSqlQuery* query;
  bool infoQuery;
  switch (operation) {
  case Add:
    qDebug() << "Add";
    query = preparedTryAddQuery;
    infoQuery = true;
    break;
  case Update:
    query = preparedUpdateQuery;
    infoQuery = true;
    break;
  case Delete:
    query = preparedDeleteQuery;
    infoQuery = false;
    break;
  case None:
    assert(paths.size() == 0);
    return;
  }
  if (operation != Update)
    query->bindValue("path", paths);
  if (infoQuery) {
    query->bindValue("size", sizes);
    query->bindValue("mtime", mtimes);
    query->bindValue("checksum", checksums);
  }
  //Because QT for some reason expect the parameters to be given in sequence
  if (operation == Update)
    query->bindValue("path", paths);


  if (!query->execBatch())
    throw SQLException(*query);

  paths.clear();
  sizes.clear();
  mtimes.clear();
  checksums.clear();
}

const char* SqliteDBLink::typeString(OperationType type)
{
  switch(type) {
  case Add:
    return "Add";
  case Update:
    return "Update";
  case Delete:
    return "Delete";
  case None:
    return "None";
  }

  return "";
}


bool SqliteDBLink::commit(const QString& prefix, bool dirty)
{
  OperationType last = None;
  QVariantList paths, sizes, mtimes, hashes;

  const Operation* operation =
    (operations.empty()) ? NULL : operations.takeFirst();

  while (operation) {
    if (operation->type() != last) {
      qDebug() << "Execute Operation first" << typeString(last);
      foreach(QVariant path, paths) {
	qDebug() << path.toString();
      }
      if (!preparedBeginQuery->exec())
	throw SQLException(*preparedBeginQuery);
      executeOperation(paths, sizes, mtimes, hashes, last);
      if (!preparedEndQuery->exec())
	throw SQLException(*preparedEndQuery);
      qDebug() << "Execute Operation Done";
    }

    switch (operation->type()) {
    case Add:
    case Update: {
      const InfoOperation* iOperation = dynamic_cast<const InfoOperation*>(operation);
      paths.push_back(iOperation->info().path());
      sizes.push_back(iOperation->info().size());
      mtimes.push_back(iOperation->info().mtime());
      hashes.push_back(iOperation->info().checksum());
      dirty = true;
      break;
    }
    case Delete: {
      const DeleteOperation* dOperation = dynamic_cast<const DeleteOperation*>(operation);
      paths.push_back(dOperation->path());
    }
    case None:
      break;
    }
    last = operation->type();
    delete operation;
    operation = (operations.empty()) ? NULL : operations.takeFirst();
  }
  if (last != None) {
    qDebug() << "Execute Operation" << typeString(last);
    foreach(QVariant path, paths) {
      qDebug() << path.toString();
    }
    if (!preparedBeginQuery->exec())
      throw SQLException(*preparedBeginQuery);
    executeOperation(paths, sizes, mtimes, hashes, last);
    if (!preparedEndQuery->exec())
      throw SQLException(*preparedEndQuery);
    qDebug() << "Execute Operation Done";
  }

  if (dirty) {
    QSqlQuery whatToUpdate(db);
    QString whatToUpdateQuery =
      "SELECT path FROM files WHERE checksum is NULL %1 AND size in "
      "(SELECT size FROM files WHERE size <> 0 %2 "
      "GROUP BY size HAVING count(*) > 1) ORDER BY size";
    if (prefix.isEmpty()) {
      whatToUpdateQuery = whatToUpdateQuery.arg("");
    }
    else {
      whatToUpdateQuery = whatToUpdateQuery.arg("AND path LIKE :prefix1");
      whatToUpdateQuery = whatToUpdateQuery.arg("AND path LIKE :prefix2");
    }
    whatToUpdate.prepare(whatToUpdateQuery);
    if (!prefix.isEmpty()) {
      whatToUpdate.bindValue("prefix1", QString("%1%").arg(prefix));
      whatToUpdate.bindValue("prefix2", QString("%1%").arg(prefix));
    }

    qDebug() << "Before whatToUpdate";
    if (!whatToUpdate.exec()) {
      throw SQLException(whatToUpdate);
    }
    qDebug() << "After whatToUpdate";

    int pathIndex = whatToUpdate.record().indexOf("path");
    QStringList updatePaths;
    while (whatToUpdate.next()) {
      updatePaths << whatToUpdate.value(pathIndex).toString();
    }
    int n = 0;
    int max = updatePaths.size();
    emit progressUpdate(0, max);
    QSqlQuery updateChecksum(db);
    updateChecksum.prepare("UPDATE files "
			   "SET checksum=:checksum "
			   "WHERE path=:path");

    foreach (const QString& path, updatePaths) {
      qDebug() << path;
      QByteArray ohash = computeHash(path);
      emit progressUpdate(++n, max);
      updateChecksum.bindValue("checksum", ohash);
      updateChecksum.bindValue("path", path);
      if (!updateChecksum.exec())
	throw SQLException(updateChecksum);

    }
  }
  return true;
}