changeset 94:93981e675d67

Full support for bulkcommits.
author Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
date Tue, 22 Oct 2013 14:11:50 +0200
parents 308a718812ba
children 7c935d3d5b74
files SqliteDBLink.cpp SqliteDBLink.hpp
diffstat 2 files changed, 100 insertions(+), 15 deletions(-) [+]
line wrap: on
line diff
--- a/SqliteDBLink.cpp	Tue Oct 22 11:53:54 2013 +0200
+++ b/SqliteDBLink.cpp	Tue Oct 22 14:11:50 2013 +0200
@@ -10,6 +10,8 @@
 #include "Exception/SQLException.hpp"
 #include "Exception/IOException.hpp"
 
+#include <cassert>
+
 const QString SqliteDBLink::connectionName("SqliteDBLink");
 
 
@@ -45,12 +47,19 @@
   preparedTryAddQuery->prepare("INSERT INTO files (path, size, mtime, checksum)"
 			       " VALUES (:path, :size, :mtime, :checksum)");
 
+  preparedUpdateQuery = new QSqlQuery(db);
+  preparedUpdateQuery->prepare(
+			 "UPDATE files "
+			 "SET size=:size, mtime=:mtime, checksum=:checksum "
+			 "WHERE path=:path");
 }
 
 SqliteDBLink::~SqliteDBLink()
 {
   delete preparedSizePrefixQuery;
   delete preparedSizeQuery;
+  delete preparedTryAddQuery;
+  delete preparedUpdateQuery;
   db.close();
 }
 
@@ -118,16 +127,12 @@
     operations.push_back(Operation(dbinfo, Update));
   }
   else {
-    QSqlQuery query(db);
-    query.prepare("UPDATE files "
-		  "SET size=:size, mtime=:mtime, checksum=:checksum "
-		  "WHERE path=:path");
-    query.bindValue(":path", dbinfo.path());
-    query.bindValue(":size", dbinfo.size());
-    query.bindValue(":mtime", dbinfo.mtime());
-    query.bindValue(":checksum", dbinfo.checksum());
-    if (!query.exec()) {
-      throw SQLException(query);
+    preparedUpdateQuery->bindValue(":path", dbinfo.path());
+    preparedUpdateQuery->bindValue(":size", dbinfo.size());
+    preparedUpdateQuery->bindValue(":mtime", dbinfo.mtime());
+    preparedUpdateQuery->bindValue(":checksum", dbinfo.checksum());
+    if (!preparedUpdateQuery->exec()) {
+      throw SQLException(*preparedUpdateQuery);
     }
   }
 }
@@ -254,19 +259,94 @@
   return values;
 }
 
+void SqliteDBLink::executeOperation(QVariantList& paths,
+				    QVariantList& sizes,
+				    QVariantList& mtimes,
+				    QVariantList& checksums,
+				    OperationType operation)
+{
+  assert(paths.size() == sizes.size());
+  assert(paths.size() == mtimes.size());
+  assert(paths.size() == checksums.size());
+  QSqlQuery* query;
+  switch (operation) {
+  case Add:
+    query = preparedTryAddQuery;
+    break;
+  case Update:
+    query = preparedUpdateQuery;
+    break;
+  case None:
+    assert(paths.size() == 0);
+    return;
+  }
+  query->bindValue("path", paths);
+  query->bindValue("size", sizes);
+  query->bindValue("mtime", mtimes);
+  query->bindValue("checksum", checksums);
+
+  if (!query->execBatch())
+    throw SQLException(*query);
+
+  paths.clear();
+  sizes.clear();
+  mtimes.clear();
+  checksums.clear();
+}
+
 bool SqliteDBLink::commit()
 {
   int n = 0;
-  int max = operations.size();
+  OperationType last = None;
+  QVariantList paths, sizes, mtimes, hashes;
+
   foreach(const Operation& operation, operations) {
+    if (operation.second != last) {
+      executeOperation(paths, sizes, mtimes, hashes, last);
+    }
+
     switch (operation.second) {
     case Add:
-      addFile(operation.first);
+    case Update:
+      paths.push_back(operation.first.path());
+      sizes.push_back(operation.first.size());
+      mtimes.push_back(operation.first.mtime());
+      hashes.push_back(operation.first.checksum());
+    case None:
       break;
-    case Update:
-      updateFile(operation.first);
     }
+    last = operation.second;
+  }
+  if (last != None) {
+    executeOperation(paths, sizes, mtimes, hashes, last);
+  }
+
+  QSqlQuery whatToUpdate(db);
+  whatToUpdate.prepare("SELECT path FROM files WHERE checksum is NULL AND size in (SELECT size FROM files WHERE size <> 0 GROUP BY size HAVING count(*) > 1 ORDER BY SIZE) ORDER BY size");
+
+  if (!whatToUpdate.exec()) {
+    throw SQLException(whatToUpdate);
+  }
+
+  int pathIndex = whatToUpdate.record().indexOf("path");
+  QStringList updatePaths;
+  while (whatToUpdate.next()) {
+    updatePaths << whatToUpdate.value(pathIndex).toString();
+  }
+  int max = updatePaths.size();
+  QSqlQuery updateChecksum(db);
+  updateChecksum.prepare("UPDATE files "
+			 "SET checksum=:checksum "
+			 "WHERE path=:path");
+
+  foreach (const QString& path, updatePaths) {
+    QByteArray ohash = computeHash(path);
     emit progressUpdate(++n, max);
+    updateChecksum.bindValue("checksum", ohash);
+    updateChecksum.bindValue("path", path);
+    if (!updateChecksum.exec())
+      throw SQLException(updateChecksum);
+
   }
   return true;
 }
--- a/SqliteDBLink.hpp	Tue Oct 22 11:53:54 2013 +0200
+++ b/SqliteDBLink.hpp	Tue Oct 22 14:11:50 2013 +0200
@@ -4,6 +4,7 @@
 
 #include <QtSql/QSqlDatabase>
 #include <QtCore/QPair>
+#include <QtCore/QVariant>
 
 class SqliteDBLink : public FileDBLink {
 public:
@@ -30,7 +31,7 @@
   bool commit();
 
 private:
-  typedef enum {Add, Update } OperationType;
+  typedef enum {None = 0, Add, Update } OperationType;
   typedef QPair<DBInfo, OperationType> Operation;
 
   void addFile(const DBInfo& info, bool lazy = false);
@@ -43,7 +44,11 @@
   QSqlQuery* preparedSizePrefixQuery;
   QSqlQuery* preparedSizeQuery;
   QSqlQuery* preparedTryAddQuery;
+  QSqlQuery* preparedUpdateQuery;
   QList<Operation> operations;
+  void executeOperation(QVariantList& paths, QVariantList& sizes,
+			QVariantList& mtimes, QVariantList& checksums,
+			OperationType operation);
 };
 
 #endif //MEMORYDBLINK_HPP