Mercurial > dedupe

changeset 104:6bc013d5788b

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Avoid unnecessary updates. Fix problems with wrong subset being selected for update with prefix. Fix some problems with to much verbosity in debug statements.
author Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
date Sat, 15 Feb 2014 13:34:10 +0100
parents 6b997f4f7e19
children 0df1552123e7
files DataController.cpp FileDBLink.cpp FileDBLink.hpp SqliteDBLink.cpp SqliteDBLink.hpp
diffstat 5 files changed, 61 insertions(+), 48 deletions(-) [+]
line wrap: on
line diff
--- a/DataController.cpp	Sat Feb 15 13:32:46 2014 +0100
+++ b/DataController.cpp	Sat Feb 15 13:34:10 2014 +0100
@@ -82,7 +82,7 @@
 
   qDebug() << "Start Delete";
   dblink.keepOnlyFromPrefix(dir.path(), paths, true);
-  dblink.commit(dir.path());
+  dblink.commit(dir.path(), false);
   qDebug() << "End Delete";
 
   std::auto_ptr<QProgressBar> bar;
--- a/FileDBLink.cpp	Sat Feb 15 13:32:46 2014 +0100
+++ b/FileDBLink.cpp	Sat Feb 15 13:34:10 2014 +0100
@@ -225,7 +225,7 @@
   return dbinf_ptr_t();
 }
 
-bool FileDBLink::commit(const QString&)
+bool FileDBLink::commit(const QString&, bool)
 {
   return true;
 }
--- a/FileDBLink.hpp	Sat Feb 15 13:32:46 2014 +0100
+++ b/FileDBLink.hpp	Sat Feb 15 13:34:10 2014 +0100
@@ -146,7 +146,7 @@
   virtual DBStatus existsWithMtime(const QString& path,
 				   const QDateTime& mtime) = 0;
 
-  virtual bool commit(const QString& prefix = QString());
+  virtual bool commit(const QString& prefix = QString(), bool dirty = true);
   virtual const QList<dbinf_ptr_t>
   values(const QString& prefix = QString() ) const = 0;
 
--- a/SqliteDBLink.cpp	Sat Feb 15 13:32:46 2014 +0100
+++ b/SqliteDBLink.cpp	Sat Feb 15 13:34:10 2014 +0100
@@ -340,12 +340,15 @@
 }
 
 
-bool SqliteDBLink::commit(const QString& prefix)
+bool SqliteDBLink::commit(const QString& prefix, bool dirty)
 {
   OperationType last = None;
   QVariantList paths, sizes, mtimes, hashes;
 
-  foreach(const Operation* operation, operations) {
+  const Operation* operation =
+    (operations.empty()) ? NULL : operations.takeFirst();
+
+  while (operation) {
     if (operation->type() != last) {
       executeOperation(paths, sizes, mtimes, hashes, last);
     }
@@ -358,6 +361,7 @@
       sizes.push_back(iOperation->info().size());
       mtimes.push_back(iOperation->info().mtime());
       hashes.push_back(iOperation->info().checksum());
+      dirty = true;
       break;
     }
     case Delete: {
@@ -368,58 +372,66 @@
       break;
     }
     last = operation->type();
+    delete operation;
+    operation = (operations.empty()) ? NULL : operations.takeFirst();
   }
   if (last != None) {
     qDebug() << "Execute Operation" << typeString(last);
-    qDebug() << paths;
+    foreach(QVariant path, paths) {
+      qDebug() << path.toString();
+    }
     executeOperation(paths, sizes, mtimes, hashes, last);
     qDebug() << "Execute Operation Done";
   }
 
-  QSqlQuery whatToUpdate(db);
-  QString whatToUpdateQuery =
-    "SELECT path FROM files WHERE checksum is NULL AND path in "
-    "(SELECT path FROM files WHERE size <> 0 %1 "
-    "GROUP BY size HAVING count(*) > 1) ORDER BY size";
-  if (prefix.isEmpty()) {
-    whatToUpdateQuery = whatToUpdateQuery.arg("");
-  }
-  else {
-    whatToUpdateQuery = whatToUpdateQuery.arg("AND path LIKE :prefix");
-  }
-  whatToUpdate.prepare(whatToUpdateQuery);
-  if (!prefix.isEmpty()) {
-    whatToUpdate.bindValue("prefix", QString("%1%").arg(prefix));
-  }
-
-  qDebug() << "Before whatToUpdate";
-  if (!whatToUpdate.exec()) {
-    throw SQLException(whatToUpdate);
-  }
-  qDebug() << "After whatToUpdate";
+  if (dirty) {
+    QSqlQuery whatToUpdate(db);
+    QString whatToUpdateQuery =
+      "SELECT path FROM files WHERE checksum is NULL %1 AND size in "
+      "(SELECT size FROM files WHERE size <> 0 %2 "
+      "GROUP BY size HAVING count(*) > 1) ORDER BY size";
+    if (prefix.isEmpty()) {
+      whatToUpdateQuery = whatToUpdateQuery.arg("");
+    }
+    else {
+      whatToUpdateQuery = whatToUpdateQuery.arg("AND path LIKE :prefix1");
+      whatToUpdateQuery = whatToUpdateQuery.arg("AND path LIKE :prefix2");
+    }
+    whatToUpdate.prepare(whatToUpdateQuery);
+    if (!prefix.isEmpty()) {
+      whatToUpdate.bindValue("prefix1", QString("%1%").arg(prefix));
+      whatToUpdate.bindValue("prefix2", QString("%1%").arg(prefix));
+    }
 
-  int pathIndex = whatToUpdate.record().indexOf("path");
-  QStringList updatePaths;
-  while (whatToUpdate.next()) {
-    updatePaths << whatToUpdate.value(pathIndex).toString();
-  }
-  int n = 0;
-  int max = updatePaths.size();
-  emit progressUpdate(0, max);
-  QSqlQuery updateChecksum(db);
-  updateChecksum.prepare("UPDATE files "
-			 "SET checksum=:checksum "
-			 "WHERE path=:path");
+    qDebug() << "Before whatToUpdate";
+    if (!whatToUpdate.exec()) {
+      throw SQLException(whatToUpdate);
+    }
+    qDebug() << "After whatToUpdate";
 
-  foreach (const QString& path, updatePaths) {
-    qDebug() << path;
-    QByteArray ohash = computeHash(path);
-    emit progressUpdate(++n, max);
-    updateChecksum.bindValue("checksum", ohash);
-    updateChecksum.bindValue("path", path);
-    if (!updateChecksum.exec())
-      throw SQLException(updateChecksum);
+    int pathIndex = whatToUpdate.record().indexOf("path");
+    QStringList updatePaths;
+    while (whatToUpdate.next()) {
+      updatePaths << whatToUpdate.value(pathIndex).toString();
+    }
+    int n = 0;
+    int max = updatePaths.size();
+    emit progressUpdate(0, max);
+    QSqlQuery updateChecksum(db);
+    updateChecksum.prepare("UPDATE files "
+			   "SET checksum=:checksum "
+			   "WHERE path=:path");
 
+    foreach (const QString& path, updatePaths) {
+      qDebug() << path;
+      QByteArray ohash = computeHash(path);
+      emit progressUpdate(++n, max);
+      updateChecksum.bindValue("checksum", ohash);
+      updateChecksum.bindValue("path", path);
+      if (!updateChecksum.exec())
+	throw SQLException(updateChecksum);
+
+    }
   }
   return true;
 }
--- a/SqliteDBLink.hpp	Sat Feb 15 13:32:46 2014 +0100
+++ b/SqliteDBLink.hpp	Sat Feb 15 13:34:10 2014 +0100
@@ -29,7 +29,7 @@
 				  const QStringList& files,
 				  bool lazy = false);
   virtual void deleteFileFromDB(const QString& path, bool lazy = false);
-  bool commit(const QString& prefix = QString() );
+  bool commit(const QString& prefix = QString(), bool dirty = true );
 
 private:
   typedef enum {None = 0, Add, Update, Delete } OperationType;
@@ -39,6 +39,7 @@
   class Operation {
   public:
     virtual OperationType type() const = 0;
+    virtual  ~Operation() {}
   };
 
   class InfoOperation: public Operation {