Mercurial > dedupe
changeset 94:93981e675d67
Full support for bulkcommits.
| author | Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> |
|---|---|
| date | Tue, 22 Oct 2013 14:11:50 +0200 |
| parents | 308a718812ba |
| children | 7c935d3d5b74 |
| files | SqliteDBLink.cpp SqliteDBLink.hpp |
| diffstat | 2 files changed, 100 insertions(+), 15 deletions(-) [+] |
line wrap: on
line diff
--- a/SqliteDBLink.cpp Tue Oct 22 11:53:54 2013 +0200 +++ b/SqliteDBLink.cpp Tue Oct 22 14:11:50 2013 +0200 @@ -10,6 +10,8 @@ #include "Exception/SQLException.hpp" #include "Exception/IOException.hpp" +#include <cassert> + const QString SqliteDBLink::connectionName("SqliteDBLink"); @@ -45,12 +47,19 @@ preparedTryAddQuery->prepare("INSERT INTO files (path, size, mtime, checksum)" " VALUES (:path, :size, :mtime, :checksum)"); + preparedUpdateQuery = new QSqlQuery(db); + preparedUpdateQuery->prepare( + "UPDATE files " + "SET size=:size, mtime=:mtime, checksum=:checksum " + "WHERE path=:path"); } SqliteDBLink::~SqliteDBLink() { delete preparedSizePrefixQuery; delete preparedSizeQuery; + delete preparedTryAddQuery; + delete preparedUpdateQuery; db.close(); } @@ -118,16 +127,12 @@ operations.push_back(Operation(dbinfo, Update)); } else { - QSqlQuery query(db); - query.prepare("UPDATE files " - "SET size=:size, mtime=:mtime, checksum=:checksum " - "WHERE path=:path"); - query.bindValue(":path", dbinfo.path()); - query.bindValue(":size", dbinfo.size()); - query.bindValue(":mtime", dbinfo.mtime()); - query.bindValue(":checksum", dbinfo.checksum()); - if (!query.exec()) { - throw SQLException(query); + preparedUpdateQuery->bindValue(":path", dbinfo.path()); + preparedUpdateQuery->bindValue(":size", dbinfo.size()); + preparedUpdateQuery->bindValue(":mtime", dbinfo.mtime()); + preparedUpdateQuery->bindValue(":checksum", dbinfo.checksum()); + if (!preparedUpdateQuery->exec()) { + throw SQLException(*preparedUpdateQuery); } } } @@ -254,19 +259,94 @@ return values; } +void SqliteDBLink::executeOperation(QVariantList& paths, + QVariantList& sizes, + QVariantList& mtimes, + QVariantList& checksums, + OperationType operation) +{ + assert(paths.size() == sizes.size()); + assert(paths.size() == mtimes.size()); + assert(paths.size() == checksums.size()); + QSqlQuery* query; + switch (operation) { + case Add: + query = preparedTryAddQuery; + break; + case Update: + query = preparedUpdateQuery; + break; + case None: + assert(paths.size() == 0); + return; + } + query->bindValue("path", paths); + query->bindValue("size", sizes); + query->bindValue("mtime", mtimes); + query->bindValue("checksum", checksums); + + if (!query->execBatch()) + throw SQLException(*query); + + paths.clear(); + sizes.clear(); + mtimes.clear(); + checksums.clear(); +} + bool SqliteDBLink::commit() { int n = 0; - int max = operations.size(); + OperationType last = None; + QVariantList paths, sizes, mtimes, hashes; + foreach(const Operation& operation, operations) { + if (operation.second != last) { + executeOperation(paths, sizes, mtimes, hashes, last); + } + switch (operation.second) { case Add: - addFile(operation.first); + case Update: + paths.push_back(operation.first.path()); + sizes.push_back(operation.first.size()); + mtimes.push_back(operation.first.mtime()); + hashes.push_back(operation.first.checksum()); + case None: break; - case Update: - updateFile(operation.first); } + last = operation.second; + } + if (last != None) { + executeOperation(paths, sizes, mtimes, hashes, last); + } + + QSqlQuery whatToUpdate(db); + whatToUpdate.prepare("SELECT path FROM files WHERE checksum is NULL AND size in (SELECT size FROM files WHERE size <> 0 GROUP BY size HAVING count(*) > 1 ORDER BY SIZE) ORDER BY size"); + + if (!whatToUpdate.exec()) { + throw SQLException(whatToUpdate); + } + + int pathIndex = whatToUpdate.record().indexOf("path"); + QStringList updatePaths; + while (whatToUpdate.next()) { + updatePaths << whatToUpdate.value(pathIndex).toString(); + } + int max = updatePaths.size(); + QSqlQuery updateChecksum(db); + updateChecksum.prepare("UPDATE files " + "SET checksum=:checksum " + "WHERE path=:path"); + + foreach (const QString& path, updatePaths) { + QByteArray ohash = computeHash(path); emit progressUpdate(++n, max); + updateChecksum.bindValue("checksum", ohash); + updateChecksum.bindValue("path", path); + if (!updateChecksum.exec()) + throw SQLException(updateChecksum); + } return true; }
--- a/SqliteDBLink.hpp Tue Oct 22 11:53:54 2013 +0200 +++ b/SqliteDBLink.hpp Tue Oct 22 14:11:50 2013 +0200 @@ -4,6 +4,7 @@ #include <QtSql/QSqlDatabase> #include <QtCore/QPair> +#include <QtCore/QVariant> class SqliteDBLink : public FileDBLink { public: @@ -30,7 +31,7 @@ bool commit(); private: - typedef enum {Add, Update } OperationType; + typedef enum {None = 0, Add, Update } OperationType; typedef QPair<DBInfo, OperationType> Operation; void addFile(const DBInfo& info, bool lazy = false); @@ -43,7 +44,11 @@ QSqlQuery* preparedSizePrefixQuery; QSqlQuery* preparedSizeQuery; QSqlQuery* preparedTryAddQuery; + QSqlQuery* preparedUpdateQuery; QList<Operation> operations; + void executeOperation(QVariantList& paths, QVariantList& sizes, + QVariantList& mtimes, QVariantList& checksums, + OperationType operation); }; #endif //MEMORYDBLINK_HPP
