Mercurial > dedupe
view SqliteDBLink.cpp @ 95:7c935d3d5b74
Better signify when starting update of checksums.
| author | Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> |
|---|---|
| date | Tue, 22 Oct 2013 14:22:00 +0200 |
| parents | 93981e675d67 |
| children | c7da835ea912 |
line wrap: on
line source
#include "SqliteDBLink.hpp" #include <QtCore/QStringList> #include <QtCore/QVariant> #include <QtSql/QSqlQuery> #include <QtSql/QSqlRecord> #include "Exception/SQLException.hpp" #include "Exception/IOException.hpp" #include <cassert> const QString SqliteDBLink::connectionName("SqliteDBLink"); SqliteDBLink::SqliteDBLink(const QString& dbPath) { if (!QSqlDatabase::contains(connectionName)) db = QSqlDatabase::addDatabase("QSQLITE", connectionName); else { db = QSqlDatabase::database(connectionName); } db.setDatabaseName(dbPath); if (!db.open()) throw IOException(QString("Unable to open SQLite database with path '%1'") .arg(dbPath)); QSqlQuery query(db); if (!query.exec(QString("SELECT * FROM files;"))) { query.exec("CREATE TABLE files(path VARCHAR PRIMARY KEY ASC," " size INTEGER, mtime TEXT, checksum TEXT);"); } if (!query.exec(QString("SELECT * FROM files;"))) { throw SQLException("No database"); } preparedSizePrefixQuery = new QSqlQuery(db); preparedSizePrefixQuery->prepare("SELECT * FROM files WHERE path LIKE :prefix AND size = :size"); preparedSizeQuery = new QSqlQuery(db); preparedSizeQuery->prepare("SELECT * FROM files WHERE size = :size"); preparedTryAddQuery = new QSqlQuery(db); preparedTryAddQuery->prepare("INSERT INTO files (path, size, mtime, checksum)" " VALUES (:path, :size, :mtime, :checksum)"); preparedUpdateQuery = new QSqlQuery(db); preparedUpdateQuery->prepare( "UPDATE files " "SET size=:size, mtime=:mtime, checksum=:checksum " "WHERE path=:path"); } SqliteDBLink::~SqliteDBLink() { delete preparedSizePrefixQuery; delete preparedSizeQuery; delete preparedTryAddQuery; delete preparedUpdateQuery; db.close(); } bool SqliteDBLink::exists(const QString& path) { QSqlQuery query(db); query.prepare("SELECT path FROM files WHERE path = :path;"); query.bindValue(":path", path); if (!query.exec()) { throw SQLException(query); } return query.last(); } FileDBLink::DBStatus SqliteDBLink::existsWithMtime(const QString& path, const QDateTime& mtime) { QSqlQuery query(db); query.prepare("SELECT mtime FROM files WHERE path = :path;"); query.bindValue(":path", path); if (!query.exec()) { throw SQLException(query); } if (query.next()) { int dateIndex = query.record().indexOf("mtime"); QDateTime mtimeEntry = query.value(dateIndex).toDateTime(); if (mtimeEntry == mtime) return SAME; return MTIME_DIFFERENT; } return NONE; } void SqliteDBLink::addFile(const QString& path, quint64 size, const QDateTime& dtime, const QByteArray& hash, bool lazy) { addFile(DBInfo(path, size, dtime, hash), lazy); } bool SqliteDBLink::tryAddFile(const DBInfo& dbinfo) { if (exists(dbinfo.path())) return false; preparedTryAddQuery->bindValue(":path", dbinfo.path()); preparedTryAddQuery->bindValue(":size", dbinfo.size()); preparedTryAddQuery->bindValue(":mtime", dbinfo.mtime()); preparedTryAddQuery->bindValue(":checksum", dbinfo.checksum()); if (!preparedTryAddQuery->exec()) { throw SQLException(*preparedTryAddQuery); } return true; } void SqliteDBLink::updateFile(const QString& path, quint64 size, const QDateTime& dtime, const QByteArray& hash, bool lazy) { updateFile(DBInfo(path, size, dtime, hash), lazy); } void SqliteDBLink::updateFile(const DBInfo& dbinfo, bool lazy) { if (lazy) { operations.push_back(Operation(dbinfo, Update)); } else { preparedUpdateQuery->bindValue(":path", dbinfo.path()); preparedUpdateQuery->bindValue(":size", dbinfo.size()); preparedUpdateQuery->bindValue(":mtime", dbinfo.mtime()); preparedUpdateQuery->bindValue(":checksum", dbinfo.checksum()); if (!preparedUpdateQuery->exec()) { throw SQLException(*preparedUpdateQuery); } } } void SqliteDBLink::addFile(const DBInfo& dbinfo, bool lazy) { if (lazy) { operations.push_back(Operation(dbinfo, Add)); } else { if (!tryAddFile(dbinfo)) { abort(); //Should throw exception } } } QStringList SqliteDBLink::toStringList() { abort(); QStringList list; /* foreach(dbinf_ptr_t info, entries) { list << info->serialize(); } */ return list; } const QList<FileDBLink::dbinf_ptr_t > SqliteDBLink::values(const QString& prefix) const { QList<FileDBLink::dbinf_ptr_t > values; QSqlQuery query(db); if (prefix.size() > 0) { query.prepare("SELECT * FROM files WHERE path LIKE :prefix"); query.bindValue(":prefix", QString("%1%").arg(prefix)); } else { query.prepare("SELECT * FROM files"); } if (!query.exec()) { throw SQLException(query); } int pathIndex = query.record().indexOf("path"); int sizeIndex = query.record().indexOf("size"); int dateIndex = query.record().indexOf("mtime"); int checksumIndex = query.record().indexOf("checksum"); while (query.next()) { QString path = query.value(pathIndex).toString(); quint64 size = query.value(sizeIndex).toInt(); QDateTime mtime = query.value(dateIndex).toDateTime(); QByteArray checksum = query.value(checksumIndex).toByteArray(); values << FileDBLink::dbinf_ptr_t(new FileDBLink::DBInfo(path, size, mtime, checksum)); } return values; } void SqliteDBLink::deleteFileFromDB(const QString& path) { QSqlQuery query(db); query.prepare("DELETE FROM files WHERE path = :path"); query.bindValue(":path", path); if (!query.exec()) { throw SQLException(query); } } void SqliteDBLink::keepOnlyFromPrefix(const QString& prefix, const QStringList& files) { QStringList list; foreach(dbinf_ptr_t info, values(prefix)) { if (!files.contains(info->path())) { list << info->path(); } } foreach(QString path, list) { deleteFileFromDB(path); } } const QList<FileDBLink::dbinf_ptr_t> SqliteDBLink::filesWithSize(quint64 size, const QString& prefix) const { QList<dbinf_ptr_t > values; QSqlQuery & query = (prefix.size() > 0) ? *preparedSizePrefixQuery : *preparedSizeQuery; if (prefix.size() > 0) { query.bindValue(":prefix", QString("%1%").arg(prefix)); } query.bindValue(":size", QString("%1%").arg(size)); if (!query.exec()) { throw SQLException(query); } int pathIndex = query.record().indexOf("path"); int sizeIndex = query.record().indexOf("size"); int dateIndex = query.record().indexOf("mtime"); int checksumIndex = query.record().indexOf("checksum"); while (query.next()) { QString path = query.value(pathIndex).toString(); quint64 size = query.value(sizeIndex).toInt(); QDateTime mtime = query.value(dateIndex).toDateTime(); QByteArray checksum = query.value(checksumIndex).toByteArray(); values << FileDBLink::dbinf_ptr_t(new FileDBLink::DBInfo(path, size, mtime, checksum)); } return values; } void SqliteDBLink::executeOperation(QVariantList& paths, QVariantList& sizes, QVariantList& mtimes, QVariantList& checksums, OperationType operation) { assert(paths.size() == sizes.size()); assert(paths.size() == mtimes.size()); assert(paths.size() == checksums.size()); QSqlQuery* query; switch (operation) { case Add: query = preparedTryAddQuery; break; case Update: query = preparedUpdateQuery; break; case None: assert(paths.size() == 0); return; } query->bindValue("path", paths); query->bindValue("size", sizes); query->bindValue("mtime", mtimes); query->bindValue("checksum", checksums); if (!query->execBatch()) throw SQLException(*query); paths.clear(); sizes.clear(); mtimes.clear(); checksums.clear(); } bool SqliteDBLink::commit() { OperationType last = None; QVariantList paths, sizes, mtimes, hashes; foreach(const Operation& operation, operations) { if (operation.second != last) { executeOperation(paths, sizes, mtimes, hashes, last); } switch (operation.second) { case Add: case Update: paths.push_back(operation.first.path()); sizes.push_back(operation.first.size()); mtimes.push_back(operation.first.mtime()); hashes.push_back(operation.first.checksum()); case None: break; } last = operation.second; } if (last != None) { executeOperation(paths, sizes, mtimes, hashes, last); } QSqlQuery whatToUpdate(db); whatToUpdate.prepare("SELECT path FROM files WHERE checksum is NULL AND size in (SELECT size FROM files WHERE size <> 0 GROUP BY size HAVING count(*) > 1 ORDER BY SIZE) ORDER BY size"); if (!whatToUpdate.exec()) { throw SQLException(whatToUpdate); } int pathIndex = whatToUpdate.record().indexOf("path"); QStringList updatePaths; while (whatToUpdate.next()) { updatePaths << whatToUpdate.value(pathIndex).toString(); } int n = 0; int max = updatePaths.size(); emit progressUpdate(0, max); QSqlQuery updateChecksum(db); updateChecksum.prepare("UPDATE files " "SET checksum=:checksum " "WHERE path=:path"); foreach (const QString& path, updatePaths) { QByteArray ohash = computeHash(path); emit progressUpdate(++n, max); updateChecksum.bindValue("checksum", ohash); updateChecksum.bindValue("path", path); if (!updateChecksum.exec()) throw SQLException(updateChecksum); } return true; }
