dedupe: SqliteDBLink.cpp annotate

annotate SqliteDBLink.cpp @ 114:d4e337567960

Add some debug statements and fix a bug in SQL execution.

author	Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
date	Tue, 03 May 2016 02:26:39 +0200
parents	6cf5eb5b0be1
children

rev	line source
101 6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	1
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	2
2 2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	3 #include "SqliteDBLink.hpp"
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	4
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	5 #include <QtCore/QStringList>
31 bf3dce7fedcb Remove all references to QDebug Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: 16 diff changeset	6 #include <QtCore/QVariant>
bf3dce7fedcb Remove all references to QDebug Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: 16 diff changeset	7
2 2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	8
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	9 #include <QtSql/QSqlQuery>
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	10 #include <QtSql/QSqlRecord>
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	11
31 bf3dce7fedcb Remove all references to QDebug Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: 16 diff changeset	12 #include "Exception/SQLException.hpp"
32 c978d4a6514d Replace unnecessary asserts with exceptions. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: 31 diff changeset	13 #include "Exception/IOException.hpp"
9 b5943e4bf676 Fix up header includes. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: 7 diff changeset	14
94 93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	15 #include <cassert>
93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	16
81 69a30d9f126e Make the DBLink reusable. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 64 diff changeset	17 const QString SqliteDBLink::connectionName("SqliteDBLink");
69a30d9f126e Make the DBLink reusable. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 64 diff changeset	18
101 6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	19 #include <QtCore/QDebug>
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	20
81 69a30d9f126e Make the DBLink reusable. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 64 diff changeset	21
2 2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	22 SqliteDBLink::SqliteDBLink(const QString& dbPath)
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	23 {
81 69a30d9f126e Make the DBLink reusable. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 64 diff changeset	24 if (!QSqlDatabase::contains(connectionName))
69a30d9f126e Make the DBLink reusable. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 64 diff changeset	25 db = QSqlDatabase::addDatabase("QSQLITE", connectionName);
69a30d9f126e Make the DBLink reusable. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 64 diff changeset	26 else {
69a30d9f126e Make the DBLink reusable. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 64 diff changeset	27 db = QSqlDatabase::database(connectionName);
69a30d9f126e Make the DBLink reusable. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 64 diff changeset	28 }
2 2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	29 db.setDatabaseName(dbPath);
32 c978d4a6514d Replace unnecessary asserts with exceptions. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: 31 diff changeset	30 if (!db.open())
64 b9515dc35fe4 Make sure no file has greater linewidth than 80. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: 62 diff changeset	31 throw
b9515dc35fe4 Make sure no file has greater linewidth than 80. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: 62 diff changeset	32 IOException(QString("Unable to open SQLite database with path '%1'")
b9515dc35fe4 Make sure no file has greater linewidth than 80. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: 62 diff changeset	33 .arg(dbPath));
16 06166d6c083b Add configuration processing. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: 9 diff changeset	34 QSqlQuery query(db);
2 2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	35 if (!query.exec(QString("SELECT * FROM files;"))) {
64 b9515dc35fe4 Make sure no file has greater linewidth than 80. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: 62 diff changeset	36 query.exec("CREATE TABLE files(path VARCHAR PRIMARY KEY ASC,"
b9515dc35fe4 Make sure no file has greater linewidth than 80. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: 62 diff changeset	37 " size INTEGER, mtime TEXT, checksum TEXT);");
2 2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	38 }
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	39 if (!query.exec(QString("SELECT * FROM files;"))) {
31 bf3dce7fedcb Remove all references to QDebug Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: 16 diff changeset	40 throw SQLException("No database");
2 2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	41 }
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	42
101 6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	43 if (!query.exec(QString("SELECT * FROM human_files;"))) {
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	44 query.exec("CREATE VIEW human_files AS SELECT path, size, date(mtime) as mdate, time(mtime) as mtime, hex(checksum) as checksum FROM files;");
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	45 }
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	46 if (!query.exec(QString("SELECT * FROM human_files;"))) {
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	47 throw SQLException("No view");
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	48 }
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	49
88 6e1d4d2fc49b Prepare size query only once. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 87 diff changeset	50
108 e9b798e80bad Setup prepared begin, end queries. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 104 diff changeset	51 preparedBeginQuery = new QSqlQuery(db);
109 6cf5eb5b0be1 Embed batch operations in BEGIN-END TRANSACTION for speed improvement. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 108 diff changeset	52 preparedBeginQuery->prepare("BEGIN TRANSACTION");
108 e9b798e80bad Setup prepared begin, end queries. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 104 diff changeset	53
e9b798e80bad Setup prepared begin, end queries. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 104 diff changeset	54 preparedEndQuery = new QSqlQuery(db);
109 6cf5eb5b0be1 Embed batch operations in BEGIN-END TRANSACTION for speed improvement. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 108 diff changeset	55 preparedEndQuery->prepare("END TRANSACTION");
108 e9b798e80bad Setup prepared begin, end queries. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 104 diff changeset	56
88 6e1d4d2fc49b Prepare size query only once. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 87 diff changeset	57 preparedSizePrefixQuery = new QSqlQuery(db);
6e1d4d2fc49b Prepare size query only once. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 87 diff changeset	58 preparedSizePrefixQuery->prepare("SELECT * FROM files WHERE path LIKE :prefix AND size = :size");
6e1d4d2fc49b Prepare size query only once. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 87 diff changeset	59
6e1d4d2fc49b Prepare size query only once. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 87 diff changeset	60 preparedSizeQuery = new QSqlQuery(db);
6e1d4d2fc49b Prepare size query only once. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 87 diff changeset	61 preparedSizeQuery->prepare("SELECT * FROM files WHERE size = :size");
6e1d4d2fc49b Prepare size query only once. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 87 diff changeset	62
90 7b7a959c993b Prepare some more queries. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 88 diff changeset	63 preparedTryAddQuery = new QSqlQuery(db);
7b7a959c993b Prepare some more queries. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 88 diff changeset	64 preparedTryAddQuery->prepare("INSERT INTO files (path, size, mtime, checksum)"
7b7a959c993b Prepare some more queries. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 88 diff changeset	65 " VALUES (:path, :size, :mtime, :checksum)");
7b7a959c993b Prepare some more queries. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 88 diff changeset	66
94 93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	67 preparedUpdateQuery = new QSqlQuery(db);
93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	68 preparedUpdateQuery->prepare(
93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	69 "UPDATE files "
93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	70 "SET size=:size, mtime=:mtime, checksum=:checksum "
93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	71 "WHERE path=:path");
101 6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	72
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	73 preparedDeleteQuery = new QSqlQuery(db);
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	74 preparedDeleteQuery->prepare("DELETE FROM files WHERE path = :path");
2 2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	75 }
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	76
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	77 SqliteDBLink::~SqliteDBLink()
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	78 {
88 6e1d4d2fc49b Prepare size query only once. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 87 diff changeset	79 delete preparedSizePrefixQuery;
6e1d4d2fc49b Prepare size query only once. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 87 diff changeset	80 delete preparedSizeQuery;
94 93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	81 delete preparedTryAddQuery;
93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	82 delete preparedUpdateQuery;
2 2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	83 db.close();
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	84 }
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	85
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	86 bool SqliteDBLink::exists(const QString& path)
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	87 {
16 06166d6c083b Add configuration processing. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: 9 diff changeset	88 QSqlQuery query(db);
2 2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	89 query.prepare("SELECT path FROM files WHERE path = :path;");
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	90 query.bindValue(":path", path);
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	91 if (!query.exec()) {
31 bf3dce7fedcb Remove all references to QDebug Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: 16 diff changeset	92 throw SQLException(query);
2 2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	93 }
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	94 return query.last();
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	95 }
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	96
64 b9515dc35fe4 Make sure no file has greater linewidth than 80. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: 62 diff changeset	97 FileDBLink::DBStatus SqliteDBLink::existsWithMtime(const QString& path,
b9515dc35fe4 Make sure no file has greater linewidth than 80. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: 62 diff changeset	98 const QDateTime& mtime)
2 2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	99 {
16 06166d6c083b Add configuration processing. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: 9 diff changeset	100 QSqlQuery query(db);
2 2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	101 query.prepare("SELECT mtime FROM files WHERE path = :path;");
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	102 query.bindValue(":path", path);
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	103 if (!query.exec()) {
31 bf3dce7fedcb Remove all references to QDebug Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: 16 diff changeset	104 throw SQLException(query);
2 2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	105 }
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	106 if (query.next()) {
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	107 int dateIndex = query.record().indexOf("mtime");
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	108 QDateTime mtimeEntry = query.value(dateIndex).toDateTime();
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	109 if (mtimeEntry == mtime)
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	110 return SAME;
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	111 return MTIME_DIFFERENT;
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	112 }
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	113 return NONE;
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	114 }
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	115
86 af7962f3274b Use quint64 for sizes, rather than qint64. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 85 diff changeset	116 void SqliteDBLink::addFile(const QString& path, quint64 size,
92 f49023c61dac Support for bulk insertion. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 90 diff changeset	117 const QDateTime& dtime, const QByteArray& hash,
f49023c61dac Support for bulk insertion. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 90 diff changeset	118 bool lazy)
2 2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	119 {
92 f49023c61dac Support for bulk insertion. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 90 diff changeset	120 addFile(DBInfo(path, size, dtime, hash), lazy);
2 2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	121 }
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	122
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	123 bool SqliteDBLink::tryAddFile(const DBInfo& dbinfo)
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	124 {
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	125 if (exists(dbinfo.path()))
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	126 return false;
90 7b7a959c993b Prepare some more queries. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 88 diff changeset	127 preparedTryAddQuery->bindValue(":path", dbinfo.path());
7b7a959c993b Prepare some more queries. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 88 diff changeset	128 preparedTryAddQuery->bindValue(":size", dbinfo.size());
7b7a959c993b Prepare some more queries. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 88 diff changeset	129 preparedTryAddQuery->bindValue(":mtime", dbinfo.mtime());
7b7a959c993b Prepare some more queries. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 88 diff changeset	130 preparedTryAddQuery->bindValue(":checksum", dbinfo.checksum());
7b7a959c993b Prepare some more queries. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 88 diff changeset	131 if (!preparedTryAddQuery->exec()) {
7b7a959c993b Prepare some more queries. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 88 diff changeset	132 throw SQLException(*preparedTryAddQuery);
2 2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	133 }
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	134 return true;
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	135 }
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	136
64 b9515dc35fe4 Make sure no file has greater linewidth than 80. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: 62 diff changeset	137 void SqliteDBLink::updateFile(const QString& path,
86 af7962f3274b Use quint64 for sizes, rather than qint64. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 85 diff changeset	138 quint64 size, const QDateTime& dtime,
92 f49023c61dac Support for bulk insertion. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 90 diff changeset	139 const QByteArray& hash, bool lazy)
7 d6fdca3bf24e Make sure everything works for MemoryDBLink. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: 5 diff changeset	140 {
92 f49023c61dac Support for bulk insertion. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 90 diff changeset	141 updateFile(DBInfo(path, size, dtime, hash), lazy);
7 d6fdca3bf24e Make sure everything works for MemoryDBLink. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: 5 diff changeset	142 }
d6fdca3bf24e Make sure everything works for MemoryDBLink. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: 5 diff changeset	143
92 f49023c61dac Support for bulk insertion. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 90 diff changeset	144 void SqliteDBLink::updateFile(const DBInfo& dbinfo, bool lazy)
2 2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	145 {
93 308a718812ba Small refactoring to allow lazy commits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 92 diff changeset	146 if (lazy) {
101 6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	147 operations.push_back(new UpdateOperation(dbinfo));
93 308a718812ba Small refactoring to allow lazy commits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 92 diff changeset	148 }
308a718812ba Small refactoring to allow lazy commits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 92 diff changeset	149 else {
94 93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	150 preparedUpdateQuery->bindValue(":path", dbinfo.path());
93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	151 preparedUpdateQuery->bindValue(":size", dbinfo.size());
93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	152 preparedUpdateQuery->bindValue(":mtime", dbinfo.mtime());
93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	153 preparedUpdateQuery->bindValue(":checksum", dbinfo.checksum());
93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	154 if (!preparedUpdateQuery->exec()) {
93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	155 throw SQLException(*preparedUpdateQuery);
93 308a718812ba Small refactoring to allow lazy commits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 92 diff changeset	156 }
2 2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	157 }
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	158 }
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	159
92 f49023c61dac Support for bulk insertion. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 90 diff changeset	160 void SqliteDBLink::addFile(const DBInfo& dbinfo, bool lazy)
2 2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	161 {
93 308a718812ba Small refactoring to allow lazy commits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 92 diff changeset	162 if (lazy) {
101 6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	163 operations.push_back(new AddOperation(dbinfo));
93 308a718812ba Small refactoring to allow lazy commits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 92 diff changeset	164 }
308a718812ba Small refactoring to allow lazy commits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 92 diff changeset	165 else {
308a718812ba Small refactoring to allow lazy commits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 92 diff changeset	166 if (!tryAddFile(dbinfo)) {
308a718812ba Small refactoring to allow lazy commits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 92 diff changeset	167 abort(); //Should throw exception
308a718812ba Small refactoring to allow lazy commits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 92 diff changeset	168 }
2 2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	169 }
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	170 }
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	171
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	172
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	173 QStringList SqliteDBLink::toStringList()
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	174 {
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	175 abort();
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	176 QStringList list;
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	177 /*
64 b9515dc35fe4 Make sure no file has greater linewidth than 80. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: 62 diff changeset	178 foreach(dbinf_ptr_t info, entries) {
2 2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	179 list << info->serialize();
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	180 }
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	181 */
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	182 return list;
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	183 }
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	184
64 b9515dc35fe4 Make sure no file has greater linewidth than 80. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: 62 diff changeset	185 const QList<FileDBLink::dbinf_ptr_t >
b9515dc35fe4 Make sure no file has greater linewidth than 80. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: 62 diff changeset	186 SqliteDBLink::values(const QString& prefix) const
2 2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	187 {
64 b9515dc35fe4 Make sure no file has greater linewidth than 80. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: 62 diff changeset	188 QList<FileDBLink::dbinf_ptr_t > values;
2 2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	189
16 06166d6c083b Add configuration processing. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: 9 diff changeset	190 QSqlQuery query(db);
2 2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	191
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	192 if (prefix.size() > 0) {
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	193 query.prepare("SELECT * FROM files WHERE path LIKE :prefix");
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	194 query.bindValue(":prefix", QString("%1%").arg(prefix));
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	195 }
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	196 else {
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	197 query.prepare("SELECT * FROM files");
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	198 }
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	199
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	200 if (!query.exec()) {
31 bf3dce7fedcb Remove all references to QDebug Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: 16 diff changeset	201 throw SQLException(query);
2 2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	202 }
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	203
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	204 int pathIndex = query.record().indexOf("path");
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	205 int sizeIndex = query.record().indexOf("size");
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	206 int dateIndex = query.record().indexOf("mtime");
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	207 int checksumIndex = query.record().indexOf("checksum");
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	208 while (query.next()) {
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	209 QString path = query.value(pathIndex).toString();
86 af7962f3274b Use quint64 for sizes, rather than qint64. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 85 diff changeset	210 quint64 size = query.value(sizeIndex).toInt();
2 2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	211 QDateTime mtime = query.value(dateIndex).toDateTime();
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	212 QByteArray checksum = query.value(checksumIndex).toByteArray();
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	213
64 b9515dc35fe4 Make sure no file has greater linewidth than 80. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: 62 diff changeset	214 values <<
b9515dc35fe4 Make sure no file has greater linewidth than 80. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: 62 diff changeset	215 FileDBLink::dbinf_ptr_t(new FileDBLink::DBInfo(path,
b9515dc35fe4 Make sure no file has greater linewidth than 80. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: 62 diff changeset	216 size, mtime, checksum));
2 2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	217 }
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	218
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	219 return values;
2833b7f8884a Sql backend is working. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: diff changeset	220 }
5 5e4985407feb Add commandline tool updateDeDupe. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: 2 diff changeset	221
101 6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	222 void SqliteDBLink::deleteFileFromDB(const QString& path, bool lazy)
5 5e4985407feb Add commandline tool updateDeDupe. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: 2 diff changeset	223 {
101 6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	224 if (lazy) {
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	225 operations.push_back(new DeleteOperation(path));
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	226 }
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	227 else {
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	228 QSqlQuery query(db);
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	229 query.prepare("DELETE FROM files WHERE path = :path");
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	230 query.bindValue(":path", path);
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	231 if (!query.exec()) {
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	232 throw SQLException(query);
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	233 }
16 06166d6c083b Add configuration processing. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: 9 diff changeset	234 }
5 5e4985407feb Add commandline tool updateDeDupe. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: 2 diff changeset	235 }
5e4985407feb Add commandline tool updateDeDupe. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: 2 diff changeset	236
5e4985407feb Add commandline tool updateDeDupe. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: 2 diff changeset	237
64 b9515dc35fe4 Make sure no file has greater linewidth than 80. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: 62 diff changeset	238 void SqliteDBLink::keepOnlyFromPrefix(const QString& prefix,
101 6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	239 const QStringList& files,
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	240 bool lazy)
5 5e4985407feb Add commandline tool updateDeDupe. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: 2 diff changeset	241 {
108 e9b798e80bad Setup prepared begin, end queries. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 104 diff changeset	242 QString keepQuerySQL =
e9b798e80bad Setup prepared begin, end queries. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 104 diff changeset	243 "DELETE FROM files WHERE %1 path NOT IN ( %2 )";
e9b798e80bad Setup prepared begin, end queries. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 104 diff changeset	244
e9b798e80bad Setup prepared begin, end queries. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 104 diff changeset	245 bool first = true;
e9b798e80bad Setup prepared begin, end queries. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 104 diff changeset	246 QString inList = "";
e9b798e80bad Setup prepared begin, end queries. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 104 diff changeset	247 foreach (const QString& file, files) {
e9b798e80bad Setup prepared begin, end queries. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 104 diff changeset	248 if (first)
e9b798e80bad Setup prepared begin, end queries. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 104 diff changeset	249 first = false;
e9b798e80bad Setup prepared begin, end queries. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 104 diff changeset	250 else
e9b798e80bad Setup prepared begin, end queries. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 104 diff changeset	251 inList+=", ";
e9b798e80bad Setup prepared begin, end queries. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 104 diff changeset	252
e9b798e80bad Setup prepared begin, end queries. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 104 diff changeset	253 inList+="'"+QString(file).replace("'","''")+"'";
5 5e4985407feb Add commandline tool updateDeDupe. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: 2 diff changeset	254 }
108 e9b798e80bad Setup prepared begin, end queries. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 104 diff changeset	255
e9b798e80bad Setup prepared begin, end queries. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 104 diff changeset	256 if (prefix.isEmpty()) {
e9b798e80bad Setup prepared begin, end queries. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 104 diff changeset	257 keepQuerySQL = keepQuerySQL.arg("");
e9b798e80bad Setup prepared begin, end queries. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 104 diff changeset	258 }
e9b798e80bad Setup prepared begin, end queries. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 104 diff changeset	259 else {
e9b798e80bad Setup prepared begin, end queries. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 104 diff changeset	260 keepQuerySQL = keepQuerySQL.arg("path LIKE :prefix1 AND ");
e9b798e80bad Setup prepared begin, end queries. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 104 diff changeset	261 }
e9b798e80bad Setup prepared begin, end queries. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 104 diff changeset	262 keepQuerySQL = keepQuerySQL.arg(inList);
e9b798e80bad Setup prepared begin, end queries. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 104 diff changeset	263 QSqlQuery keepQuery(db);
e9b798e80bad Setup prepared begin, end queries. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 104 diff changeset	264 keepQuery.prepare(keepQuerySQL);
e9b798e80bad Setup prepared begin, end queries. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 104 diff changeset	265 if (prefix.size() > 0) {
e9b798e80bad Setup prepared begin, end queries. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 104 diff changeset	266 keepQuery.bindValue(":prefix", QString("%1%").arg(prefix));
e9b798e80bad Setup prepared begin, end queries. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 104 diff changeset	267 }
e9b798e80bad Setup prepared begin, end queries. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 104 diff changeset	268 if (!keepQuery.exec()) {
e9b798e80bad Setup prepared begin, end queries. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 104 diff changeset	269 throw SQLException(keepQuery);
5 5e4985407feb Add commandline tool updateDeDupe. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: 2 diff changeset	270 }
5e4985407feb Add commandline tool updateDeDupe. Tom Fredrik Blenning Klaussen <bfg@blenning.no> parents: 2 diff changeset	271 }
85 1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 81 diff changeset	272
86 af7962f3274b Use quint64 for sizes, rather than qint64. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 85 diff changeset	273 const QList<FileDBLink::dbinf_ptr_t>
af7962f3274b Use quint64 for sizes, rather than qint64. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 85 diff changeset	274 SqliteDBLink::filesWithSize(quint64 size, const QString& prefix) const
85 1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 81 diff changeset	275 {
88 6e1d4d2fc49b Prepare size query only once. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 87 diff changeset	276 QList<dbinf_ptr_t > values;
87 9e337bd96bd3 Improve efficiency of SqliteDBLink::filesWithSize Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 86 diff changeset	277
88 6e1d4d2fc49b Prepare size query only once. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 87 diff changeset	278 QSqlQuery & query = (prefix.size() > 0) ? preparedSizePrefixQuery : preparedSizeQuery;
87 9e337bd96bd3 Improve efficiency of SqliteDBLink::filesWithSize Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 86 diff changeset	279
9e337bd96bd3 Improve efficiency of SqliteDBLink::filesWithSize Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 86 diff changeset	280 if (prefix.size() > 0) {
9e337bd96bd3 Improve efficiency of SqliteDBLink::filesWithSize Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 86 diff changeset	281 query.bindValue(":prefix", QString("%1%").arg(prefix));
9e337bd96bd3 Improve efficiency of SqliteDBLink::filesWithSize Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 86 diff changeset	282 }
9e337bd96bd3 Improve efficiency of SqliteDBLink::filesWithSize Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 86 diff changeset	283 query.bindValue(":size", QString("%1%").arg(size));
9e337bd96bd3 Improve efficiency of SqliteDBLink::filesWithSize Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 86 diff changeset	284
9e337bd96bd3 Improve efficiency of SqliteDBLink::filesWithSize Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 86 diff changeset	285
9e337bd96bd3 Improve efficiency of SqliteDBLink::filesWithSize Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 86 diff changeset	286 if (!query.exec()) {
9e337bd96bd3 Improve efficiency of SqliteDBLink::filesWithSize Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 86 diff changeset	287 throw SQLException(query);
9e337bd96bd3 Improve efficiency of SqliteDBLink::filesWithSize Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 86 diff changeset	288 }
9e337bd96bd3 Improve efficiency of SqliteDBLink::filesWithSize Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 86 diff changeset	289
9e337bd96bd3 Improve efficiency of SqliteDBLink::filesWithSize Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 86 diff changeset	290 int pathIndex = query.record().indexOf("path");
9e337bd96bd3 Improve efficiency of SqliteDBLink::filesWithSize Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 86 diff changeset	291 int sizeIndex = query.record().indexOf("size");
9e337bd96bd3 Improve efficiency of SqliteDBLink::filesWithSize Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 86 diff changeset	292 int dateIndex = query.record().indexOf("mtime");
9e337bd96bd3 Improve efficiency of SqliteDBLink::filesWithSize Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 86 diff changeset	293 int checksumIndex = query.record().indexOf("checksum");
9e337bd96bd3 Improve efficiency of SqliteDBLink::filesWithSize Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 86 diff changeset	294 while (query.next()) {
9e337bd96bd3 Improve efficiency of SqliteDBLink::filesWithSize Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 86 diff changeset	295 QString path = query.value(pathIndex).toString();
9e337bd96bd3 Improve efficiency of SqliteDBLink::filesWithSize Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 86 diff changeset	296 quint64 size = query.value(sizeIndex).toInt();
9e337bd96bd3 Improve efficiency of SqliteDBLink::filesWithSize Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 86 diff changeset	297 QDateTime mtime = query.value(dateIndex).toDateTime();
9e337bd96bd3 Improve efficiency of SqliteDBLink::filesWithSize Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 86 diff changeset	298 QByteArray checksum = query.value(checksumIndex).toByteArray();
9e337bd96bd3 Improve efficiency of SqliteDBLink::filesWithSize Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 86 diff changeset	299
9e337bd96bd3 Improve efficiency of SqliteDBLink::filesWithSize Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 86 diff changeset	300 values <<
9e337bd96bd3 Improve efficiency of SqliteDBLink::filesWithSize Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 86 diff changeset	301 FileDBLink::dbinf_ptr_t(new FileDBLink::DBInfo(path,
9e337bd96bd3 Improve efficiency of SqliteDBLink::filesWithSize Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 86 diff changeset	302 size, mtime, checksum));
9e337bd96bd3 Improve efficiency of SqliteDBLink::filesWithSize Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 86 diff changeset	303 }
9e337bd96bd3 Improve efficiency of SqliteDBLink::filesWithSize Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 86 diff changeset	304
9e337bd96bd3 Improve efficiency of SqliteDBLink::filesWithSize Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 86 diff changeset	305 return values;
85 1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 81 diff changeset	306 }
93 308a718812ba Small refactoring to allow lazy commits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 92 diff changeset	307
94 93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	308 void SqliteDBLink::executeOperation(QVariantList& paths,
93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	309 QVariantList& sizes,
93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	310 QVariantList& mtimes,
93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	311 QVariantList& checksums,
93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	312 OperationType operation)
93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	313 {
101 6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	314 assert(sizes.size() == 0 \|\| paths.size() == sizes.size());
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	315 assert(mtimes.size() == 0 \|\| paths.size() == mtimes.size());
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	316 assert(checksums.size() == 0 \|\| paths.size() == checksums.size());
94 93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	317 QSqlQuery* query;
101 6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	318 bool infoQuery;
94 93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	319 switch (operation) {
93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	320 case Add:
114 d4e337567960 Add some debug statements and fix a bug in SQL execution. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 109 diff changeset	321 qDebug() << "Add";
94 93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	322 query = preparedTryAddQuery;
101 6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	323 infoQuery = true;
94 93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	324 break;
93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	325 case Update:
93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	326 query = preparedUpdateQuery;
101 6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	327 infoQuery = true;
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	328 break;
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	329 case Delete:
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	330 query = preparedDeleteQuery;
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	331 infoQuery = false;
94 93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	332 break;
93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	333 case None:
93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	334 assert(paths.size() == 0);
93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	335 return;
93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	336 }
114 d4e337567960 Add some debug statements and fix a bug in SQL execution. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 109 diff changeset	337 if (operation != Update)
d4e337567960 Add some debug statements and fix a bug in SQL execution. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 109 diff changeset	338 query->bindValue("path", paths);
101 6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	339 if (infoQuery) {
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	340 query->bindValue("size", sizes);
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	341 query->bindValue("mtime", mtimes);
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	342 query->bindValue("checksum", checksums);
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	343 }
114 d4e337567960 Add some debug statements and fix a bug in SQL execution. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 109 diff changeset	344 //Because QT for some reason expect the parameters to be given in sequence
d4e337567960 Add some debug statements and fix a bug in SQL execution. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 109 diff changeset	345 if (operation == Update)
d4e337567960 Add some debug statements and fix a bug in SQL execution. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 109 diff changeset	346 query->bindValue("path", paths);
d4e337567960 Add some debug statements and fix a bug in SQL execution. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 109 diff changeset	347
94 93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	348
93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	349 if (!query->execBatch())
93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	350 throw SQLException(*query);
93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	351
93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	352 paths.clear();
93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	353 sizes.clear();
93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	354 mtimes.clear();
93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	355 checksums.clear();
93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	356 }
93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	357
101 6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	358 const char* SqliteDBLink::typeString(OperationType type)
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	359 {
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	360 switch(type) {
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	361 case Add:
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	362 return "Add";
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	363 case Update:
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	364 return "Update";
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	365 case Delete:
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	366 return "Delete";
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	367 case None:
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	368 return "None";
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	369 }
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	370
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	371 return "";
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	372 }
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	373
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	374
104 6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	375 bool SqliteDBLink::commit(const QString& prefix, bool dirty)
93 308a718812ba Small refactoring to allow lazy commits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 92 diff changeset	376 {
94 93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	377 OperationType last = None;
93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	378 QVariantList paths, sizes, mtimes, hashes;
93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	379
104 6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	380 const Operation* operation =
6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	381 (operations.empty()) ? NULL : operations.takeFirst();
6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	382
6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	383 while (operation) {
101 6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	384 if (operation->type() != last) {
114 d4e337567960 Add some debug statements and fix a bug in SQL execution. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 109 diff changeset	385 qDebug() << "Execute Operation first" << typeString(last);
d4e337567960 Add some debug statements and fix a bug in SQL execution. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 109 diff changeset	386 foreach(QVariant path, paths) {
d4e337567960 Add some debug statements and fix a bug in SQL execution. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 109 diff changeset	387 qDebug() << path.toString();
d4e337567960 Add some debug statements and fix a bug in SQL execution. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 109 diff changeset	388 }
109 6cf5eb5b0be1 Embed batch operations in BEGIN-END TRANSACTION for speed improvement. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 108 diff changeset	389 if (!preparedBeginQuery->exec())
6cf5eb5b0be1 Embed batch operations in BEGIN-END TRANSACTION for speed improvement. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 108 diff changeset	390 throw SQLException(*preparedBeginQuery);
94 93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	391 executeOperation(paths, sizes, mtimes, hashes, last);
109 6cf5eb5b0be1 Embed batch operations in BEGIN-END TRANSACTION for speed improvement. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 108 diff changeset	392 if (!preparedEndQuery->exec())
6cf5eb5b0be1 Embed batch operations in BEGIN-END TRANSACTION for speed improvement. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 108 diff changeset	393 throw SQLException(*preparedEndQuery);
114 d4e337567960 Add some debug statements and fix a bug in SQL execution. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 109 diff changeset	394 qDebug() << "Execute Operation Done";
94 93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	395 }
93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	396
101 6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	397 switch (operation->type()) {
93 308a718812ba Small refactoring to allow lazy commits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 92 diff changeset	398 case Add:
101 6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	399 case Update: {
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	400 const InfoOperation* iOperation = dynamic_cast<const InfoOperation*>(operation);
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	401 paths.push_back(iOperation->info().path());
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	402 sizes.push_back(iOperation->info().size());
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	403 mtimes.push_back(iOperation->info().mtime());
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	404 hashes.push_back(iOperation->info().checksum());
104 6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	405 dirty = true;
101 6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	406 break;
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	407 }
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	408 case Delete: {
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	409 const DeleteOperation* dOperation = dynamic_cast<const DeleteOperation*>(operation);
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	410 paths.push_back(dOperation->path());
6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	411 }
94 93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	412 case None:
93 308a718812ba Small refactoring to allow lazy commits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 92 diff changeset	413 break;
308a718812ba Small refactoring to allow lazy commits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 92 diff changeset	414 }
101 6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	415 last = operation->type();
104 6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	416 delete operation;
6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	417 operation = (operations.empty()) ? NULL : operations.takeFirst();
94 93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	418 }
93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	419 if (last != None) {
101 6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	420 qDebug() << "Execute Operation" << typeString(last);
104 6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	421 foreach(QVariant path, paths) {
6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	422 qDebug() << path.toString();
6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	423 }
109 6cf5eb5b0be1 Embed batch operations in BEGIN-END TRANSACTION for speed improvement. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 108 diff changeset	424 if (!preparedBeginQuery->exec())
6cf5eb5b0be1 Embed batch operations in BEGIN-END TRANSACTION for speed improvement. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 108 diff changeset	425 throw SQLException(*preparedBeginQuery);
94 93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	426 executeOperation(paths, sizes, mtimes, hashes, last);
109 6cf5eb5b0be1 Embed batch operations in BEGIN-END TRANSACTION for speed improvement. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 108 diff changeset	427 if (!preparedEndQuery->exec())
6cf5eb5b0be1 Embed batch operations in BEGIN-END TRANSACTION for speed improvement. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 108 diff changeset	428 throw SQLException(*preparedEndQuery);
101 6c6f3a5f96ea Better support for batch operations. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 96 diff changeset	429 qDebug() << "Execute Operation Done";
94 93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	430 }
93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	431
104 6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	432 if (dirty) {
6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	433 QSqlQuery whatToUpdate(db);
6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	434 QString whatToUpdateQuery =
6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	435 "SELECT path FROM files WHERE checksum is NULL %1 AND size in "
6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	436 "(SELECT size FROM files WHERE size <> 0 %2 "
6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	437 "GROUP BY size HAVING count(*) > 1) ORDER BY size";
6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	438 if (prefix.isEmpty()) {
6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	439 whatToUpdateQuery = whatToUpdateQuery.arg("");
6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	440 }
6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	441 else {
6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	442 whatToUpdateQuery = whatToUpdateQuery.arg("AND path LIKE :prefix1");
6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	443 whatToUpdateQuery = whatToUpdateQuery.arg("AND path LIKE :prefix2");
6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	444 }
6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	445 whatToUpdate.prepare(whatToUpdateQuery);
6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	446 if (!prefix.isEmpty()) {
6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	447 whatToUpdate.bindValue("prefix1", QString("%1%").arg(prefix));
6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	448 whatToUpdate.bindValue("prefix2", QString("%1%").arg(prefix));
6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	449 }
94 93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	450
104 6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	451 qDebug() << "Before whatToUpdate";
6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	452 if (!whatToUpdate.exec()) {
6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	453 throw SQLException(whatToUpdate);
6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	454 }
6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	455 qDebug() << "After whatToUpdate";
94 93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	456
104 6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	457 int pathIndex = whatToUpdate.record().indexOf("path");
6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	458 QStringList updatePaths;
6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	459 while (whatToUpdate.next()) {
6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	460 updatePaths << whatToUpdate.value(pathIndex).toString();
6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	461 }
6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	462 int n = 0;
6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	463 int max = updatePaths.size();
6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	464 emit progressUpdate(0, max);
6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	465 QSqlQuery updateChecksum(db);
6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	466 updateChecksum.prepare("UPDATE files "
6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	467 "SET checksum=:checksum "
6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	468 "WHERE path=:path");
94 93981e675d67 Full support for bulkcommits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 93 diff changeset	469
104 6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	470 foreach (const QString& path, updatePaths) {
6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	471 qDebug() << path;
6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	472 QByteArray ohash = computeHash(path);
6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	473 emit progressUpdate(++n, max);
6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	474 updateChecksum.bindValue("checksum", ohash);
6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	475 updateChecksum.bindValue("path", path);
6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	476 if (!updateChecksum.exec())
6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	477 throw SQLException(updateChecksum);
6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	478
6bc013d5788b Avoid unnecessary updates. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 101 diff changeset	479 }
93 308a718812ba Small refactoring to allow lazy commits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 92 diff changeset	480 }
308a718812ba Small refactoring to allow lazy commits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 92 diff changeset	481 return true;
308a718812ba Small refactoring to allow lazy commits. Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> parents: 92 diff changeset	482 }

Mercurial > dedupe

annotate SqliteDBLink.cpp @ 114:d4e337567960