Mercurial > dedupe

annotate FileDBLink.cpp @ 115:404795616b1e default tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Added a lot of common files to ignore
author Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
date Sat, 25 Mar 2017 17:43:57 +0100
parents 6bc013d5788b
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
40
f711ddb56ae7 Sort up includes.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 34
diff changeset
1 #include "FileDBLink.hpp"
f711ddb56ae7 Sort up includes.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 34
diff changeset
2
78
9744ec195be3 Encapsulate EditDistance with caching.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 64
diff changeset
3 #include "CachedEditDistance.hpp"
40
f711ddb56ae7 Sort up includes.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 34
diff changeset
4
28
b2c2c2bf2bbd Refactor Exceptions into a separate directory.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 15
diff changeset
5 #include "Exception/PermissionException.hpp"
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
6
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
7 #include <QtCore/QtConcurrentMap>
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
8
40
f711ddb56ae7 Sort up includes.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 34
diff changeset
9 #include <boost/bind.hpp>
f711ddb56ae7 Sort up includes.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 34
diff changeset
10
92
f49023c61dac Support for bulk insertion.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 91
diff changeset
11 FileDBLink::~FileDBLink()
f49023c61dac Support for bulk insertion.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 91
diff changeset
12 {
f49023c61dac Support for bulk insertion.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 91
diff changeset
13 commit();
f49023c61dac Support for bulk insertion.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 91
diff changeset
14 }
f49023c61dac Support for bulk insertion.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 91
diff changeset
15
85
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
16 void FileDBLink::updateIfModified(const QString& path, bool lazy)
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
17 {
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
18 QFileInfo fileinfo(path);
2
2833b7f8884a Sql backend is working.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 1
diff changeset
19 FileDBLink::DBStatus status = existsWithMtime(path, fileinfo.lastModified());
2833b7f8884a Sql backend is working.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 1
diff changeset
20
2833b7f8884a Sql backend is working.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 1
diff changeset
21 switch (status) {
2833b7f8884a Sql backend is working.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 1
diff changeset
22 case FileDBLink::NONE: {
85
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
23 addFile(fileinfo, lazy);
2
2833b7f8884a Sql backend is working.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 1
diff changeset
24 break;
2833b7f8884a Sql backend is working.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 1
diff changeset
25 }
2833b7f8884a Sql backend is working.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 1
diff changeset
26 case FileDBLink::MTIME_DIFFERENT: {
85
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
27 updateFile(fileinfo, lazy);
2
2833b7f8884a Sql backend is working.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 1
diff changeset
28 }
2833b7f8884a Sql backend is working.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 1
diff changeset
29 default: {
2833b7f8884a Sql backend is working.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 1
diff changeset
30 }
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
31 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
32 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
33
91
a5788991ca9f Refactor lazy update, with one common routine.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 86
diff changeset
34 void FileDBLink::addFile(const QString& path, quint64 size,
a5788991ca9f Refactor lazy update, with one common routine.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 86
diff changeset
35 const QDateTime& lastModified, bool lazy)
a5788991ca9f Refactor lazy update, with one common routine.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 86
diff changeset
36 {
a5788991ca9f Refactor lazy update, with one common routine.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 86
diff changeset
37 QByteArray hash;
a5788991ca9f Refactor lazy update, with one common routine.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 86
diff changeset
38 if (!lazy)
a5788991ca9f Refactor lazy update, with one common routine.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 86
diff changeset
39 hash = computeHash(path);
92
f49023c61dac Support for bulk insertion.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 91
diff changeset
40 addFile(path, size, lastModified, hash, lazy);
91
a5788991ca9f Refactor lazy update, with one common routine.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 86
diff changeset
41 }
a5788991ca9f Refactor lazy update, with one common routine.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 86
diff changeset
42
85
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
43 void FileDBLink::addFile(const QFileInfo& fileinfo, bool lazy)
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
44 {
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
45 addFile(fileinfo.absoluteFilePath(), fileinfo.size(),
85
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
46 fileinfo.lastModified(), lazy);
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
47 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
48
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
49 QByteArray FileDBLink::computeHash(const QString& path,
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
50 QCryptographicHash::Algorithm algorithm)
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
51 {
15
199fc63c60c1 Increase buffersize for filehash computing.
Tom Fredrik Blenning Klaussen <bfg@sim.no>
parents: 13
diff changeset
52 const static uint buffersize = 32768;
13
9463c0c22969 Refactor computeHash into separate function.
Tom Fredrik Blenning Klaussen <bfg@sim.no>
parents: 9
diff changeset
53 QCryptographicHash hash(algorithm);
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
54 QFile file(path);
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
55 if ( file.open( QIODevice::ReadOnly ) ) {
13
9463c0c22969 Refactor computeHash into separate function.
Tom Fredrik Blenning Klaussen <bfg@sim.no>
parents: 9
diff changeset
56 while(!file.atEnd()){
9463c0c22969 Refactor computeHash into separate function.
Tom Fredrik Blenning Klaussen <bfg@sim.no>
parents: 9
diff changeset
57 hash.addData(file.read(buffersize));
9463c0c22969 Refactor computeHash into separate function.
Tom Fredrik Blenning Klaussen <bfg@sim.no>
parents: 9
diff changeset
58 }
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
59 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
60 else {
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
61 QString errorMsg = path + ": " + file.errorString();
9
b5943e4bf676 Fix up header includes.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 2
diff changeset
62
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
63 switch (file.error()) {
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
64 case QFile::PermissionsError:
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
65 throw PermissionException(errorMsg);
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
66 default:
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
67 throw IOException(errorMsg);
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
68 }
34
fda70a362ed5 Remove whitespace.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 31
diff changeset
69 }
13
9463c0c22969 Refactor computeHash into separate function.
Tom Fredrik Blenning Klaussen <bfg@sim.no>
parents: 9
diff changeset
70 return hash.result();
9463c0c22969 Refactor computeHash into separate function.
Tom Fredrik Blenning Klaussen <bfg@sim.no>
parents: 9
diff changeset
71 }
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
72
86
af7962f3274b Use quint64 for sizes, rather than qint64.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 85
diff changeset
73 bool FileDBLink::updateAllWithSize(quint64 size)
85
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
74 {
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
75 const QList<dbinf_ptr_t> others = filesWithSize(size);
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
76 if (!others.empty()) {
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
77 foreach( const dbinf_ptr_t other, others) {
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
78 if (other->checksum().isEmpty()) {
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
79 QByteArray ohash = computeHash(other->path());
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
80 updateFile(other->path(), other->size(), other->mtime(), ohash);
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
81 }
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
82 }
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
83 return true;
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
84 }
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
85 return false;
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
86 }
13
9463c0c22969 Refactor computeHash into separate function.
Tom Fredrik Blenning Klaussen <bfg@sim.no>
parents: 9
diff changeset
87
85
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
88 void FileDBLink::updateFile(const QFileInfo& fileinfo, bool lazy)
2
2833b7f8884a Sql backend is working.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 1
diff changeset
89 {
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
90 updateFile(fileinfo.absoluteFilePath(), fileinfo.size(),
85
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
91 fileinfo.lastModified(), lazy);
2
2833b7f8884a Sql backend is working.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 1
diff changeset
92 }
2833b7f8884a Sql backend is working.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 1
diff changeset
93
86
af7962f3274b Use quint64 for sizes, rather than qint64.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 85
diff changeset
94 void FileDBLink::updateFile(const QString& path, quint64 size,
85
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
95 const QDateTime& lastModified, bool lazy)
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
96 {
85
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
97 QByteArray hash;
91
a5788991ca9f Refactor lazy update, with one common routine.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 86
diff changeset
98 if (!lazy)
85
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
99 hash = computeHash(path);
93
308a718812ba Small refactoring to allow lazy commits.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 92
diff changeset
100 updateFile(path, size, lastModified, hash, lazy);
2
2833b7f8884a Sql backend is working.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 1
diff changeset
101 }
2833b7f8884a Sql backend is working.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 1
diff changeset
102
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
103 const QList<FileDBLink::dbinf_ptr_t >
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
104 FileDBLink::sortOn(const QString& prefix, SORTORDER order, bool extended)
2
2833b7f8884a Sql backend is working.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 1
diff changeset
105 {
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
106 QList<dbinf_ptr_t > list =
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
107 (extended) ? computedValues(prefix) : values(prefix);
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
108
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
109 switch (order) {
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
110 case PATH:
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
111 {
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
112 QList<FileDBLink::dbinf_ptr_t > oList;
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
113 foreach(dbinf_ptr_t info, list) {
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
114 oList.push_back(info);
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
115 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
116 return oList;
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
117 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
118 case SIZE:
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
119 {
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
120 QMultiMap<quint64, dbinf_ptr_t > oList;
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
121 foreach(dbinf_ptr_t info, list) {
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
122 oList.insert(info->size(), info);
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
123 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
124 return oList.values();
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
125 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
126 case MTIME:
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
127 {
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
128 QMultiMap<QDateTime, dbinf_ptr_t > oList;
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
129 foreach(dbinf_ptr_t info, list) {
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
130 oList.insert(info->mtime(), info);
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
131 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
132 return oList.values();
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
133 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
134 case CHECKSUM:
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
135 {
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
136 QMultiMap<QByteArray, dbinf_ptr_t > oList;
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
137 foreach(dbinf_ptr_t info, list) {
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
138 oList.insert(info->checksum(), info);
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
139 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
140 return oList.values();
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
141 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
142 case EDIT:
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
143 {
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
144 assert(extended);
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
145 QMultiMap<int, dbinf_ptr_t > oList;
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
146 foreach(dbinf_ptr_t info, list) {
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
147 QSharedPointer<ExtendedDBInfo> ptr;
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
148 ptr = info.dynamicCast<ExtendedDBInfo>();
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
149 oList.insert(ptr->editDistance(), info);
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
150 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
151 return oList.values();
34
fda70a362ed5 Remove whitespace.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 31
diff changeset
152
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
153 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
154 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
155 abort();
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
156 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
157
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
158 FileDBLink::dbinf_ptr_t
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
159 FileDBLink::computedValue(const dbinf_ptr_t& info,
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
160 const QList<dbinf_ptr_t >& entries)
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
161 {
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
162 QString p1 = info->name();
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
163 int minDist = 100000;
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
164 QString other;
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
165 for (QList<dbinf_ptr_t >::const_iterator it2 = entries.begin();
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
166 it2 != entries.end(); ++it2) {
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
167 if (info == *it2)
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
168 continue;
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
169 QString p2 = (*it2)->name();
78
9744ec195be3 Encapsulate EditDistance with caching.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 64
diff changeset
170 int dist = CachedEditDistance::Compute(p1, p2, false);
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
171 if (dist < minDist) {
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
172 minDist = dist;
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
173 other = (*it2)->path();
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
174 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
175 }
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
176 return dbinf_ptr_t(new ExtendedDBInfo(*info, other, minDist));
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
177 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
178
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
179 const QList<FileDBLink::dbinf_ptr_t>
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
180 FileDBLink::computedValues(const QString& prefix) const
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
181 {
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
182 QList<dbinf_ptr_t > list;
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
183 QList<dbinf_ptr_t > entries = values(prefix);
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
184
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
185 #if 1
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
186 list =
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
187 QtConcurrent::blockingMapped(entries,
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
188 boost::bind( &FileDBLink::computedValue,
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
189 _1,
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
190 entries));
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
191 #else
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
192 for (QList<dbinf_ptr_t >::const_iterator it1 = entries.begin();
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
193 it1 != entries.end(); ++it1) {
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
194 dbinf_ptr_t ext = computedValue(*it1, entries);
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
195 list.push_back(ext);
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
196 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
197 #endif
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
198 return list;
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
199 }
85
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
200
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
201
86
af7962f3274b Use quint64 for sizes, rather than qint64.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 85
diff changeset
202 const QList<FileDBLink::dbinf_ptr_t>
af7962f3274b Use quint64 for sizes, rather than qint64.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 85
diff changeset
203 FileDBLink::filesWithSize(quint64 size, const QString& prefix) const
85
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
204 {
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
205 QList<dbinf_ptr_t> retVal;
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
206 const QList<dbinf_ptr_t> vals = values(prefix);
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
207 foreach (const dbinf_ptr_t val, vals) {
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
208 QString path = val->path();
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
209 if (val->size() == size) {
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
210 retVal << val;
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
211 }
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
212 }
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
213 return retVal;
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
214 }
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
215
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
216 FileDBLink::dbinf_ptr_t FileDBLink::value(const QString& path) const
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
217 {
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
218 const QList<dbinf_ptr_t> vals = values();
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
219 foreach (const dbinf_ptr_t val, vals) {
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
220 QString vpath = val->path();
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
221 if (vpath == path) {
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
222 return val;
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
223 }
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
224 }
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
225 return dbinf_ptr_t();
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
226 }
92
f49023c61dac Support for bulk insertion.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 91
diff changeset
227
104
6bc013d5788b Avoid unnecessary updates.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 96
diff changeset
228 bool FileDBLink::commit(const QString&, bool)
92
f49023c61dac Support for bulk insertion.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 91
diff changeset
229 {
f49023c61dac Support for bulk insertion.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 91
diff changeset
230 return true;
f49023c61dac Support for bulk insertion.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 91
diff changeset
231 }