Mercurial > dedupe

annotate FileDBLink.cpp @ 91:a5788991ca9f

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Refactor lazy update, with one common routine.
author Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
date Mon, 21 Oct 2013 16:21:54 +0200
parents af7962f3274b
children f49023c61dac
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
40
f711ddb56ae7 Sort up includes.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 34
diff changeset
1 #include "FileDBLink.hpp"
f711ddb56ae7 Sort up includes.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 34
diff changeset
2
78
9744ec195be3 Encapsulate EditDistance with caching.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 64
diff changeset
3 #include "CachedEditDistance.hpp"
40
f711ddb56ae7 Sort up includes.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 34
diff changeset
4
28
b2c2c2bf2bbd Refactor Exceptions into a separate directory.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 15
diff changeset
5 #include "Exception/PermissionException.hpp"
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
6
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
7 #include <QtCore/QtConcurrentMap>
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
8
40
f711ddb56ae7 Sort up includes.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 34
diff changeset
9 #include <boost/bind.hpp>
f711ddb56ae7 Sort up includes.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 34
diff changeset
10
85
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
11 void FileDBLink::updateIfModified(const QString& path, bool lazy)
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
12 {
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
13 QFileInfo fileinfo(path);
2
2833b7f8884a Sql backend is working.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 1
diff changeset
14 FileDBLink::DBStatus status = existsWithMtime(path, fileinfo.lastModified());
2833b7f8884a Sql backend is working.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 1
diff changeset
15
2833b7f8884a Sql backend is working.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 1
diff changeset
16 switch (status) {
2833b7f8884a Sql backend is working.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 1
diff changeset
17 case FileDBLink::NONE: {
85
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
18 addFile(fileinfo, lazy);
2
2833b7f8884a Sql backend is working.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 1
diff changeset
19 break;
2833b7f8884a Sql backend is working.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 1
diff changeset
20 }
2833b7f8884a Sql backend is working.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 1
diff changeset
21 case FileDBLink::MTIME_DIFFERENT: {
85
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
22 updateFile(fileinfo, lazy);
2
2833b7f8884a Sql backend is working.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 1
diff changeset
23 }
2833b7f8884a Sql backend is working.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 1
diff changeset
24 default: {
2833b7f8884a Sql backend is working.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 1
diff changeset
25 }
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
26 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
27 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
28
91
a5788991ca9f Refactor lazy update, with one common routine.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 86
diff changeset
29 void FileDBLink::addFile(const QString& path, quint64 size,
a5788991ca9f Refactor lazy update, with one common routine.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 86
diff changeset
30 const QDateTime& lastModified, bool lazy)
a5788991ca9f Refactor lazy update, with one common routine.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 86
diff changeset
31 {
a5788991ca9f Refactor lazy update, with one common routine.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 86
diff changeset
32 QByteArray hash;
a5788991ca9f Refactor lazy update, with one common routine.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 86
diff changeset
33 //if (!lazy || updateAllWithSize(size))
a5788991ca9f Refactor lazy update, with one common routine.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 86
diff changeset
34 if (!lazy)
a5788991ca9f Refactor lazy update, with one common routine.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 86
diff changeset
35 hash = computeHash(path);
a5788991ca9f Refactor lazy update, with one common routine.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 86
diff changeset
36 addFile(path, size, lastModified, hash);
a5788991ca9f Refactor lazy update, with one common routine.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 86
diff changeset
37 }
a5788991ca9f Refactor lazy update, with one common routine.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 86
diff changeset
38
85
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
39 void FileDBLink::addFile(const QFileInfo& fileinfo, bool lazy)
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
40 {
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
41 addFile(fileinfo.absoluteFilePath(), fileinfo.size(),
85
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
42 fileinfo.lastModified(), lazy);
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
43 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
44
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
45 QByteArray FileDBLink::computeHash(const QString& path,
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
46 QCryptographicHash::Algorithm algorithm)
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
47 {
15
199fc63c60c1 Increase buffersize for filehash computing.
Tom Fredrik Blenning Klaussen <bfg@sim.no>
parents: 13
diff changeset
48 const static uint buffersize = 32768;
13
9463c0c22969 Refactor computeHash into separate function.
Tom Fredrik Blenning Klaussen <bfg@sim.no>
parents: 9
diff changeset
49 QCryptographicHash hash(algorithm);
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
50 QFile file(path);
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
51 if ( file.open( QIODevice::ReadOnly ) ) {
13
9463c0c22969 Refactor computeHash into separate function.
Tom Fredrik Blenning Klaussen <bfg@sim.no>
parents: 9
diff changeset
52 while(!file.atEnd()){
9463c0c22969 Refactor computeHash into separate function.
Tom Fredrik Blenning Klaussen <bfg@sim.no>
parents: 9
diff changeset
53 hash.addData(file.read(buffersize));
9463c0c22969 Refactor computeHash into separate function.
Tom Fredrik Blenning Klaussen <bfg@sim.no>
parents: 9
diff changeset
54 }
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
55 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
56 else {
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
57 QString errorMsg = path + ": " + file.errorString();
9
b5943e4bf676 Fix up header includes.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 2
diff changeset
58
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
59 switch (file.error()) {
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
60 case QFile::PermissionsError:
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
61 throw PermissionException(errorMsg);
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
62 default:
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
63 throw IOException(errorMsg);
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
64 }
34
fda70a362ed5 Remove whitespace.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 31
diff changeset
65 }
13
9463c0c22969 Refactor computeHash into separate function.
Tom Fredrik Blenning Klaussen <bfg@sim.no>
parents: 9
diff changeset
66 return hash.result();
9463c0c22969 Refactor computeHash into separate function.
Tom Fredrik Blenning Klaussen <bfg@sim.no>
parents: 9
diff changeset
67 }
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
68
86
af7962f3274b Use quint64 for sizes, rather than qint64.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 85
diff changeset
69 bool FileDBLink::updateAllWithSize(quint64 size)
85
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
70 {
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
71 const QList<dbinf_ptr_t> others = filesWithSize(size);
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
72 if (!others.empty()) {
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
73 foreach( const dbinf_ptr_t other, others) {
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
74 if (other->checksum().isEmpty()) {
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
75 QByteArray ohash = computeHash(other->path());
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
76 updateFile(other->path(), other->size(), other->mtime(), ohash);
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
77 }
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
78 }
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
79 return true;
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
80 }
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
81 return false;
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
82 }
13
9463c0c22969 Refactor computeHash into separate function.
Tom Fredrik Blenning Klaussen <bfg@sim.no>
parents: 9
diff changeset
83
85
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
84 void FileDBLink::updateFile(const QFileInfo& fileinfo, bool lazy)
2
2833b7f8884a Sql backend is working.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 1
diff changeset
85 {
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
86 updateFile(fileinfo.absoluteFilePath(), fileinfo.size(),
85
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
87 fileinfo.lastModified(), lazy);
2
2833b7f8884a Sql backend is working.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 1
diff changeset
88 }
2833b7f8884a Sql backend is working.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 1
diff changeset
89
86
af7962f3274b Use quint64 for sizes, rather than qint64.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 85
diff changeset
90 void FileDBLink::updateFile(const QString& path, quint64 size,
85
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
91 const QDateTime& lastModified, bool lazy)
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
92 {
85
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
93 QByteArray hash;
91
a5788991ca9f Refactor lazy update, with one common routine.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 86
diff changeset
94 //if (!lazy || updateAllWithSize(size))
a5788991ca9f Refactor lazy update, with one common routine.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 86
diff changeset
95 if (!lazy)
85
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
96 hash = computeHash(path);
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
97 updateFile(path, size, lastModified, hash);
2
2833b7f8884a Sql backend is working.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 1
diff changeset
98 }
2833b7f8884a Sql backend is working.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 1
diff changeset
99
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
100 const QList<FileDBLink::dbinf_ptr_t >
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
101 FileDBLink::sortOn(const QString& prefix, SORTORDER order, bool extended)
2
2833b7f8884a Sql backend is working.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 1
diff changeset
102 {
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
103 QList<dbinf_ptr_t > list =
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
104 (extended) ? computedValues(prefix) : values(prefix);
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
105
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
106 switch (order) {
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
107 case PATH:
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
108 {
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
109 QList<FileDBLink::dbinf_ptr_t > oList;
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
110 foreach(dbinf_ptr_t info, list) {
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
111 oList.push_back(info);
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
112 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
113 return oList;
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
114 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
115 case SIZE:
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
116 {
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
117 QMultiMap<quint64, dbinf_ptr_t > oList;
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
118 foreach(dbinf_ptr_t info, list) {
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
119 oList.insert(info->size(), info);
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
120 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
121 return oList.values();
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
122 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
123 case MTIME:
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
124 {
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
125 QMultiMap<QDateTime, dbinf_ptr_t > oList;
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
126 foreach(dbinf_ptr_t info, list) {
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
127 oList.insert(info->mtime(), info);
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
128 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
129 return oList.values();
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
130 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
131 case CHECKSUM:
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
132 {
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
133 QMultiMap<QByteArray, dbinf_ptr_t > oList;
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
134 foreach(dbinf_ptr_t info, list) {
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
135 oList.insert(info->checksum(), info);
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
136 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
137 return oList.values();
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
138 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
139 case EDIT:
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
140 {
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
141 assert(extended);
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
142 QMultiMap<int, dbinf_ptr_t > oList;
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
143 foreach(dbinf_ptr_t info, list) {
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
144 QSharedPointer<ExtendedDBInfo> ptr;
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
145 ptr = info.dynamicCast<ExtendedDBInfo>();
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
146 oList.insert(ptr->editDistance(), info);
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
147 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
148 return oList.values();
34
fda70a362ed5 Remove whitespace.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 31
diff changeset
149
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
150 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
151 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
152 abort();
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
153 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
154
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
155 FileDBLink::dbinf_ptr_t
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
156 FileDBLink::computedValue(const dbinf_ptr_t& info,
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
157 const QList<dbinf_ptr_t >& entries)
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
158 {
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
159 QString p1 = info->name();
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
160 int minDist = 100000;
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
161 QString other;
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
162 for (QList<dbinf_ptr_t >::const_iterator it2 = entries.begin();
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
163 it2 != entries.end(); ++it2) {
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
164 if (info == *it2)
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
165 continue;
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
166 QString p2 = (*it2)->name();
78
9744ec195be3 Encapsulate EditDistance with caching.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 64
diff changeset
167 int dist = CachedEditDistance::Compute(p1, p2, false);
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
168 if (dist < minDist) {
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
169 minDist = dist;
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
170 other = (*it2)->path();
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
171 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
172 }
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
173 return dbinf_ptr_t(new ExtendedDBInfo(*info, other, minDist));
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
174 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
175
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
176 const QList<FileDBLink::dbinf_ptr_t>
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
177 FileDBLink::computedValues(const QString& prefix) const
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
178 {
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
179 QList<dbinf_ptr_t > list;
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
180 QList<dbinf_ptr_t > entries = values(prefix);
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
181
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
182 #if 1
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
183 list =
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
184 QtConcurrent::blockingMapped(entries,
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
185 boost::bind( &FileDBLink::computedValue,
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
186 _1,
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
187 entries));
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
188 #else
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
189 for (QList<dbinf_ptr_t >::const_iterator it1 = entries.begin();
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
190 it1 != entries.end(); ++it1) {
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
191 dbinf_ptr_t ext = computedValue(*it1, entries);
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
192 list.push_back(ext);
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
193 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
194 #endif
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
195 return list;
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
196 }
85
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
197
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
198
86
af7962f3274b Use quint64 for sizes, rather than qint64.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 85
diff changeset
199 const QList<FileDBLink::dbinf_ptr_t>
af7962f3274b Use quint64 for sizes, rather than qint64.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 85
diff changeset
200 FileDBLink::filesWithSize(quint64 size, const QString& prefix) const
85
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
201 {
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
202 QList<dbinf_ptr_t> retVal;
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
203 const QList<dbinf_ptr_t> vals = values(prefix);
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
204 foreach (const dbinf_ptr_t val, vals) {
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
205 QString path = val->path();
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
206 if (val->size() == size) {
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
207 retVal << val;
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
208 }
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
209 }
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
210 return retVal;
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
211 }
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
212
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
213 FileDBLink::dbinf_ptr_t FileDBLink::value(const QString& path) const
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
214 {
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
215 const QList<dbinf_ptr_t> vals = values();
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
216 foreach (const dbinf_ptr_t val, vals) {
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
217 QString vpath = val->path();
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
218 if (vpath == path) {
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
219 return val;
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
220 }
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
221 }
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
222 return dbinf_ptr_t();
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
223 }