Mercurial > dedupe

annotate FileDBLink.cpp @ 85:1f9e27a0bd7f

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Allow for lazy calculation of checksums, ignore them, if only one file of given size.
author Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
date Thu, 10 Oct 2013 15:55:30 +0200
parents 9744ec195be3
children af7962f3274b
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
40
f711ddb56ae7 Sort up includes.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 34
diff changeset
1 #include "FileDBLink.hpp"
f711ddb56ae7 Sort up includes.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 34
diff changeset
2
78
9744ec195be3 Encapsulate EditDistance with caching.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 64
diff changeset
3 #include "CachedEditDistance.hpp"
40
f711ddb56ae7 Sort up includes.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 34
diff changeset
4
28
b2c2c2bf2bbd Refactor Exceptions into a separate directory.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 15
diff changeset
5 #include "Exception/PermissionException.hpp"
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
6
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
7 #include <QtCore/QtConcurrentMap>
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
8
40
f711ddb56ae7 Sort up includes.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 34
diff changeset
9 #include <boost/bind.hpp>
f711ddb56ae7 Sort up includes.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 34
diff changeset
10
85
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
11 void FileDBLink::updateIfModified(const QString& path, bool lazy)
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
12 {
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
13 QFileInfo fileinfo(path);
2
2833b7f8884a Sql backend is working.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 1
diff changeset
14 FileDBLink::DBStatus status = existsWithMtime(path, fileinfo.lastModified());
2833b7f8884a Sql backend is working.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 1
diff changeset
15
2833b7f8884a Sql backend is working.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 1
diff changeset
16 switch (status) {
2833b7f8884a Sql backend is working.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 1
diff changeset
17 case FileDBLink::NONE: {
85
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
18 addFile(fileinfo, lazy);
2
2833b7f8884a Sql backend is working.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 1
diff changeset
19 break;
2833b7f8884a Sql backend is working.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 1
diff changeset
20 }
2833b7f8884a Sql backend is working.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 1
diff changeset
21 case FileDBLink::MTIME_DIFFERENT: {
85
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
22 updateFile(fileinfo, lazy);
2
2833b7f8884a Sql backend is working.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 1
diff changeset
23 }
2833b7f8884a Sql backend is working.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 1
diff changeset
24 default: {
2833b7f8884a Sql backend is working.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 1
diff changeset
25 }
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
26 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
27 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
28
85
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
29 void FileDBLink::addFile(const QFileInfo& fileinfo, bool lazy)
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
30 {
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
31 addFile(fileinfo.absoluteFilePath(), fileinfo.size(),
85
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
32 fileinfo.lastModified(), lazy);
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
33 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
34
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
35 QByteArray FileDBLink::computeHash(const QString& path,
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
36 QCryptographicHash::Algorithm algorithm)
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
37 {
15
199fc63c60c1 Increase buffersize for filehash computing.
Tom Fredrik Blenning Klaussen <bfg@sim.no>
parents: 13
diff changeset
38 const static uint buffersize = 32768;
13
9463c0c22969 Refactor computeHash into separate function.
Tom Fredrik Blenning Klaussen <bfg@sim.no>
parents: 9
diff changeset
39 QCryptographicHash hash(algorithm);
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
40 QFile file(path);
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
41 if ( file.open( QIODevice::ReadOnly ) ) {
13
9463c0c22969 Refactor computeHash into separate function.
Tom Fredrik Blenning Klaussen <bfg@sim.no>
parents: 9
diff changeset
42 while(!file.atEnd()){
9463c0c22969 Refactor computeHash into separate function.
Tom Fredrik Blenning Klaussen <bfg@sim.no>
parents: 9
diff changeset
43 hash.addData(file.read(buffersize));
9463c0c22969 Refactor computeHash into separate function.
Tom Fredrik Blenning Klaussen <bfg@sim.no>
parents: 9
diff changeset
44 }
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
45 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
46 else {
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
47 QString errorMsg = path + ": " + file.errorString();
9
b5943e4bf676 Fix up header includes.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 2
diff changeset
48
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
49 switch (file.error()) {
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
50 case QFile::PermissionsError:
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
51 throw PermissionException(errorMsg);
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
52 default:
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
53 throw IOException(errorMsg);
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
54 }
34
fda70a362ed5 Remove whitespace.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 31
diff changeset
55 }
13
9463c0c22969 Refactor computeHash into separate function.
Tom Fredrik Blenning Klaussen <bfg@sim.no>
parents: 9
diff changeset
56 return hash.result();
9463c0c22969 Refactor computeHash into separate function.
Tom Fredrik Blenning Klaussen <bfg@sim.no>
parents: 9
diff changeset
57 }
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
58
85
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
59 bool FileDBLink::updateAllWithSize(qint64 size)
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
60 {
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
61 const QList<dbinf_ptr_t> others = filesWithSize(size);
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
62 if (!others.empty()) {
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
63 foreach( const dbinf_ptr_t other, others) {
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
64 if (other->checksum().isEmpty()) {
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
65 QByteArray ohash = computeHash(other->path());
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
66 updateFile(other->path(), other->size(), other->mtime(), ohash);
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
67 }
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
68 }
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
69 return true;
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
70 }
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
71 return false;
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
72 }
13
9463c0c22969 Refactor computeHash into separate function.
Tom Fredrik Blenning Klaussen <bfg@sim.no>
parents: 9
diff changeset
73
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
74 void FileDBLink::addFile(const QString& path, qint64 size,
85
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
75 const QDateTime& lastModified, bool lazy)
13
9463c0c22969 Refactor computeHash into separate function.
Tom Fredrik Blenning Klaussen <bfg@sim.no>
parents: 9
diff changeset
76 {
85
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
77 QByteArray hash;
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
78 // std::cout << path.toStdString() << "::" << lazy << std::endl;
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
79 if (!lazy || updateAllWithSize(size))
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
80 hash = computeHash(path);
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
81 addFile(path, size, lastModified, hash);
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
82 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
83
85
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
84 void FileDBLink::updateFile(const QFileInfo& fileinfo, bool lazy)
2
2833b7f8884a Sql backend is working.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 1
diff changeset
85 {
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
86 updateFile(fileinfo.absoluteFilePath(), fileinfo.size(),
85
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
87 fileinfo.lastModified(), lazy);
2
2833b7f8884a Sql backend is working.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 1
diff changeset
88 }
2833b7f8884a Sql backend is working.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 1
diff changeset
89
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
90 void FileDBLink::updateFile(const QString& path, qint64 size,
85
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
91 const QDateTime& lastModified, bool lazy)
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
92 {
85
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
93 QByteArray hash;
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
94 if (!lazy || updateAllWithSize(size))
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
95 hash = computeHash(path);
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
96 updateFile(path, size, lastModified, hash);
2
2833b7f8884a Sql backend is working.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 1
diff changeset
97 }
2833b7f8884a Sql backend is working.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 1
diff changeset
98
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
99 const QList<FileDBLink::dbinf_ptr_t >
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
100 FileDBLink::sortOn(const QString& prefix, SORTORDER order, bool extended)
2
2833b7f8884a Sql backend is working.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 1
diff changeset
101 {
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
102 QList<dbinf_ptr_t > list =
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
103 (extended) ? computedValues(prefix) : values(prefix);
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
104
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
105 switch (order) {
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
106 case PATH:
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
107 {
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
108 QList<FileDBLink::dbinf_ptr_t > oList;
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
109 foreach(dbinf_ptr_t info, list) {
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
110 oList.push_back(info);
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
111 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
112 return oList;
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
113 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
114 case SIZE:
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
115 {
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
116 QMultiMap<quint64, dbinf_ptr_t > oList;
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
117 foreach(dbinf_ptr_t info, list) {
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
118 oList.insert(info->size(), info);
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
119 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
120 return oList.values();
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
121 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
122 case MTIME:
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
123 {
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
124 QMultiMap<QDateTime, dbinf_ptr_t > oList;
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
125 foreach(dbinf_ptr_t info, list) {
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
126 oList.insert(info->mtime(), info);
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
127 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
128 return oList.values();
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
129 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
130 case CHECKSUM:
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
131 {
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
132 QMultiMap<QByteArray, dbinf_ptr_t > oList;
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
133 foreach(dbinf_ptr_t info, list) {
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
134 oList.insert(info->checksum(), info);
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
135 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
136 return oList.values();
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
137 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
138 case EDIT:
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
139 {
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
140 assert(extended);
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
141 QMultiMap<int, dbinf_ptr_t > oList;
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
142 foreach(dbinf_ptr_t info, list) {
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
143 QSharedPointer<ExtendedDBInfo> ptr;
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
144 ptr = info.dynamicCast<ExtendedDBInfo>();
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
145 oList.insert(ptr->editDistance(), info);
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
146 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
147 return oList.values();
34
fda70a362ed5 Remove whitespace.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 31
diff changeset
148
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
149 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
150 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
151 abort();
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
152 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
153
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
154 FileDBLink::dbinf_ptr_t
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
155 FileDBLink::computedValue(const dbinf_ptr_t& info,
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
156 const QList<dbinf_ptr_t >& entries)
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
157 {
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
158 QString p1 = info->name();
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
159 int minDist = 100000;
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
160 QString other;
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
161 for (QList<dbinf_ptr_t >::const_iterator it2 = entries.begin();
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
162 it2 != entries.end(); ++it2) {
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
163 if (info == *it2)
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
164 continue;
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
165 QString p2 = (*it2)->name();
78
9744ec195be3 Encapsulate EditDistance with caching.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 64
diff changeset
166 int dist = CachedEditDistance::Compute(p1, p2, false);
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
167 if (dist < minDist) {
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
168 minDist = dist;
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
169 other = (*it2)->path();
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
170 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
171 }
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
172 return dbinf_ptr_t(new ExtendedDBInfo(*info, other, minDist));
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
173 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
174
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
175 const QList<FileDBLink::dbinf_ptr_t>
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
176 FileDBLink::computedValues(const QString& prefix) const
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
177 {
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
178 QList<dbinf_ptr_t > list;
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
179 QList<dbinf_ptr_t > entries = values(prefix);
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
180
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
181 #if 1
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
182 list =
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
183 QtConcurrent::blockingMapped(entries,
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
184 boost::bind( &FileDBLink::computedValue,
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
185 _1,
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
186 entries));
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
187 #else
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
188 for (QList<dbinf_ptr_t >::const_iterator it1 = entries.begin();
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
189 it1 != entries.end(); ++it1) {
64
b9515dc35fe4 Make sure no file has greater linewidth than 80.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents: 62
diff changeset
190 dbinf_ptr_t ext = computedValue(*it1, entries);
0
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
191 list.push_back(ext);
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
192 }
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
193 #endif
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
194 return list;
a3834af36579 Working with memory backend.
Tom Fredrik Blenning Klaussen <bfg@blenning.no>
parents:
diff changeset
195 }
85
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
196
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
197
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
198 const QList<FileDBLink::dbinf_ptr_t>
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
199 FileDBLink::filesWithSize(qint64 size, const QString& prefix) const
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
200 {
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
201 QList<dbinf_ptr_t> retVal;
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
202 const QList<dbinf_ptr_t> vals = values(prefix);
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
203 foreach (const dbinf_ptr_t val, vals) {
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
204 QString path = val->path();
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
205 if (val->size() == size) {
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
206 retVal << val;
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
207 }
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
208 }
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
209 return retVal;
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
210 }
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
211
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
212 FileDBLink::dbinf_ptr_t FileDBLink::value(const QString& path) const
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
213 {
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
214 const QList<dbinf_ptr_t> vals = values();
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
215 foreach (const dbinf_ptr_t val, vals) {
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
216 QString vpath = val->path();
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
217 if (vpath == path) {
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
218 return val;
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
219 }
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
220 }
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
221 return dbinf_ptr_t();
1f9e27a0bd7f Allow for lazy calculation of checksums, ignore them, if only one file of given size.
Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no>
parents: 78
diff changeset
222 }