Mercurial > dedupe

comparison DBCache.hpp @ 16:06166d6c083b

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Add configuration processing. Cache DB values Add a custom RBTree to save space. Track multiple DB connections properly. More testing. Add ValueExistsException.
author Tom Fredrik Blenning Klaussen <bfg@blenning.no>
date Tue, 28 Aug 2012 18:58:02 +0200
parents
children 5d14d8c2c299
comparison
equal deleted inserted replaced
15:199fc63c60c1 16:06166d6c083b
1 #ifndef DBCACHE_HPP
2 #define DBCACHE_HPP
3
4 #include <QtSql/QSqlDatabase>
5 #include <QtSql/QSqlQuery>
6 #include <QtSql/QSqlError>
7 #include <QtSql/QSqlRecord>
8
9 #include <QtCore/QDebug>
10 #include <QtCore/QStringList>
11 #include <cassert>
12 #include "OrderedPair.hpp"
13
14 #include <boost/optional.hpp>
15
16 #include "ThreadSafeLookup.hpp"
17 #include "UniqueString.hpp"
18
19
20 template<typename T>
21 struct SQLGenerator
22 {
23 };
24
25 template<>
26 struct SQLGenerator<int>
27 {
28 static QString fieldName(const QString &prefix = QString())
29 {
30 return QString("%1_Int").arg(prefix);
31 }
32
33 static QString createFields(const QString &prefix = QString())
34 {
35 return QString("%1 INTEGER").arg(fieldName(prefix));
36 }
37
38 static QString valueString(const QString &prefix = QString())
39 {
40 return QString(":") + fieldName(prefix);
41 }
42
43 static void bindValue(QSqlQuery& query, int value, const QString& prefix = QString())
44 {
45 query.bindValue(valueString(prefix), value);
46 }
47
48 static void bindValues(QSqlQuery& query, const QVariantList& values, const QString& prefix = QString())
49 {
50 query.bindValue(valueString(prefix), values);
51 }
52
53 static void bindValues(QSqlQuery& query, const QList<int>& values, const QString& prefix = QString())
54 {
55 QVariantList list;
56 foreach(int value, values) {
57 list << value;
58 }
59 bindValues(query, list, prefix);
60 }
61
62 static boost::optional<int> extract(QSqlQuery& query, const QString& prefix = QString())
63 {
64 int fieldNo = query.record().indexOf(fieldName(prefix));
65 if (query.at() >= 0)
66 return query.value(fieldNo).toInt();
67 else
68 return boost::optional<int>();
69 }
70
71 };
72
73 template<>
74 struct SQLGenerator<QString>
75 {
76 static QString fieldName(const QString &prefix = QString())
77 {
78 return QString("%1_QString").arg(prefix);
79 }
80
81 static QString createFields(const QString &prefix = QString())
82 {
83 return QString("%1 TEXT").arg(fieldName(prefix));
84 }
85
86 static QString valueString(const QString &prefix = QString())
87 {
88 return QString(":") + fieldName(prefix);
89 }
90
91 static QString restrict(const QString &prefix = QString())
92 {
93 return QString("%1 = %2").arg(fieldName(prefix)).arg(valueString(prefix));
94 }
95
96 static void bindValue(QSqlQuery& query, const QString& value, const QString& prefix = QString())
97 {
98 query.bindValue(valueString(prefix), value);
99 }
100
101 static void bindValues(QSqlQuery& query, const QVariantList& value, const QString& prefix = QString())
102 {
103 query.bindValue(valueString(prefix), value);
104 }
105
106 static void bindValues(QSqlQuery& query, const QList<QString>& values, const QString& prefix = QString())
107 {
108 QVariantList list;
109 foreach(const QString& value, values) {
110 list << value;
111 }
112 bindValues(query, list, prefix);
113 }
114
115 static boost::optional<QString> extract(QSqlQuery& query, const QString& prefix = QString())
116 {
117 int fieldNo = query.record().indexOf(fieldName(prefix));
118 if (query.at() >= 0)
119 return query.value(fieldNo).toString();
120 else
121 return boost::optional<QString>();
122 }
123
124
125 };
126
127 template<>
128 struct SQLGenerator<UniqueString>
129 {
130 static QString fieldName(const QString &prefix = QString())
131 {
132 return QString("%1_QString").arg(prefix);
133 }
134
135 static QString createFields(const QString &prefix = QString())
136 {
137 return QString("%1 TEXT").arg(fieldName(prefix));
138 }
139
140 static QString valueString(const QString &prefix = QString())
141 {
142 return QString(":") + fieldName(prefix);
143 }
144
145 static QString restrict(const QString &prefix = QString())
146 {
147 return QString("%1 = %2").arg(fieldName(prefix)).arg(valueString(prefix));
148 }
149
150 static void bindValue(QSqlQuery& query, const QString& value, const QString& prefix = QString())
151 {
152 query.bindValue(valueString(prefix), value);
153 }
154
155 static void bindValues(QSqlQuery& query, const QVariantList& value, const QString& prefix = QString())
156 {
157 query.bindValue(valueString(prefix), value);
158 }
159
160 static void bindValues(QSqlQuery& query, const QList<UniqueString>& values, const QString& prefix = QString())
161 {
162 QVariantList list;
163 foreach(const QString& value, values) {
164 list << value;
165 }
166 bindValues(query, list, prefix);
167 }
168
169 static boost::optional<QString> extract(QSqlQuery& query, const QString& prefix = QString())
170 {
171 int fieldNo = query.record().indexOf(fieldName(prefix));
172 if (query.at() >= 0)
173 return query.value(fieldNo).toString();
174 else
175 return boost::optional<QString>();
176 }
177
178
179 };
180
181 template<typename T>
182 struct SQLGenerator<OrderedPair<T> >
183 {
184 static QString fieldName(const QString &prefix = QString())
185 {
186 return SQLGenerator<T>::fieldName(prefix + "_1") + ", " +
187 SQLGenerator<T>::fieldName(prefix + "_2");
188 }
189
190 static QString createFields(const QString &prefix = QString())
191 {
192 return SQLGenerator<T>::createFields(prefix + "_1") + ", " +
193 SQLGenerator<T>::createFields(prefix + "_2");
194 }
195
196 static QString restriction(const QString& prefix = QString())
197 {
198 return SQLGenerator<T>::restrict(prefix + "_1") + " AND " +
199 SQLGenerator<T>::restrict(prefix + "_2");
200 }
201
202 static QString valueString(const QString &prefix = QString())
203 {
204 return SQLGenerator<T>::valueString(prefix + "_1") + ", " +
205 SQLGenerator<T>::valueString(prefix + "_2");
206 }
207
208 static void bindValue(QSqlQuery& query, const OrderedPair<T>& value, const QString& prefix = QString())
209 {
210 SQLGenerator<T>::bindValue(query, value.first, prefix + "_1");
211 SQLGenerator<T>::bindValue(query, value.second, prefix + "_2");
212 }
213
214 static void bindValues(QSqlQuery& query, const QList<OrderedPair<T> >& values, const QString& prefix = QString())
215 {
216 QList<T> first;
217 QList<T> second;
218 foreach(OrderedPair<T> value, values) {
219 first << value.first;
220 second << value.second;
221 }
222 SQLGenerator<T>::bindValues(query, first, prefix + "_1");
223 SQLGenerator<T>::bindValues(query, second, prefix + "_2");
224 }
225
226 static boost::optional<OrderedPair<T> > extract(QSqlQuery& query, const QString& prefix = QString())
227 {
228 if (query.at() >= 0)
229 return OrderedPair<T> (*SQLGenerator<T>::extract(query, prefix + "_1"),
230 *SQLGenerator<T>::extract(query, prefix + "_2"));
231 else
232 return boost::optional<OrderedPair<T> >();
233 }
234
235
236 };
237
238 template<typename Key, typename Value, bool memoryMapped = false>
239 class DBCache
240 {
241 private:
242 ThreadSafeLookup<Key, Value> memoryMap;
243 QList<Key> unsyncedKeys;
244
245 void setup(const QSqlDatabase& db, const QString& dictName)
246 {
247 this->dictName = dictName;
248 if (!db.tables().contains(dictName)) {
249 QString keyFields = SQLGenerator<Key>::createFields("key");
250 QString valueFields = SQLGenerator<Value>::createFields("value");
251 QString createQuery = QString("CREATE TABLE %1(%2, %3);").arg(dictName).arg(keyFields).arg(valueFields);
252 QSqlQuery query(db);
253 query.exec(createQuery);
254 }
255 if (!db.tables().contains(dictName)) {
256 qDebug()<<"No database";
257 exit(1);
258 }
259 if (memoryMapped) {
260 QString keyFields = SQLGenerator<Key>::fieldName("key");
261 QString valueFields = SQLGenerator<Value>::fieldName("value");
262 QString repopulateQuery = QString("SELECT %1, %2 FROM %3;").arg(keyFields).arg(valueFields).arg(dictName);
263 QSqlQuery query(db);
264 if (!query.exec(repopulateQuery)) {
265 qDebug() << query.lastError() << repopulateQuery;
266 }
267 while (query.next()) {
268 Key key = *SQLGenerator<Key>::extract(query, "key");
269 Value value = *SQLGenerator<Value>::extract(query, "value");
270 memoryMap.insert(key, value);
271 }
272 }
273 QString queryString = QString("INSERT into %1 (%2, %3) VALUES(%4, %5);")
274 .arg(dictName)
275 .arg(SQLGenerator<Key>::fieldName("key"))
276 .arg(SQLGenerator<Value>::fieldName("value"))
277 .arg(SQLGenerator<Key>::valueString("key"))
278 .arg(SQLGenerator<Value>::valueString("value"));
279 insertQuery = QSqlQuery(db);
280 insertQuery.prepare(queryString);
281 }
282
283 void syncInsert()
284 {
285 SQLGenerator<Key>::bindValues(insertQuery, unsyncedKeys, "key");
286 QList<Value> values;
287 foreach(Key key, unsyncedKeys) {
288 values << *memoryMap.value(key);
289 }
290 SQLGenerator<Value>::bindValues(insertQuery, values, "value");
291 if (!insertQuery.exec()) {
292 qDebug() << insertQuery.lastError() << insertQuery.lastQuery();
293 }
294 insertQuery.finish();
295 unsyncedKeys.clear();
296 }
297
298 public:
299 DBCache(const QString& dbName, const QString& dictName)
300 {
301 db = QSqlDatabase::addDatabase("QSQLITE", "dictName");
302 db.setDatabaseName(dbName);
303 bool ok = db.open();
304 assert(ok);
305 setup(db, dictName);
306 }
307
308 DBCache(const QSqlDatabase& db, const QString& dictName)
309 {
310 setup(db, dictName);
311 }
312
313 boost::optional<Value> value(const Key& key) const
314 {
315 if (memoryMapped) {
316 return memoryMap.value(key);
317 }
318 else {
319 QString queryString = QString("SELECT %1 FROM %2 WHERE %3")
320 .arg(SQLGenerator<Value>::fieldName("value"))
321 .arg(dictName)
322 .arg(SQLGenerator<Key>::restriction("key"));
323 QSqlQuery query(db);
324 query.prepare(queryString);
325 SQLGenerator<Key>::bindValue(query, key, "key");
326 if (!query.exec()) {
327 qDebug() << query.lastError() << queryString;
328 }
329 query.next();
330 return SQLGenerator<Value>::extract(query, "value");
331 }
332 }
333
334 void insert(const Key& key, const Value& value)
335 {
336 if (memoryMapped) {
337 memoryMap.insert(key, value);
338 unsyncedKeys.append(key);
339 if (unsyncedKeys.size() > 1024) {
340 syncInsert();
341 }
342 }
343 else {
344 SQLGenerator<Key>::bindValue(insertQuery, key, "key");
345 SQLGenerator<Value>::bindValue(insertQuery, value, "value");
346 if (!insertQuery.exec()) {
347 qDebug() << insertQuery.lastError() << insertQuery.lastQuery();
348 }
349 insertQuery.finish();
350 }
351
352 }
353
354 private:
355 QSqlDatabase db;
356 QString dictName;
357 QSqlQuery insertQuery;
358 };
359
360 #endif //DBCACHE_HPP