Mercurial > dedupe

changeset 58:7b7e84356b39

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Introduce API for fine tuning inserts. Reduce the amount of rebuilds.
author Tom Fredrik Blenning Klaussen <bfg@blenning.no>
date Fri, 14 Sep 2012 00:04:24 +0200
parents c8111de2e0bb
children 74be5a2f49db
files DBCache.hpp EditDistance.hpp
diffstat 2 files changed, 36 insertions(+), 4 deletions(-) [+]
line wrap: on
line diff
--- a/DBCache.hpp	Thu Sep 13 23:53:35 2012 +0200
+++ b/DBCache.hpp	Fri Sep 14 00:04:24 2012 +0200
@@ -18,6 +18,15 @@
 
 #include <boost/optional.hpp>
 
+#include <QtCore/QString>
+
+template<typename Key, typename Value>
+struct InsertRegulator {
+  static void start() {}
+  static void finish() {}
+  static void next() {}
+};
+
 template<typename T>
 struct SQLGenerator
 {
@@ -264,11 +273,15 @@
       if (!query.exec(repopulateQuery)) {
 	throw SQLException(query);
       }
+      InsertRegulator<Key, Value> regulator;
+      regulator.start();
       while (query.next()) {
 	Key key = *SQLGenerator<Key>::extract(query, "key");
 	Value value = *SQLGenerator<Value>::extract(query, "value");
 	memoryMap.insert(key, value);
+	regulator.next();
       }
+      regulator.finish();
     }
     QString queryString = QString("INSERT into %1 (%2, %3) VALUES(%4, %5);")
       .arg(dictName)
--- a/EditDistance.hpp	Thu Sep 13 23:53:35 2012 +0200
+++ b/EditDistance.hpp	Fri Sep 14 00:04:24 2012 +0200
@@ -10,18 +10,37 @@
 #include <QtCore/QMap>
 #include <QtCore/QString>
 
+template<typename Value>
+struct InsertRegulator<OrderedPair<UniqueString>, Value >
+{
+  uint n;
+  void start()
+  {
+    n = 0;
+    HuffmanString::getSet().setAutoRebuild(false);
+  }
+
+  static void finish()
+  {
+    HuffmanString::getSet().rebuild();
+    HuffmanString::getSet().setAutoRebuild(true);
+  }
+
+  void next()
+  {
+    if (++n == 2048)
+      HuffmanString::getSet().rebuild();
+  }
+};
+
 class EditDistance {
 protected:
-  //typedef ThreadSafeLookup<OrderedPair<QString>, int> cacheType;
   typedef DBCache<OrderedPair<UniqueString>, int, true> cacheType;
-  //typedef QMap<OrderedPair<QString>, int> cacheType;
-  //typedef QHash<OrderedPair<QString>, int> cacheType;
 public:
   static int Compute(QString a, QString b, bool removeDiacritics = false);
   static void removeDiacriticsNoCopy(QString& in);
   static QString removeDiacritics(const QString& in);
 
-  //static cacheType cache;
   static cacheType* cache;
 };