Mercurial > dedupe

changeset 55:19b2a2d98788

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Use define to multiplex between different types of decoders. Expand the API.
author Tom Fredrik Blenning Klaussen <bfg@blenning.no>
date Thu, 13 Sep 2012 23:47:29 +0200
parents f339499ecd79
children 76846cb92b5c
files HuffmanSet.cpp HuffmanSet.hpp
diffstat 2 files changed, 47 insertions(+), 11 deletions(-) [+]
line wrap: on
line diff
--- a/HuffmanSet.cpp	Thu Sep 13 23:40:46 2012 +0200
+++ b/HuffmanSet.cpp	Thu Sep 13 23:47:29 2012 +0200
@@ -8,15 +8,25 @@
 
 #include <QtCore/QHash>
 
-HuffmanSet::HuffmanSet() : cutoff(256), numInserts(0), lut(0)
+HuffmanSet::HuffmanSet() : cutoff(1024), numInserts(0), lut(0)
 {
 }
 
+HuffmanSet::~HuffmanSet()
+{
+  delete lut;
+}
+
 void HuffmanSet::setCutoff(uint cutoff)
 {
   this->cutoff = cutoff;
 }
 
+void HuffmanSet::setAutoRebuild(bool autoRebuild)
+{
+  this->autoRebuild = autoRebuild;
+}
+
 QStringList HuffmanSet::chunks(const QString& str)
 {
   return str.split("", QString::SkipEmptyParts);
@@ -74,10 +84,8 @@
   return newStrings.size() + map.size();
 }
 
-void HuffmanSet::rebuild()
+void HuffmanSet::generateFreqTable(QMap<QString, uint>& freqTable) const
 {
-  QMap<QString, uint> freqTable;
-
   foreach(key_t key, map.keys()) {
     foreach(const QString& chunk, chunks(decode(map.value(key)))) {
       ++freqTable[chunk];
@@ -88,6 +96,21 @@
       ++freqTable[chunk];
     }
   }
+}
+
+QMap<QString, uint> HuffmanSet::generateFreqTable() const
+{
+  QMap<QString, uint> freqTable;
+  generateFreqTable(freqTable);
+  return freqTable;
+}
+
+
+void HuffmanSet::rebuild()
+{
+  QMap<QString, uint> freqTable;
+
+  generateFreqTable(freqTable);
 
   BitDecoder* newLut = createLut(freqTable);
 
@@ -101,11 +124,12 @@
   }
   numInserts = 0;
   delete lut;
-  /*
+#if FASTENCODE
   lut = new FastBitDecoder(encoder);
   delete newLut;
-  */
+#else
   lut = newLut;
+#endif //FASTENCODE
   newStrings.clear();
 }
 
@@ -133,7 +157,7 @@
     catch (InvalidDataException& e) {
       newStrings.insert(key, str);
     }
-    if (++numInserts >= cutoff) {
+    if ((++numInserts >= cutoff) && autoRebuild) {
       rebuild();
     }
   }
--- a/HuffmanSet.hpp	Thu Sep 13 23:40:46 2012 +0200
+++ b/HuffmanSet.hpp	Thu Sep 13 23:47:29 2012 +0200
@@ -10,27 +10,39 @@
 class BitDecoder;
 class FastBitDecoder;
 
+#define FASTENCODE 1
+
 class HuffmanSet {
 public:
   typedef uint key_t;
 
 private:
+  uint cutoff;
+  uint numInserts;
+  bool autoRebuild;
+
   QMap<key_t, QString> newStrings;
   QMap<key_t, BitArray> map;
   QMap<QString, BitArray> encoder;
-  uint cutoff;
-  uint numInserts;
-  //FastBitDecoder* lut;
+#if FASTENCODE
+  FastBitDecoder* lut;
+#else
   BitDecoder* lut;
+#endif //FASTENCODE
 
 
 public:
   HuffmanSet();
+  ~HuffmanSet();
   void setCutoff(uint cutoff);
+  void setAutoRebuild(bool);
   static QStringList chunks(const QString& str);
   BitDecoder* createLut(const QMap<QString, uint>& freqTable);
   QString decode(const BitArray& bits) const;
-  static BitArray encode(const QString& string, const QMap<QString, BitArray>& encoder);
+  static BitArray encode(const QString& string,
+			 const QMap<QString, BitArray>& encoder);
+  void generateFreqTable(QMap<QString, uint>& freqTable) const;
+  QMap<QString, uint> generateFreqTable() const;
   void rebuild();
   bool contains(key_t key) const;
   uint totalElements() const;