Mercurial > dedupe

view DataController.cpp @ 4:f489b0c9bf99

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Refactored to allow commandline tools to use DataController.
author Tom Fredrik Blenning Klaussen <bfg@blenning.no>
date Tue, 21 Aug 2012 15:27:29 +0200
parents 7a44ba08673d
children 5e4985407feb
line wrap: on
line source

#include "CompileTimeConstants.h"

#include "DataController.hpp"

#include "MemoryDBLink.hpp"
#include "SqliteDBLink.hpp"

#include "PermissionException.hpp"
#include "DataController.hpp"

#include <QtGui/QApplication>
#include <QtCore/QDir>

#include <QtCore/QDebug>
#include <QtCore/QTimer>
#include <QtCore/QCryptographicHash>
#include <QtCore/QDateTime>

#include <QtGui/QMainWindow>
#include <QtGui/QDesktopServices>
#include <QtGui/QTreeWidget>
#include <QtGui/QHeaderView>
#include <QtGui/QMenuBar>
#include <QtGui/QToolBar>
#include <QtGui/QDoubleSpinBox>
#include <QtGui/QLabel>
#include <QtGui/QHBoxLayout>
#include <QtGui/QProgressBar>

#include "EditDistance.hpp"

void DataController::findFiles(const QDir& dir, QStringList& list)
{
  foreach(QString filename, dir.entryList(QDir::NoDotAndDotDot | QDir::Dirs)) {
    filename = dir.absoluteFilePath(filename);
    findFiles(QDir(filename), list);
  }

  foreach(QString filename, dir.entryList(QDir::Files)) {
    list << dir.absoluteFilePath(filename);
  }
}

QStringList DataController::findFiles(const QDir& dir)
{
  QStringList list;
  findFiles(dir, list);
  return list;
}

void DataController::findFiles(const QDir& dir, FileDBLink& dblink)
{
  QStringList list = findFiles(dir);

  QProgressBar bar;

  QDateTime last = QDateTime::currentDateTime();

  bar.resize(200,25);
  bar.setValue(0);
  bar.setMinimum(0);
  bar.setMaximum(list.size());
  bar.show();

  connect(this, SIGNAL(populateProgress(int)), &bar, SLOT(setValue(int)));

  int n = 0;
  foreach(QString filename, findFiles(dir)) {
    try {
      dblink.updateIfModified(filename);
    }
    catch (const PermissionException& e) {
      qDebug() << e.toString();
    }
    catch (Exception& e) {
      qDebug() << e.toString();
      exit(1);
    }

    emit populateProgress(++n);
    
    QDateTime now = QDateTime::currentDateTime();
    if (last.msecsTo(now) > 500) {
      QCoreApplication::processEvents();
      last = now;
    }
  }

}


QTreeWidgetItem* DataController::createItem(const FileDBLink::DBInfo& info)
{
  QTreeWidgetItem* item = new QTreeWidgetItem();
  item->setData(0, Qt::DisplayRole, info.name());
  item->setData(0, 32, info.path());
  item->setData(0, 33, info.name());
  
  item->setData(1, Qt::DisplayRole, info.size());
  item->setData(2, Qt::DisplayRole, info.mtime());
  item->setData(3, Qt::DisplayRole, info.checksum().toHex());
  return item;
}

void DataController::delayPopulate()
{
  populateDelay->start();
}


void DataController::populate()
{
  populate(nameFilter->isChecked(), sizeFilter->isChecked(), mtimeFilter->isChecked(),
	   checksumFilter->isChecked(), (100 - editCutoffSpin->value()) * .01);
}

void DataController::populate(bool showNameDups, bool showSizeDups,
			      bool showMTimeDups, bool showCheckSumDups,
			      float editDistanceCutoff)
{
  tw->clear();

  const QList<QSharedPointer<FileDBLink::DBInfo> > elems = dblink->values(dir.path());

  QProgressBar bar;

  bar.resize(200,25);
  bar.setValue(0);
  bar.setMinimum(0);
  bar.setMaximum(elems.size());
  bar.show();

  connect(this, SIGNAL(populateProgress(int)), &bar, SLOT(setValue(int)));

  int n = 0;

  tw->setUpdatesEnabled(false);

  
  QMultiMap<QString, QSharedPointer<FileDBLink::DBInfo> > nameLUP;
  QMultiMap<quint64, QSharedPointer<FileDBLink::DBInfo> > sizeLUP;
  QMultiMap<QDateTime, QSharedPointer<FileDBLink::DBInfo> > mtimeLUP;
  QMultiMap<QByteArray, QSharedPointer<FileDBLink::DBInfo> > checksumLUP;

  foreach(QSharedPointer<FileDBLink::DBInfo> line, elems) {
    if (showNameDups) {
      nameLUP.insertMulti(line->name(), line);
    }
    if (showSizeDups) {
      sizeLUP.insertMulti(line->size(), line);
    }
    if (showMTimeDups) {
      mtimeLUP.insertMulti(line->mtime(), line);
    }
    if (showCheckSumDups) {
      checksumLUP.insertMulti(line->checksum(), line);
    }
  }

  foreach(QSharedPointer<FileDBLink::DBInfo> line, elems) {
    QTreeWidgetItem* item = 0;
    bool anyAdded = false;

    if (showNameDups) {
      QTreeWidgetItem* topLevelItem = 0;
      foreach(QSharedPointer<FileDBLink::DBInfo> dup, nameLUP.values(line->name())) {
	if(dup != line) {
	  if (!topLevelItem) {
	    topLevelItem =  new QTreeWidgetItem();
	    topLevelItem->setData(0, Qt::DisplayRole, "Name");
	    if(!item)
	      item = createItem(*line);
	    item->addChild(topLevelItem);
	    anyAdded = true;
	  }
	  topLevelItem->addChild(createItem(*dup));
	}
      }
    }

    if (showSizeDups) {
      QTreeWidgetItem* topLevelItem = 0;
      foreach(QSharedPointer<FileDBLink::DBInfo> dup, sizeLUP.values(line->size())) {
	if(dup != line ) {
	  if (!topLevelItem) {
	    topLevelItem =  new QTreeWidgetItem();
	    topLevelItem->setData(0, Qt::DisplayRole, "Size");
	    if(!item)
	      item = createItem(*line);
	    item->addChild(topLevelItem);
	    anyAdded = true;
	  }
	  topLevelItem->addChild(createItem(*dup));
	}
      }
    }

    if (showMTimeDups) {
      QTreeWidgetItem* topLevelItem = 0;
      foreach(QSharedPointer<FileDBLink::DBInfo> dup, mtimeLUP.values(line->mtime())) {
	if(dup != line ) {
	  if (!topLevelItem) {
	    topLevelItem =  new QTreeWidgetItem();
	    topLevelItem->setData(0, Qt::DisplayRole, "MTime");
	    if(!item)
	      item = createItem(*line);
	    item->addChild(topLevelItem);
	    anyAdded = true;
	  }
	  topLevelItem->addChild(createItem(*dup));
	}
      }
    }

    if (showCheckSumDups) {
      QTreeWidgetItem* topLevelItem = 0;
      foreach(QSharedPointer<FileDBLink::DBInfo> dup, checksumLUP.values(line->checksum())) {
	if(dup != line) {
	  if (!topLevelItem) {
	    topLevelItem =  new QTreeWidgetItem();
	    topLevelItem->setData(0, Qt::DisplayRole, "Checksum");
	    if(!item)
	      item = createItem(*line);
	    item->addChild(topLevelItem);
	    anyAdded = true;
	  }
	  topLevelItem->addChild(createItem(*dup));
	}
      }
    }

    if (editDistanceCutoff < 1.0) {
      QTreeWidgetItem* topLevelItem = 0;
      QMultiMap<int, QSharedPointer<FileDBLink::DBInfo> > oList;

      int absoluteCutoff = line->name().length() * editDistanceCutoff;
      foreach(QSharedPointer<FileDBLink::DBInfo> dup, elems) {
	if(dup != line) {
	  int distance = EditDistance::Compute(line->name(), dup->name());

	  if (distance <= absoluteCutoff) {
	    oList.insert(distance, dup);
	  }
	}
      }

      if (oList.size() > 0 ) {
	topLevelItem =  new QTreeWidgetItem();
	topLevelItem->setData(0, Qt::DisplayRole, "Editdistance");
	if(!item)
	  item = createItem(*line);
	item->addChild(topLevelItem);
	anyAdded = true;
      }
      
      foreach(QSharedPointer<FileDBLink::DBInfo> dup, oList.values()) {
	topLevelItem->addChild(createItem(*dup));
      }
    }



    if (item)
      tw->addTopLevelItem(item);
    emit populateProgress(++n);
    if (n % 64 == 0) {
      QCoreApplication::processEvents();
    }
  }
  tw->setUpdatesEnabled(true);

}

void DataController::setDir(const QDir& dir)
{
  this->dir = dir.absolutePath();
}

void DataController::buildDB(const QDir& dir)
{
  findFiles(dir, *dblink);
}


void DataController::setupGUI()
{
  populateDelay = new QTimer(this);
  populateDelay->setSingleShot(true);
  populateDelay->setInterval(500);
  connect(populateDelay, SIGNAL(timeout()), this, SLOT(populate()));

  mw = new QMainWindow();
  QMenuBar* mb = new QMenuBar();

  QMenu* menu = mb->addMenu("&View");
  QAction* action = menu->addAction("Show full path");
  action->setCheckable(true);
  connect(action, SIGNAL(toggled(bool)), this, SLOT(setShowFullPath(bool)));

  mw->setMenuBar(mb);

  QToolBar* filterBar = new QToolBar("Filters");

  nameFilter = filterBar->addAction("Name");
  nameFilter->setCheckable(true);
  connect(nameFilter, SIGNAL(toggled(bool)), this, SLOT(delayPopulate()));

  sizeFilter = filterBar->addAction("Size");
  sizeFilter->setCheckable(true);
  connect(sizeFilter, SIGNAL(toggled(bool)), this, SLOT(delayPopulate()));

  mtimeFilter = filterBar->addAction("MTime");
  mtimeFilter->setCheckable(true);
  connect(mtimeFilter, SIGNAL(toggled(bool)), this, SLOT(delayPopulate()));

  checksumFilter = filterBar->addAction("Checksum");
  checksumFilter->setCheckable(true);
  checksumFilter->setChecked(true);
  connect(checksumFilter, SIGNAL(toggled(bool)), this, SLOT(delayPopulate()));

  QWidget* widget = new QWidget();
  QLayout* layout = new QHBoxLayout();
  layout->setContentsMargins(0,0,0,0);
  widget->setLayout(layout);
  layout->addWidget(new QLabel("Edit distance", widget));
  editCutoffSpin = new QSpinBox(widget);
  layout->addWidget(editCutoffSpin);
  editCutoffSpin->setMinimum(0);
  editCutoffSpin->setMaximum(100);
  editCutoffSpin->setValue(0);
  editCutoffSpin->setSingleStep(10);
  editCutoffSpin->setSuffix("%");
  connect(editCutoffSpin, SIGNAL(valueChanged(int)), this, SLOT(delayPopulate()));
  filterBar->addWidget(widget);

  mw->addToolBar(filterBar);

  tw = new QTreeWidget(mw);
  mw->setCentralWidget(tw);
  tw->setEditTriggers(QAbstractItemView::NoEditTriggers);

  tw->setHeaderLabels(QString("Path;Size;Date;Checksum").split(";"));	

  tw->setSortingEnabled(true);
  tw->setSizePolicy(QSizePolicy::Expanding, QSizePolicy::Expanding);
  tw->setSelectionBehavior(QAbstractItemView::SelectRows);
  //tw->resizeColumnsToContents();
  mw->resize(800,800);
  mw->show();
}


DataController::DataController(const QString& path, bool showGUI)
{
  setup(QString(), path, showGUI);
}

DataController::DataController(bool showGUI)
{
  setup(QString(), QString(), showGUI);
}

void DataController::setup(const QString& dbpath_in, const QString& searchPath_in, bool showGUI)
{
  QString dbpath;
  if (dbpath_in.size() > 0) {
    dbpath = dbpath_in;
  }
  else {
    dbpath = DB_DEFAULT_LOCATION;
    
    dbpath.replace(QRegExp("^~/"),
		   QString("%1%2").arg(QDesktopServices::storageLocation(QDesktopServices::HomeLocation)).arg(QDir::separator()));
  }
    
  dblink = new SqliteDBLink(dbpath);

  setDir((searchPath_in.size() > 0) ? searchPath_in : QDir("."));

  showFullPath = false;

  if (showGUI)
    setupGUI();
  buildDB(dir);
  if (showGUI)
    populate();

}

DataController::~DataController()
{
}


void DataController::cellDoubleClicked(int row, int column)
{
  if(column == 0) {
    toggleShowFullPath();
  }
}


void DataController::setShowFullPath(bool showFullPath)
{
  this->showFullPath = showFullPath;

  tw->setSortingEnabled(false);

  int role = (showFullPath)?32:33;
  for (int row = 0; row < tw->topLevelItemCount(); ++row) {
    QTreeWidgetItem* pathItem = tw->topLevelItem(row);
    pathItem->setData(0, Qt::DisplayRole, pathItem->data(0,role));
  }
  tw->setSortingEnabled(true);
}

bool DataController::toggleShowFullPath()
{
  bool showFullPath = ! this->showFullPath;
  setShowFullPath(showFullPath);
  return showFullPath;
}