Mercurial > dedupe
view DataController.cpp @ 115:404795616b1e default tip
Added a lot of common files to ignore
| author | Tom Fredrik Blenning Klaussen <bfg@bfgconsult.no> |
|---|---|
| date | Sat, 25 Mar 2017 17:43:57 +0100 |
| parents | 95fb1bcf7e24 |
| children |
line wrap: on
line source
#include "DataController.hpp" #include "CompileTimeConstants.h" #include "ConfigurationProcessing.hpp" #include "CachedEditDistance.hpp" #include "SqliteDBLink.hpp" #include "Exception/PermissionException.hpp" #include <QtCore/QDebug> #include <QtCore/QTimer> #include <QtCore/QUrl> #include <QtGui/QApplication> #include <QtGui/QDesktopServices> #include <QtGui/QDoubleSpinBox> #include <QtGui/QHBoxLayout> #include <QtGui/QLabel> #include <QtGui/QMainWindow> #include <QtGui/QMenuBar> #include <QtGui/QMessageBox> #include <QtGui/QProgressBar> #include <QtGui/QToolBar> #include <QtGui/QTreeWidget> #include <boost/filesystem.hpp> void DataController::findFiles(const QDir& dir, QStringList& list) { const bool FOLLOW_SYMLINKS = false; #if USE_BOOST_FIND namespace fs = boost::filesystem; fs::path someDir(dir.path().toStdString()); fs::directory_iterator end_iter; boost::system::error_code ec; if ( fs::exists(someDir) && fs::is_directory(someDir)) { for( fs::directory_iterator dir_iter(someDir, ec) ; dir_iter != end_iter ; ++dir_iter) { std::wstring wpath; try { wpath = dir_iter->path().wstring(); } catch (boost::system::system_error &e) { printf("%s\n", dir_iter->path().c_str()); std::cout << e.what() << std::endl; } if (!wpath.empty()) { if (fs::is_directory(FOLLOW_SYMLINKS? dir_iter->status(): dir_iter->symlink_status()) ) { findFiles(QString::fromStdWString(wpath), list); } else if (fs::is_regular_file(dir_iter->symlink_status()) ) { list << QString::fromStdWString(wpath); } } } } #else foreach(QString filename, dir.entryList(QDir::NoDotAndDotDot | QDir::Dirs)) { filename = dir.absoluteFilePath(filename); findFiles(QDir(filename), list); } foreach(QString filename, dir.entryList(QDir::Files | QDir::NoSymLinks)) { list << dir.absoluteFilePath(filename); } #endif } QStringList DataController::findFiles(const QDir& dir) { QStringList list; findFiles(dir, list); return list; } void DataController::findFiles(const QDir& dir, FileDBLink& dblink) { QStringList paths = findFiles(dir); QDateTime last = QDateTime::currentDateTime(); qDebug() << "Start Delete"; dblink.keepOnlyFromPrefix(dir.path(), paths, true); dblink.commit(dir.path(), false); qDebug() << "End Delete"; std::auto_ptr<QProgressBar> bar; progressMax = paths.size(); if (showGUI) { bar = std::auto_ptr<QProgressBar>(new QProgressBar()); bar->resize(200,25); bar->setValue(0); bar->setMinimum(0); bar->setMaximum(progressMax); bar->show(); connect(this, SIGNAL(populateProgress(int)), bar.get(), SLOT(setValue(int))); } int n = 0; int max = paths.size(); emit populateProgress(n, max); foreach(QString filename, paths) { try { dblink.updateIfModified(filename, true); } catch (const PermissionException& e) { dblink.deleteFileFromDB(filename); } catch (const IOException& e) { //FIXME: We have some strange errors, avoid them by ignoring //them for now dblink.deleteFileFromDB(filename); } catch (Exception& e) { e.raise(); } emit populateProgress(++n, max); QDateTime now = QDateTime::currentDateTime(); if (last.msecsTo(now) > 1000) { QCoreApplication::processEvents(); last = now; } } dblink.commit(dir.path()); } QTreeWidgetItem* DataController::createItem(const FileDBLink::DBInfo& info) { QTreeWidgetItem* item = new QTreeWidgetItem(); item->setData(0, Qt::DisplayRole, info.name()); item->setData(0, 32, info.path()); item->setData(0, 33, info.name()); item->setData(1, Qt::DisplayRole, info.size()); item->setData(2, Qt::DisplayRole, info.mtime()); item->setData(3, Qt::DisplayRole, info.checksum().toHex()); return item; } void DataController::delayPopulate() { populateDelay->start(); } void DataController::populate() { populate(nameFilter->isChecked(), sizeFilter->isChecked(), mtimeFilter->isChecked(), checksumFilter->isChecked(), (100 - editCutoffSpin->value()) * .01); } void DataController::populate(bool showNameDups, bool showSizeDups, bool showMTimeDups, bool showCheckSumDups, float editDistanceCutoff) { tw->clear(); const QList<QSharedPointer<FileDBLink::DBInfo> > elems = dblink->values(dir.path()); QProgressBar bar; bar.resize(200,25); bar.setValue(0); bar.setMinimum(0); bar.setMaximum(elems.size()); bar.show(); connect(this, SIGNAL(populateProgress(int)), &bar, SLOT(setValue(int))); int n = 0; tw->setUpdatesEnabled(false); QMultiMap<QString, QSharedPointer<FileDBLink::DBInfo> > nameLUP; QMultiMap<quint64, QSharedPointer<FileDBLink::DBInfo> > sizeLUP; QMultiMap<QDateTime, QSharedPointer<FileDBLink::DBInfo> > mtimeLUP; QMultiMap<QByteArray, QSharedPointer<FileDBLink::DBInfo> > checksumLUP; int max = elems.size() - 1; foreach(QSharedPointer<FileDBLink::DBInfo> line, elems) { if (showNameDups) { nameLUP.insertMulti(line->name(), line); } if (showSizeDups) { sizeLUP.insertMulti(line->size(), line); } if (showMTimeDups) { mtimeLUP.insertMulti(line->mtime(), line); } if (showCheckSumDups) { checksumLUP.insertMulti(line->checksum(), line); } } foreach(QSharedPointer<FileDBLink::DBInfo> line, elems) { QTreeWidgetItem* item = 0; if (showNameDups) { QTreeWidgetItem* topLevelItem = 0; foreach(QSharedPointer<FileDBLink::DBInfo> dup, nameLUP.values(line->name())) { if(dup != line) { if (!topLevelItem) { topLevelItem = new QTreeWidgetItem(); topLevelItem->setData(0, Qt::DisplayRole, "Name"); if(!item) item = createItem(*line); item->addChild(topLevelItem); } topLevelItem->addChild(createItem(*dup)); } } } if (showSizeDups) { QTreeWidgetItem* topLevelItem = 0; foreach(QSharedPointer<FileDBLink::DBInfo> dup, sizeLUP.values(line->size())) { if(dup != line ) { if (!topLevelItem) { topLevelItem = new QTreeWidgetItem(); topLevelItem->setData(0, Qt::DisplayRole, "Size"); if(!item) item = createItem(*line); item->addChild(topLevelItem); } topLevelItem->addChild(createItem(*dup)); } } } if (showMTimeDups) { QTreeWidgetItem* topLevelItem = 0; foreach(QSharedPointer<FileDBLink::DBInfo> dup, mtimeLUP.values(line->mtime())) { if(dup != line ) { if (!topLevelItem) { topLevelItem = new QTreeWidgetItem(); topLevelItem->setData(0, Qt::DisplayRole, "MTime"); if(!item) item = createItem(*line); item->addChild(topLevelItem); } topLevelItem->addChild(createItem(*dup)); } } } if (showCheckSumDups) { QTreeWidgetItem* topLevelItem = 0; foreach(QSharedPointer<FileDBLink::DBInfo> dup, checksumLUP.values(line->checksum())) { if(dup != line) { if (!topLevelItem) { topLevelItem = new QTreeWidgetItem(); topLevelItem->setData(0, Qt::DisplayRole, "Checksum"); if(!item) item = createItem(*line); item->addChild(topLevelItem); } topLevelItem->addChild(createItem(*dup)); } } } if (editDistanceCutoff < 1.0) { QTreeWidgetItem* topLevelItem = 0; QMultiMap<int, QSharedPointer<FileDBLink::DBInfo> > oList; int absoluteCutoff = line->name().length() * editDistanceCutoff; foreach(QSharedPointer<FileDBLink::DBInfo> dup, elems) { if(dup != line) { int distance = CachedEditDistance::Compute(line->name(), dup->name()); if (distance <= absoluteCutoff) { oList.insert(distance, dup); } } } if (oList.size() > 0 ) { topLevelItem = new QTreeWidgetItem(); topLevelItem->setData(0, Qt::DisplayRole, "Editdistance"); if(!item) item = createItem(*line); item->addChild(topLevelItem); } foreach(QSharedPointer<FileDBLink::DBInfo> dup, oList.values()) { topLevelItem->addChild(createItem(*dup)); } } if (item) tw->addTopLevelItem(item); emit populateProgress(++n, max); if (n % 64 == 0) { QCoreApplication::processEvents(); } } tw->setUpdatesEnabled(true); } void DataController::contextMenuRequested(const QPoint& point) { contextMenuItem = tw->itemAt(point); if (!contextMenu) { contextMenu = new QMenu(tw); QAction* deleteAction = contextMenu->addAction("Delete"); connect(deleteAction, SIGNAL(triggered()), this, SLOT(deleteFile())); } contextMenu->popup(tw->mapToGlobal(point)); } void DataController::setDir(const QDir& dir) { this->dir = dir.absolutePath(); } void DataController::buildDB(const QDir& dir) { findFiles(dir, *dblink); } void DataController::setupGUI() { populateDelay = new QTimer(this); populateDelay->setSingleShot(true); populateDelay->setInterval(500); connect(populateDelay, SIGNAL(timeout()), this, SLOT(populate())); mw = new QMainWindow(); QMenuBar* mb = new QMenuBar(); QMenu* menu = mb->addMenu("&View"); QAction* action = menu->addAction("Show full path"); action->setCheckable(true); connect(action, SIGNAL(toggled(bool)), this, SLOT(setShowFullPath(bool))); mw->setMenuBar(mb); QToolBar* filterBar = new QToolBar("Filters"); nameFilter = filterBar->addAction("Name"); nameFilter->setCheckable(true); connect(nameFilter, SIGNAL(toggled(bool)), this, SLOT(delayPopulate())); sizeFilter = filterBar->addAction("Size"); sizeFilter->setCheckable(true); connect(sizeFilter, SIGNAL(toggled(bool)), this, SLOT(delayPopulate())); mtimeFilter = filterBar->addAction("MTime"); mtimeFilter->setCheckable(true); connect(mtimeFilter, SIGNAL(toggled(bool)), this, SLOT(delayPopulate())); checksumFilter = filterBar->addAction("Checksum"); checksumFilter->setCheckable(true); checksumFilter->setChecked(true); connect(checksumFilter, SIGNAL(toggled(bool)), this, SLOT(delayPopulate())); QWidget* widget = new QWidget(); QLayout* layout = new QHBoxLayout(); layout->setContentsMargins(0,0,0,0); widget->setLayout(layout); layout->addWidget(new QLabel("Edit distance", widget)); editCutoffSpin = new QSpinBox(widget); layout->addWidget(editCutoffSpin); editCutoffSpin->setMinimum(0); editCutoffSpin->setMaximum(100); editCutoffSpin->setValue(0); editCutoffSpin->setSingleStep(10); editCutoffSpin->setSuffix("%"); connect(editCutoffSpin, SIGNAL(valueChanged(int)), this, SLOT(delayPopulate())); filterBar->addWidget(widget); mw->addToolBar(filterBar); tw = new QTreeWidget(mw); tw->setContextMenuPolicy( Qt::CustomContextMenu); connect(tw, SIGNAL(customContextMenuRequested (const QPoint&)), this, SLOT(contextMenuRequested(const QPoint&))); connect(tw, SIGNAL(itemDoubleClicked(QTreeWidgetItem*, int)), this, SLOT(itemDoubleClicked(QTreeWidgetItem*, int))); mw->setCentralWidget(tw); tw->setEditTriggers(QAbstractItemView::NoEditTriggers); tw->setHeaderLabels(QString("Path;Size;Date;Checksum").split(";")); tw->setSortingEnabled(true); tw->setSizePolicy(QSizePolicy::Expanding, QSizePolicy::Expanding); tw->setSelectionBehavior(QAbstractItemView::SelectRows); //tw->resizeColumnsToContents(); mw->resize(800,800); mw->setEnabled(false); mw->show(); } DataController::DataController(const QString& path, bool showGUI) { setup(QString(), path, showGUI); } DataController::DataController(bool showGUI) { setup(QString(), QString(), showGUI); } void DataController::progressUpdate(int p, int max) { QString str; if (max == 0) { str.sprintf("Progress %6.2f%%", 100.0); } else { if (p == 0) str.sprintf("Progress %6.2f%%", p * 100.0 / max); else if (p == max) { str.sprintf("\b\b\b\b\b\b\b%6.2f%%\n", 100.0); } else { str.sprintf("\b\b\b\b\b\b\b%6.2f%%", p * 100.0 / max); } } if (p == max) { str+="\n"; } std::cout<<str.toStdString(); std::cout.flush(); } void DataController::progressUpdate(int p) { progressUpdate(p, progressMax); } void DataController::deleteFile() { QString path = contextMenuItem->data(0, 32).toString(); QMessageBox::StandardButton button = QMessageBox::question(tw, "Confirm delete", QString("Do you really want to delete \"%1\"?") .arg(path), QMessageBox::Cancel | QMessageBox::Ok, QMessageBox::Cancel); if (button == QMessageBox::Ok) { QFile file(path); if (file.remove()) { dblink->deleteFileFromDB(path); populate(); } else { QMessageBox::warning(tw, "Delete failed", QString("Could not delete \"%1\"?") .arg(file.fileName())); } } } void DataController::itemDoubleClicked (QTreeWidgetItem * item, int column) { QUrl url = QUrl::fromLocalFile(item->data(0, 32).toString()); QDesktopServices::openUrl(url); } void DataController::setup(const QString& dbpath_in, const QString& searchPath_in, bool showGUI) { this->showGUI = showGUI; contextMenu = 0; connect(this, SIGNAL(populateProgress(int, int)), this, SLOT(progressUpdate(int))); QString dbpath; if (dbpath_in.size() > 0) { dbpath = dbpath_in; } else { dbpath = processSetupVariables(DB_DEFAULT_LOCATION); } #if 1 dblink = new SqliteDBLink(dbpath); #else dblink = new MemoryDBLink(); #endif connect(dblink, SIGNAL(progressUpdate(int, int)), this, SLOT(progressUpdate(int, int))); setDir((searchPath_in.size() > 0) ? searchPath_in : QDir(".")); showFullPath = false; if (showGUI) { setupGUI(); QTimer* populator = new QTimer(this); populator->setSingleShot(true); populator->setInterval(50); connect(populator, SIGNAL(timeout()), this, SLOT(initialPopulate())); populator->start(); } else { buildDB(dir); } } void DataController::initialPopulate() { buildDB(dir); populate(); mw->setEnabled(true); } DataController::~DataController() { delete dblink; } void DataController::cellDoubleClicked(int row, int column) { if(column == 0) { toggleShowFullPath(); } } void DataController::setShowFullPath(bool showFullPath) { this->showFullPath = showFullPath; tw->setSortingEnabled(false); int role = (showFullPath) ? 32 : 33; for (int row = 0; row < tw->topLevelItemCount(); ++row) { QTreeWidgetItem* pathItem = tw->topLevelItem(row); pathItem->setData(0, Qt::DisplayRole, pathItem->data(0, role)); setShowFullPath(pathItem, showFullPath); } tw->setSortingEnabled(true); } void DataController::setShowFullPath(QTreeWidgetItem* item, bool showFullPath) { int role = (showFullPath) ? 32 : 33; for (int row = 0; row < item->childCount(); ++row) { QTreeWidgetItem* child = item->child(row); QVariant data = child->data(0,role); if (data.isValid()) child->setData(0, Qt::DisplayRole, data); setShowFullPath(child, showFullPath); } } bool DataController::toggleShowFullPath() { bool showFullPath = ! this->showFullPath; setShowFullPath(showFullPath); return showFullPath; }
