ShapeFinder2/QtShapeFinder2

This example requires Qt5 >= 5.9 setup for building.

You may build it with Ubuntu 18.04's default Qt5 after installing:

• qt5-default

#ifdef _MSC_VER
#pragma warning ( push, 1)
#pragma warning ( disable : 4702)
#endif

#include <QApplication>
#include <QIcon>

#include "main_widget.hpp"



#ifdef _MSC_VER
#pragma warning ( pop)
#endif

int main(int argc, char* argv[])
{

  QApplication application(argc, argv);
  MainWidget mainWidget;
  mainWidget.setWindowIcon(QIcon(":/qttutorial.png"));
  mainWidget.show();
  application.exec();
}

#pragma once

#include <QSplitter>
#include <QCheckBox>
#include <QLabel>

#include <cvb/ui/image_view.hpp>
#include <cvb/async/single_stream_handler.hpp>
#include <cvb/shapefinder2/classifier.hpp>

// The custom image scene to draw overlays
class CustomImageScene : public Cvb::UI::ImageScene
{
Q_OBJECT

public:
  explicit CustomImageScene(QWidget* parent = 0);
  void SetSearchResult(const Cvb::Image & image, std::vector<Cvb::ShapeFinder2::SearchResult> result);
  void ClearScene();
  void mousePressEvent(QGraphicsSceneMouseEvent* event);
  void mouseReleaseEvent(QGraphicsSceneMouseEvent* event);

signals:
  void SelectionDone(QRectF selectedArea);

private:
  std::vector<Cvb::ShapeFinder2::SearchResult> searchResult_;
  QPointF pointStart_;
  QPointF pointEnd_;
};

// The control widget
class ControlWidget : public QWidget
{
Q_OBJECT

public:
  explicit ControlWidget(QWidget* parent);
  ~ControlWidget() = default;
  void EnableGrabCheckBox(bool value);
  void SetResultInfo(size_t numResults);
  void SetTrainView(const Cvb::Image & image);

private:
  Cvb::UI::ImageView* trainImageView_ = nullptr;
  QCheckBox* grabCheckBox_ = nullptr;
  QCheckBox* contSearchCheckBox_ = nullptr;
  QLabel* resultLabel_ = nullptr;
};

// The main widget
class MainWidget : public QSplitter
{
  Q_OBJECT

public:
  explicit MainWidget();
  virtual ~MainWidget() {}

  void Search();

public Q_SLOTS:
  void HandleLoad();
  void HandleGrab(int);
  void HandleSnap();
  void HandleSave();
  void HandleLoadModel();
  void HandleSaveModel();
  void HandleSearch();
  void HandleTrain();
  void HandleContSearch(int);
  void HandleSelection(QRectF);

protected Q_SLOTS:
  void ShowSearchResult();

signals:
  void SearchDone();

private:
  CustomImageScene*                 imageScene_ = nullptr;
  Cvb::UI::ImageView*               imageView_ = nullptr;
  ControlWidget*                    ctrlWidget_ = nullptr;
  Cvb::DevicePtr                    device_ = nullptr;
  Cvb::ShapeFinder2::ClassifierPtr  classifier_;
  bool                              continuedSearch_ = false;
  bool                              isSelected_ = false;
  Cvb::ImagePtr                     selection_ = nullptr;
  Cvb::Rect<double>                 selectionRect_;
  std::vector<Cvb::ShapeFinder2::SearchResult> searchResult_;

  // The custom stream handler to handle image processing while grabbing
  class CustomStreamHandler : public Cvb::SingleStreamHandler
  {
  public:
    static std::unique_ptr<CustomStreamHandler> Create(const Cvb::StreamPtr& stream, MainWidget& mainWidget)
    {
      return std::unique_ptr<CustomStreamHandler>(new CustomStreamHandler(stream, mainWidget));
    }

    ~CustomStreamHandler()
    {
      TryFinish();
    }

  private:
    CustomStreamHandler(const Cvb::StreamPtr& stream, MainWidget& mainWidget)
      : Cvb::SingleStreamHandler(stream)
      , mainWidget_(mainWidget)
    {}

    void HandleAsyncStream(const Cvb::StreamPtr & stream) override
    {
      auto waitResult = stream->WaitFor(std::chrono::milliseconds(10000));
      if (!waitResult.Image)
        return;

      if (mainWidget_.continuedSearch_)
        mainWidget_.Search();
    }

    MainWidget& mainWidget_;

  };

  std::shared_ptr<CustomStreamHandler> streamHandler_ = nullptr;
};

// ---------------------------------------------------------------------------
// ---------------------------------------------------------------------------
#include "main_widget.hpp"


#include <QVBoxLayout>
#include <QPushButton>
#include <QFileDialog>
#include <QGroupBox>
#include <QMessageBox>
#include <QElapsedTimer>
#include <QGraphicsSceneMouseEvent>
#include <QRubberBand>
#include <QGraphicsRectItem>

#include <cvb/image.hpp>
#include <cvb/device_factory.hpp>
#include <cvb/device.hpp>
#include <cvb/async/single_stream_handler.hpp>
#include <cvb/driver/driver.hpp>
#include <cvb/shapefinder2/classifier_factory.hpp>

MainWidget::MainWidget()
  : QSplitter()
{
  imageScene_ = new CustomImageScene(this);

  imageView_ = new Cvb::UI::ImageView(imageScene_, this);
  imageView_->SetUploadMode(Cvb::UI::UploadMode::Viewport);
  imageView_->SetRenderEngine(Cvb::UI::RenderEngine::Raster);
  imageView_->resize(800, 600);

  imageView_->setDragMode(QGraphicsView::RubberBandDrag);
  QObject::connect(imageScene_, SIGNAL(SelectionDone(QRectF)), this, SLOT(HandleSelection(QRectF)));

  ctrlWidget_ = new ControlWidget(this);
  addWidget(imageView_);
  addWidget(ctrlWidget_);

  device_ = Cvb::DeviceFactory::Open(Cvb::InstallPath() + Cvb::String(CVB_LIT("drivers/CVMock.vin")), Cvb::AcquisitionStack::Vin);
  // Connect the device with the ImageView
  imageView_->Refresh(device_->DeviceImage(), Cvb::UI::AutoRefresh::On);
  // Create a stream handler
  streamHandler_ = CustomStreamHandler::Create(device_->Stream(), *this);
  // Enable the grab check box for devices
  ctrlWidget_->EnableGrabCheckBox(true);

  QObject::connect(this, SIGNAL(SearchDone()), this, SLOT(ShowSearchResult()));

  resize(1000, 800);

  // first snapshot
  device_->Stream()->GetSnapshot();
}

// Load an image or driver
void MainWidget::HandleLoad()
{
  imageScene_->ClearScene();

  auto fileName = QFileDialog::getOpenFileName(this, "Load Image"
    , Cvb::UI::CvbToQt(Cvb::ExpandPath(Cvb::InstallPath() + CVB_LIT("tutorial/ShapeFinder/Images/SF2/Lusterterminal")))
    , "CVB Emulator Image Lists (*.emu);;CVB Drivers (*.vin);;Images (*.bmp *.png *.tif *.tiff *.jpg)");
  if (fileName.isEmpty())
    return;

  if (fileName.contains(".emu", Qt::CaseInsensitive))
    fileName = fileName.replace("/", "\\"); // necessary for emu files

  try
  {
    // Either load an image or device (vin or emu)
    if (fileName.contains(".bmp", Qt::CaseInsensitive) || fileName.contains(".png", Qt::CaseInsensitive) || fileName.contains(".tif", Qt::CaseInsensitive) || fileName.contains(".jpg", Qt::CaseInsensitive))
    {
      Cvb::ImagePtr image = Cvb::Image::Load(Cvb::UI::QtToCvb(fileName));
      imageView_->Refresh(image);
      // Disable the grab check box for images
      ctrlWidget_->EnableGrabCheckBox(false);
    }
    else
    {
      device_ = Cvb::DeviceFactory::Open(Cvb::UI::QtToCvb(fileName), Cvb::AcquisitionStack::Vin);
      // Connect the device with the ImageView
      imageView_->Refresh(device_->DeviceImage(), Cvb::UI::AutoRefresh::On);
      // Create a stream handler
      streamHandler_ = CustomStreamHandler::Create(device_->Stream(), *this);
      // Enable the grab check box for devices
      ctrlWidget_->EnableGrabCheckBox(true);
    }
  }
  catch (const std::exception& error)
  {
    QMessageBox::critical(this, "Error", QString(error.what()));
  }
}

// Handle start / stop grab
void MainWidget::HandleGrab(int state)
{
  if (state == Qt::CheckState::Checked)
  {
    imageScene_->ClearScene();

    // Start streaming
    streamHandler_->Run();
  }
  else
  {
    // Disable possible blocking of the UI if necessary
    bool wait = imageView_->IsWaitForRepaintEnabled();
    if (wait)
      imageView_->SetWaitForRepaintEnabled(!wait);
    // Stop streaming
    streamHandler_->Finish();
    // Reset the previous state
    imageView_->SetWaitForRepaintEnabled(wait);
  }
}

// Handle a snap
void MainWidget::HandleSnap()
{
  device_->Stream()->GetSnapshot();
  if (continuedSearch_)
    HandleSearch();
  else
    imageScene_->ClearScene();
}

// Save the image
void MainWidget::HandleSave()
{
  auto fileName = QFileDialog::getSaveFileName(this, "Save Image"
    , Cvb::UI::CvbToQt(Cvb::InstallPath())
    , "Bitmap File (*.bmp);;CVB Emulator Image Lists (*.emu);;JPEG (*.jpg, *.jpeg);;Portable Network Graphic (*.png);;Tagged Image File Format (*.tif, *.tiff);;All Files (*.*)");
  if (fileName.isEmpty())
    return;
  imageView_->Image()->Save(Cvb::UI::QtToCvb(fileName));
}

// Load an SF2 model (classifier)
void MainWidget::HandleLoadModel()
{
  imageScene_->ClearScene();

  auto fileName = QFileDialog::getOpenFileName(this, "Load SF2 Model"
    , Cvb::UI::CvbToQt(Cvb::ExpandPath(Cvb::InstallPath() + CVB_LIT("tutorial/ShapeFinder/Images/SF2/Lusterterminal")))
    , "ShapeFinder2 models (*.sf2)");
  if (fileName.isEmpty())
    return;

  classifier_ = Cvb::Classifier::Create(Cvb::UI::QtToCvb(fileName));
}

// Save the SF2 model
void MainWidget::HandleSaveModel()
{
  auto fileName = QFileDialog::getSaveFileName(this, "Save SF2 Model"
    , Cvb::UI::CvbToQt(Cvb::ExpandPath(Cvb::InstallPath() + CVB_LIT("tutorial/ShapeFinder/Images/SF2/Lusterterminal")))
    , "ShapeFinder2 models (*.sf2)");
  if (fileName.isEmpty())
    return;

  classifier_->Save(Cvb::UI::QtToCvb(fileName));
}

// Train / learn the selection and create classifier
void MainWidget::HandleTrain()
{
  if (isSelected_)
  {
    ctrlWidget_->SetTrainView(*selection_);
    imageScene_->ClearScene();
    isSelected_ = false;
  }
  else
  {
    ctrlWidget_->SetTrainView(*imageView_->Image());
    auto aoi = imageView_->Image()->Bounds();
    selectionRect_ = Cvb::Rect<double>(aoi.Left(), aoi.Top(), aoi.Right(), aoi.Bottom());
  }

  Cvb::ShapeFinder2::ClassifierFactory factory;

  double centerX = selectionRect_.Left() + ((selectionRect_.Right() - selectionRect_.Left()) / 2);
  double centerY = selectionRect_.Top() + ((selectionRect_.Bottom() - selectionRect_.Top()) / 2);

  Cvb::Rect<int> teachWindow(selectionRect_.Left() - centerX, selectionRect_.Top() - centerY, selectionRect_.Right() - centerX, selectionRect_.Bottom() - centerY);
  try
  {
    classifier_ = factory.Learn(imageView_->Image()->Plane(0), Cvb::Point2D<double>(centerX, centerY), teachWindow);
  }
  catch (const std::exception & e)
  {
    QMessageBox::critical(this, "Error", e.what());
  }
}

// Run a search (search button clicked)
void MainWidget::HandleSearch()
{
  if (!classifier_)
  {
    QMessageBox::warning(this, "SF2 Search", "Please create (train) or load a model/classifier first.");
    return;
  }

  Search();
}

// Activate the continuous search
void MainWidget::HandleContSearch(int state)
{
  imageScene_->ClearScene();

  if (state == Qt::Checked)
    continuedSearch_ = true;
  else
    continuedSearch_ = false;
}

// Area has been selected
void MainWidget::HandleSelection(QRectF selection)
{
  Cvb::Area2D area(Cvb::UI::QtToCvb(selection));
  // Convert the rectangle to pixel coordinates
  auto convertedArea = imageView_->Image()->ImageToPixelCoordinates(area);
  // Get the bounds
  Cvb::Rect<double> boundingRect = convertedArea.BoundingRectangle();
  // Get the selected part of the image
  selection_ = imageView_->Image()->Map(Cvb::Rect<int>(boundingRect.Left(), boundingRect.Top(), boundingRect.Right(), boundingRect.Bottom()));
  isSelected_ = true;

  selectionRect_ = boundingRect;
}

// Run the search on the classifier loaded
void MainWidget::Search()
{
  if (!classifier_)
    return;

  Cvb::ShapeFinder2::PrecisionMode mode = Cvb::ShapeFinder2::PrecisionMode::CorrelationFine; // Defines how many and which search phases will be carried out.
  double relativeThreshold  = 0.55; // Defines the quality span between the best and the worst result candidate.
  int minimumThreshold      = 20; // Absolute quality threshold a result must exceed to make it into the result list.
  int coarseLocality        = 10; // Minimum distance for solutions in the coarse layer before correlation hill climbing happens.

  if (isSelected_)
  {
    Cvb::Rect<int> aoi(selectionRect_.Left(), selectionRect_.Top(), selectionRect_.Right(), selectionRect_.Bottom());
    searchResult_ = classifier_->SearchAll(imageView_->Image()->Plane(0), aoi, mode, relativeThreshold, minimumThreshold, coarseLocality);
  }
  else
  {
    searchResult_ = classifier_->SearchAll(imageView_->Image()->Plane(0), imageView_->Image()->Bounds(), mode, relativeThreshold, minimumThreshold, coarseLocality);
  }

  emit SearchDone();
}

// Display the search result (result info and overlay)
void MainWidget::ShowSearchResult()
{
  isSelected_ = false;
  ctrlWidget_->SetResultInfo(searchResult_.size());
  imageScene_->SetSearchResult(*imageView_->Image(), searchResult_);
}


ControlWidget::ControlWidget(QWidget* parent)
  : QWidget(parent)
{
  auto mainLayout = new QGridLayout;

  trainImageView_ = new Cvb::UI::ImageView(this);
  trainImageView_->SetUploadMode(Cvb::UI::UploadMode::Image);
  trainImageView_->SetRenderEngine(Cvb::UI::RenderEngine::Raster);

  QPushButton* openImgBtn = new QPushButton(tr("Open Image"));
  QObject::connect(openImgBtn, SIGNAL(clicked()), parent, SLOT(HandleLoad()));

  grabCheckBox_ = new QCheckBox(tr("Grab"));
  QObject::connect(grabCheckBox_, SIGNAL(stateChanged(int)), parent, SLOT(HandleGrab(int)));

  QPushButton* snapBtn = new QPushButton(tr("Snap"));
  QObject::connect(snapBtn, SIGNAL(clicked()), parent, SLOT(HandleSnap()));

  QPushButton* saveBtn = new QPushButton(tr("Save"));
  QObject::connect(saveBtn, SIGNAL(clicked()), parent, SLOT(HandleSave()));

  contSearchCheckBox_ = new QCheckBox(tr("Continued Search"));
  QObject::connect(contSearchCheckBox_, SIGNAL(stateChanged(int)), parent, SLOT(HandleContSearch(int)));

  QGroupBox* groupBox = new QGroupBox(tr(""));

  QPushButton* loadModelBtn = new QPushButton(tr("Load Model"));
  QObject::connect(loadModelBtn, SIGNAL(clicked()), parent, SLOT(HandleLoadModel()));

  QPushButton* saveModelBtn = new QPushButton(tr("Save Model"));
  QObject::connect(saveModelBtn, SIGNAL(clicked()), parent, SLOT(HandleSaveModel()));

  QPushButton* trainBtn = new QPushButton(tr("Train"));
  QObject::connect(trainBtn, SIGNAL(clicked()), parent, SLOT(HandleTrain()));

  QPushButton* searchBtn = new QPushButton(tr("Search"));
  QObject::connect(searchBtn, SIGNAL(clicked()), parent, SLOT(HandleSearch()));

  resultLabel_ = new QLabel(tr("Number of search results: 0"));

  QGridLayout* gbLayout = new QGridLayout;
  gbLayout->addWidget(trainBtn, 0, 0);
  gbLayout->addWidget(loadModelBtn, 0, 1);
  gbLayout->addWidget(searchBtn, 1, 0);
  gbLayout->addWidget(saveModelBtn, 1, 1);
  gbLayout->addWidget(resultLabel_, 2, 0, 1, -1);
  groupBox->setLayout(gbLayout);

  mainLayout->addWidget(trainImageView_, 0, 0, 1, -1);
  mainLayout->addWidget(openImgBtn, 1, 0, 1, 1);
  mainLayout->addWidget(grabCheckBox_, 1, 1, 1, 1);
  mainLayout->addWidget(snapBtn, 1, 2, 1, 1);
  mainLayout->addWidget(saveBtn, 2, 0, 1, 1);
  mainLayout->addWidget(contSearchCheckBox_, 2, 1, 1, -1);
  mainLayout->addWidget(groupBox, 3, 0, 1, -1);

  setLayout(mainLayout);
}

void ControlWidget::EnableGrabCheckBox(bool value)
{
  grabCheckBox_->setEnabled(value);
}

void ControlWidget::SetResultInfo(size_t numResults)
{
  QString count = "Number of search results: " + QString::number(numResults);
  resultLabel_->setText(count);
}

void ControlWidget::SetTrainView(const Cvb::Image & image)
{
  if (trainImageView_)
    trainImageView_->Refresh(image);
}


CustomImageScene::CustomImageScene(QWidget* parent)
  : Cvb::UI::ImageScene(parent)
{}

void CustomImageScene::SetSearchResult(const Cvb::Image & image, std::vector<Cvb::ShapeFinder2::SearchResult> result)
{
  // Clear the scene
  ClearScene();

  searchResult_ = result;

  // Draw overlays for search results
  for (int i = 0; i < searchResult_.size(); i++)
  {
    Cvb::ShapeFinder2::SearchResult res = result.at(i);
    double x = res.X();
    double y = res.Y();

    // Draw a small rectangle at the result pixel location
    auto point = image.PixelToImageCoordinates(Cvb::Point2D<double>(x, y));
    QGraphicsRectItem* item = new QGraphicsRectItem(QRectF(point.X(), point.Y(), 5, 5));
    item->setBrush(QBrush(Qt::red));
    this->addItem(item);

    // Draw a label right beside the pixel
    QString label = "Q: " + QString::number(res.Quality(), 'f', 2) + ", A: " + QString::number(res.Rotation().Deg(), 'f', 2);
    QGraphicsRectItem* textLabel = new QGraphicsRectItem(QRectF(point.X() + 5, point.Y() + 5, 100, 15));
    textLabel->setBrush(QBrush(Qt::red));
    this->addItem(textLabel);
    QGraphicsTextItem* text = this->addText(label);
    text->setPos(point.X() + 5, point.Y() + 2.5);
    text->setDefaultTextColor(Qt::black);
  }
}

void CustomImageScene::ClearScene()
{
  this->clear();
  this->update();
}

void CustomImageScene::mousePressEvent(QGraphicsSceneMouseEvent* event)
{
  ClearScene();
  this->ImageView()->setDragMode(QGraphicsView::RubberBandDrag);

  // remember the start point
  pointStart_ = event->scenePos();
}

void CustomImageScene::mouseReleaseEvent(QGraphicsSceneMouseEvent* event)
{
  // remember the end point
  pointEnd_ = event->scenePos();
  // draw a rectangle to mark the selection
  this->addRect(QRectF(pointStart_, pointEnd_), QPen(Qt::blue, 0));
  // signal to react to the selection
  emit SelectionDone(QRectF(pointStart_, pointEnd_));
}