ShapeFinder/Cvb++/QtShapeFinder2Cuda

This example program is located in your CVB installation under %CVB%Tutorial/ShapeFinder/Cvb++/QtShapeFinder2Cuda.

main.cpp:

#ifdef _MSC_VER
#pragma warning(push, 1)
#pragma warning(disable : 4702)
#endif
 
#include <QApplication>
#include <QIcon>
 
#include "main_widget.hpp"
 
#ifdef _MSC_VER
#pragma warning(pop)
#endif
 
int main(int argc, char *argv[]) {
 
  QApplication application(argc, argv);
  MainWidget mainWidget;
  mainWidget.setWindowIcon(QIcon(":/qttutorial.png"));
  mainWidget.show();
  application.exec();
}

main_widget.cpp:

// ---------------------------------------------------------------------------
// ---------------------------------------------------------------------------
#include "main_widget.hpp"
 
#include <QElapsedTimer>
#include <QFileDialog>
#include <QGraphicsRectItem>
#include <QGraphicsSceneMouseEvent>
#include <QGroupBox>
#include <QMessageBox>
#include <QPushButton>
#include <QRubberBand>
#include <QVBoxLayout>
 
#include <cvb/async/single_stream_handler.hpp>
#include <cvb/device.hpp>
#include <cvb/device_factory.hpp>
#include <cvb/driver/driver.hpp>
#include <cvb/image.hpp>
#include <cvb/shapefinder2/classifier_factory.hpp>
 
MainWidget::MainWidget() : QSplitter() {
  imageScene_ = new CustomImageScene(this);
 
  imageView_ = new Cvb::UI::ImageView(imageScene_, this);
  imageView_->SetUploadMode(Cvb::UI::UploadMode::Viewport);
  imageView_->SetRenderEngine(Cvb::UI::RenderEngine::Raster);
  imageView_->resize(800, 600);
 
  imageView_->setDragMode(QGraphicsView::RubberBandDrag);
  QObject::connect(imageScene_, SIGNAL(SelectionDone(QRectF)), this,
                   SLOT(HandleSelection(QRectF)));
 
  ctrlWidget_ = new ControlWidget(this);
  addWidget(imageView_);
  addWidget(ctrlWidget_);
 
  device_ = Cvb::DeviceFactory::Open(
      Cvb::InstallPath() + Cvb::String(CVB_LIT("drivers/CVMock.vin")),
      Cvb::AcquisitionStack::Vin);
  // Connect the device with the ImageView
  imageView_->Refresh(device_->DeviceImage(), Cvb::UI::AutoRefresh::On);
  // Create a stream handler
  streamHandler_ = CustomStreamHandler::Create(device_->Stream(), *this);
  // Enable the grab check box for devices
  ctrlWidget_->EnableGrabCheckBox(true);
 
  QObject::connect(this, SIGNAL(SearchDone()), this, SLOT(ShowSearchResult()));
 
  resize(1000, 800);
 
  // first snapshot
  device_->Stream()->GetSnapshot();
}
 
// Load an image or driver
void MainWidget::HandleLoad() {
  imageScene_->ClearScene();
 
  auto fileName = QFileDialog::getOpenFileName(
      this, "Load Image",
      Cvb::UI::CvbToQt(Cvb::ExpandPath(
          Cvb::InstallPath() +
          CVB_LIT("tutorial/ShapeFinder/Images/SF2/Lusterterminal"))),
      "CVB Emulator Image Lists (*.emu);;CVB Drivers (*.vin);;Images (*.bmp "
      "*.png *.tif *.tiff *.jpg)");
  if (fileName.isEmpty())
    return;
 
  if (fileName.contains(".emu", Qt::CaseInsensitive))
    fileName = fileName.replace("/", "\\"); // necessary for emu files
 
  try {
    // Either load an image or device (vin or emu)
    if (fileName.contains(".bmp", Qt::CaseInsensitive) ||
        fileName.contains(".png", Qt::CaseInsensitive) ||
        fileName.contains(".tif", Qt::CaseInsensitive) ||
        fileName.contains(".jpg", Qt::CaseInsensitive)) {
      Cvb::ImagePtr image = Cvb::Image::Load(Cvb::UI::QtToCvb(fileName));
      imageView_->Refresh(image);
      // Disable the grab check box for images
      ctrlWidget_->EnableGrabCheckBox(false);
    } else {
      device_ = Cvb::DeviceFactory::Open(Cvb::UI::QtToCvb(fileName),
                                         Cvb::AcquisitionStack::Vin);
      // Connect the device with the ImageView
      imageView_->Refresh(device_->DeviceImage(), Cvb::UI::AutoRefresh::On);
      // Create a stream handler
      streamHandler_ = CustomStreamHandler::Create(device_->Stream(), *this);
      // Enable the grab check box for devices
      ctrlWidget_->EnableGrabCheckBox(true);
    }
  } catch (const std::exception &error) {
    QMessageBox::critical(this, "Error", QString(error.what()));
  }
}
 
// Handle start / stop grab
void MainWidget::HandleGrab(int state) {
  if (state == Qt::CheckState::Checked) {
    imageScene_->ClearScene();
 
    // Start streaming
    streamHandler_->Run();
  } else {
    // Disable possible blocking of the UI if necessary
    bool wait = imageView_->IsWaitForRepaintEnabled();
    if (wait)
      imageView_->SetWaitForRepaintEnabled(!wait);
    // Stop streaming
    streamHandler_->Finish();
    // Reset the previous state
    imageView_->SetWaitForRepaintEnabled(wait);
  }
}
 
// Handle a snap
void MainWidget::HandleSnap() {
  device_->Stream()->GetSnapshot();
  if (continuedSearch_)
    HandleSearch();
  else
    imageScene_->ClearScene();
}
 
// Save the image
void MainWidget::HandleSave() {
  auto fileName = QFileDialog::getSaveFileName(
      this, "Save Image", Cvb::UI::CvbToQt(Cvb::InstallPath()),
      "Bitmap File (*.bmp);;CVB Emulator Image Lists (*.emu);;JPEG (*.jpg, "
      "*.jpeg);;Portable Network Graphic (*.png);;Tagged Image File Format "
      "(*.tif, *.tiff);;All Files (*.*)");
  if (fileName.isEmpty())
    return;
  imageView_->Image()->Save(Cvb::UI::QtToCvb(fileName));
}
 
// Load an SF2 model (classifier)
void MainWidget::HandleLoadModel() {
  imageScene_->ClearScene();
 
  auto fileName = QFileDialog::getOpenFileName(
      this, "Load SF2 Model",
      Cvb::UI::CvbToQt(Cvb::ExpandPath(
          Cvb::InstallPath() +
          CVB_LIT("tutorial/ShapeFinder/Images/SF2/Lusterterminal"))),
      "ShapeFinder2 models (*.sf2)");
  if (fileName.isEmpty())
    return;
 
  classifier_ = Cvb::Classifier::Create(Cvb::UI::QtToCvb(fileName));
}
 
// Save the SF2 model
void MainWidget::HandleSaveModel() {
  auto fileName = QFileDialog::getSaveFileName(
      this, "Save SF2 Model",
      Cvb::UI::CvbToQt(Cvb::ExpandPath(
          Cvb::InstallPath() +
          CVB_LIT("tutorial/ShapeFinder/Images/SF2/Lusterterminal"))),
      "ShapeFinder2 models (*.sf2)");
  if (fileName.isEmpty())
    return;
 
  classifier_->Save(Cvb::UI::QtToCvb(fileName));
}
 
// Train / learn the selection and create classifier
void MainWidget::HandleTrain() {
  if (isSelected_) {
    ctrlWidget_->SetTrainView(*selection_);
    imageScene_->ClearScene();
    isSelected_ = false;
  } else {
    ctrlWidget_->SetTrainView(*imageView_->Image());
    auto aoi = imageView_->Image()->Bounds();
    selectionRect_ =
        Cvb::Rect<double>(aoi.Left(), aoi.Top(), aoi.Right(), aoi.Bottom());
  }
 
  Cvb::ShapeFinder2::ClassifierFactory factory;
 
  auto centerX = static_cast<int>(
      std::round(selectionRect_.Left() +
                 ((selectionRect_.Right() - selectionRect_.Left()) / 2)));
  auto centerY = static_cast<int>(
      std::round(selectionRect_.Top() +
                 ((selectionRect_.Bottom() - selectionRect_.Top()) / 2)));
 
  Cvb::Rect<int> teachWindow(
      selectionRect_.Left() - centerX, selectionRect_.Top() - centerY,
      selectionRect_.Right() - centerX, selectionRect_.Bottom() - centerY);
  try {
    classifier_ =
        factory.Learn(imageView_->Image()->Plane(0),
                      Cvb::Point2D<int>(centerX, centerY), teachWindow);
  } catch (const std::exception &e) {
    QMessageBox::critical(this, "Error", e.what());
  }
}
 
// Run a search (search button clicked)
void MainWidget::HandleSearch() {
  if (!classifier_) {
    QMessageBox::warning(
        this, "SF2 Search",
        "Please create (train) or load a model/classifier first.");
    return;
  }
 
  Search();
}
 
// Activate the continuous search
void MainWidget::HandleContSearch(int state) {
  imageScene_->ClearScene();
 
  if (state == Qt::Checked)
    continuedSearch_ = true;
  else
    continuedSearch_ = false;
}
 
template <typename T> Cvb::Rect<T> AsRect(const Cvb::RectLT<T> &rectLT) {
  return Cvb::Rect<T>(Cvb::Point2D<T>(rectLT.Location()),
                      Cvb::Size2D<T>(rectLT.Size()));
}
 
// Area has been selected
void MainWidget::HandleSelection(QRectF selection) {
  Cvb::Area2D area(AsRect(Cvb::UI::QtToCvb(selection)));
  // Convert the rectangle to pixel coordinates
  auto convertedArea = imageView_->Image()->ImageToPixelCoordinates(area);
  // Get the bounds
  Cvb::Rect<double> boundingRect = convertedArea.BoundingRectangle();
  // Get the selected part of the image
  selection_ = imageView_->Image()->Map(
      Cvb::Rect<int>(boundingRect.Left(), boundingRect.Top(),
                     boundingRect.Right(), boundingRect.Bottom()));
  isSelected_ = true;
 
  selectionRect_ = boundingRect;
}
 
// Run the search on the classifier loaded
void MainWidget::Search() {
  if (!classifier_)
    return;
 
  Cvb::ShapeFinder2::PrecisionMode mode =
      Cvb::ShapeFinder2::PrecisionMode::NoCorrelation; // Defines how many and
                                                       // which search phases
                                                       // will be carried out.
  double relativeThreshold = 0.55; // Defines the quality span between the best
                                   // and the worst result candidate.
  int minimumThreshold = 20; // Absolute quality threshold a result must exceed
                             // to make it into the result list.
  int coarseLocality = 10; // Minimum distance for solutions in the coarse layer
                           // before correlation hill climbing happens.
 
  try {
    if (isSelected_) {
      Cvb::Rect<int> aoi(selectionRect_.Left(), selectionRect_.Top(),
                         selectionRect_.Right(), selectionRect_.Bottom());
      SearchWithCuda(imageView_->Image()->Plane(0), aoi, mode,
                     relativeThreshold, minimumThreshold, coarseLocality);
      SearchWithoutCuda(imageView_->Image()->Plane(0), aoi, mode,
                        relativeThreshold, minimumThreshold, coarseLocality);
    } else {
      SearchWithCuda(imageView_->Image()->Plane(0),
                     imageView_->Image()->Bounds(), mode, relativeThreshold,
                     minimumThreshold, coarseLocality);
      SearchWithoutCuda(imageView_->Image()->Plane(0),
                        imageView_->Image()->Bounds(), mode, relativeThreshold,
                        minimumThreshold, coarseLocality);
    }
  } catch (const std::exception &e) {
    QMessageBox::critical(this, "Error", e.what());
  }
 
  emit SearchDone();
}
 
void MainWidget::SearchWithCuda(const Cvb::ImagePlane &plane,
                                Cvb::Rect<int> aoi,
                                Cvb::ShapeFinder2::PrecisionMode mode,
                                double relativeThreshold, int minimumThreshold,
                                int coarseLocality) {
  classifier_->UseCuda(Cvb::ShapeFinder2::CudaStatus::Default);
  std::chrono::steady_clock::time_point begin =
      std::chrono::steady_clock::now();
  searchResult_ = classifier_->SearchAll(plane, aoi, mode, relativeThreshold,
                                         minimumThreshold, coarseLocality);
  std::chrono::steady_clock::time_point end = std::chrono::steady_clock::now();
  durationWithCuda_ =
      std::chrono::duration_cast<std::chrono::milliseconds>(end - begin)
          .count();
}
 
void MainWidget::SearchWithoutCuda(const Cvb::ImagePlane &plane,
                                   Cvb::Rect<int> aoi,
                                   Cvb::ShapeFinder2::PrecisionMode mode,
                                   double relativeThreshold,
                                   int minimumThreshold, int coarseLocality) {
  classifier_->UseCuda(Cvb::ShapeFinder2::CudaStatus::ForceDisable);
  std::chrono::steady_clock::time_point begin =
      std::chrono::steady_clock::now();
  searchResult_ = classifier_->SearchAll(plane, aoi, mode, relativeThreshold,
                                         minimumThreshold, coarseLocality);
  std::chrono::steady_clock::time_point end = std::chrono::steady_clock::now();
  durationWithoutCuda_ =
      std::chrono::duration_cast<std::chrono::milliseconds>(end - begin)
          .count();
}
 
// Display the search result (result info and overlay)
void MainWidget::ShowSearchResult() {
  isSelected_ = false;
  ctrlWidget_->SetResultInfo(searchResult_.size());
  imageScene_->SetSearchResult(*imageView_->Image(), searchResult_,
                               durationWithCuda_, durationWithoutCuda_);
}
 
ControlWidget::ControlWidget(QWidget *parent) : QWidget(parent) {
  auto mainLayout = new QGridLayout;
 
  trainImageView_ = new Cvb::UI::ImageView(this);
  trainImageView_->SetUploadMode(Cvb::UI::UploadMode::Image);
  trainImageView_->SetRenderEngine(Cvb::UI::RenderEngine::Raster);
 
  QPushButton *openImgBtn = new QPushButton(tr("Open Image"));
  QObject::connect(openImgBtn, SIGNAL(clicked()), parent, SLOT(HandleLoad()));
 
  grabCheckBox_ = new QCheckBox(tr("Grab"));
  QObject::connect(grabCheckBox_, SIGNAL(stateChanged(int)), parent,
                   SLOT(HandleGrab(int)));
 
  QPushButton *snapBtn = new QPushButton(tr("Snap"));
  QObject::connect(snapBtn, SIGNAL(clicked()), parent, SLOT(HandleSnap()));
 
  QPushButton *saveBtn = new QPushButton(tr("Save"));
  QObject::connect(saveBtn, SIGNAL(clicked()), parent, SLOT(HandleSave()));
 
  contSearchCheckBox_ = new QCheckBox(tr("Continued Search"));
  QObject::connect(contSearchCheckBox_, SIGNAL(stateChanged(int)), parent,
                   SLOT(HandleContSearch(int)));
 
  QGroupBox *groupBox = new QGroupBox(tr(""));
 
  QPushButton *loadModelBtn = new QPushButton(tr("Load Model"));
  QObject::connect(loadModelBtn, SIGNAL(clicked()), parent,
                   SLOT(HandleLoadModel()));
 
  QPushButton *saveModelBtn = new QPushButton(tr("Save Model"));
  QObject::connect(saveModelBtn, SIGNAL(clicked()), parent,
                   SLOT(HandleSaveModel()));
 
  QPushButton *trainBtn = new QPushButton(tr("Train"));
  QObject::connect(trainBtn, SIGNAL(clicked()), parent, SLOT(HandleTrain()));
 
  QPushButton *searchBtn = new QPushButton(tr("Search"));
  QObject::connect(searchBtn, SIGNAL(clicked()), parent, SLOT(HandleSearch()));
 
  resultLabel_ = new QLabel(tr("Number of search results: 0"));
 
  QGridLayout *gbLayout = new QGridLayout;
  gbLayout->addWidget(trainBtn, 0, 0);
  gbLayout->addWidget(loadModelBtn, 0, 1);
  gbLayout->addWidget(searchBtn, 1, 0);
  gbLayout->addWidget(saveModelBtn, 1, 1);
  gbLayout->addWidget(resultLabel_, 2, 0, 1, -1);
  groupBox->setLayout(gbLayout);
 
  mainLayout->addWidget(trainImageView_, 0, 0, 1, -1);
  mainLayout->addWidget(openImgBtn, 1, 0, 1, 1);
  mainLayout->addWidget(grabCheckBox_, 1, 1, 1, 1);
  mainLayout->addWidget(snapBtn, 1, 2, 1, 1);
  mainLayout->addWidget(saveBtn, 2, 0, 1, 1);
  mainLayout->addWidget(contSearchCheckBox_, 2, 1, 1, -1);
  mainLayout->addWidget(groupBox, 3, 0, 1, -1);
 
  setLayout(mainLayout);
}
 
void ControlWidget::EnableGrabCheckBox(bool value) {
  grabCheckBox_->setEnabled(value);
}
 
void ControlWidget::SetResultInfo(size_t numResults) {
  QString count = "Number of search results: " + QString::number(numResults);
  resultLabel_->setText(count);
}
 
void ControlWidget::SetTrainView(const Cvb::Image &image) {
  if (trainImageView_)
    trainImageView_->Refresh(image);
}
 
CustomImageScene::CustomImageScene(QWidget *parent)
    : Cvb::UI::ImageScene(parent) {}
 
void CustomImageScene::SetSearchResult(
    const Cvb::Image &image,
    std::vector<Cvb::ShapeFinder2::SearchResult> result,
    long long durationWithCuda, long long durationWithoutCuda) {
  // Clear the scene
  ClearScene();
 
  searchResult_ = result;
 
  // Draw overlays for search results
  for (int i = 0; i < searchResult_.size(); i++) {
    Cvb::ShapeFinder2::SearchResult res = result.at(i);
    double x = res.X();
    double y = res.Y();
 
    // Draw a small rectangle at the result pixel location
    auto point = image.PixelToImageCoordinates(Cvb::Point2D<double>(x, y));
    QGraphicsRectItem *item =
        new QGraphicsRectItem(QRectF(point.X(), point.Y(), 5, 5));
    item->setBrush(QBrush(Qt::red));
    this->addItem(item);
 
    // Draw a label right beside the pixel
    QString label = "Q: " + QString::number(res.Quality(), 'f', 2) +
                    ", A: " + QString::number(res.Rotation().Deg(), 'f', 2);
    QGraphicsRectItem *textLabel =
        new QGraphicsRectItem(QRectF(point.X() + 5, point.Y() + 5, 100, 15));
    textLabel->setBrush(QBrush(Qt::red));
    this->addItem(textLabel);
    QGraphicsTextItem *text = this->addText(label);
    text->setPos(point.X() + 5, point.Y() + 2.5);
    text->setDefaultTextColor(Qt::black);
 
    // Draw label for search duration
    QString label1 = "Duration with Cuda: ";
    QString label2 = "Duration without Cuda: ";
    label1.append(QString::number(durationWithCuda)).append("ms");
    label2.append(QString::number(durationWithoutCuda)).append("ms");
    QGraphicsTextItem *text1 = this->addText(label1);
    text1->setPos(0, 0);
    text1->setDefaultTextColor(Qt::green);
    text1->setScale(2);
    QGraphicsTextItem *text2 = this->addText(label2);
    text2->setPos(0, 30);
    text2->setDefaultTextColor(Qt::red);
    text2->setScale(2);
  }
}
 
void CustomImageScene::ClearScene() {
  this->clear();
  this->update();
}
 
void CustomImageScene::mousePressEvent(QGraphicsSceneMouseEvent *event) {
  ClearScene();
  this->ImageView()->setDragMode(QGraphicsView::RubberBandDrag);
 
  // remember the start point
  pointStart_ = event->scenePos();
}
 
void CustomImageScene::mouseReleaseEvent(QGraphicsSceneMouseEvent *event) {
  // remember the end point
  pointEnd_ = event->scenePos();
  // draw a rectangle to mark the selection
  this->addRect(QRectF(pointStart_, pointEnd_), QPen(Qt::blue, 0));
  // signal to react to the selection
  emit SelectionDone(QRectF(pointStart_, pointEnd_));
}

main_widget.hpp:

#pragma once
 
#include <chrono>
 
#include <QCheckBox>
#include <QLabel>
#include <QSplitter>
 
#include <cvb/async/single_stream_handler.hpp>
#include <cvb/shapefinder2/classifier.hpp>
#include <cvb/ui/image_view.hpp>
 
// The custom image scene to draw overlays
class CustomImageScene : public Cvb::UI::ImageScene {
  Q_OBJECT
 
public:
  explicit CustomImageScene(QWidget *parent = 0);
  void SetSearchResult(const Cvb::Image &image,
                       std::vector<Cvb::ShapeFinder2::SearchResult> result,
                       long long durationWithCuda,
                       long long durationWithoutCuda);
  void ClearScene();
  void mousePressEvent(QGraphicsSceneMouseEvent *event);
  void mouseReleaseEvent(QGraphicsSceneMouseEvent *event);
 
signals:
  void SelectionDone(QRectF selectedArea);
 
private:
  std::vector<Cvb::ShapeFinder2::SearchResult> searchResult_;
  QPointF pointStart_;
  QPointF pointEnd_;
};
 
// The control widget
class ControlWidget : public QWidget {
  Q_OBJECT
 
public:
  explicit ControlWidget(QWidget *parent);
  ~ControlWidget() = default;
  void EnableGrabCheckBox(bool value);
  void SetResultInfo(size_t numResults);
  void SetTrainView(const Cvb::Image &image);
 
private:
  Cvb::UI::ImageView *trainImageView_ = nullptr;
  QCheckBox *grabCheckBox_ = nullptr;
  QCheckBox *contSearchCheckBox_ = nullptr;
  QLabel *resultLabel_ = nullptr;
};
 
// The main widget
class MainWidget : public QSplitter {
  Q_OBJECT
 
public:
  explicit MainWidget();
  virtual ~MainWidget() {}
 
  void Search();
 
public Q_SLOTS:
  void HandleLoad();
  void HandleGrab(int);
  void HandleSnap();
  void HandleSave();
  void HandleLoadModel();
  void HandleSaveModel();
  void HandleSearch();
  void HandleTrain();
  void HandleContSearch(int);
  void HandleSelection(QRectF);
 
protected Q_SLOTS:
  void ShowSearchResult();
 
signals:
  void SearchDone();
 
private:
  CustomImageScene *imageScene_ = nullptr;
  Cvb::UI::ImageView *imageView_ = nullptr;
  ControlWidget *ctrlWidget_ = nullptr;
  Cvb::DevicePtr device_ = nullptr;
  Cvb::ShapeFinder2::ClassifierPtr classifier_;
  bool continuedSearch_ = false;
  bool isSelected_ = false;
  Cvb::ImagePtr selection_ = nullptr;
  Cvb::Rect<double> selectionRect_;
  std::vector<Cvb::ShapeFinder2::SearchResult> searchResult_;
  long long durationWithCuda_ = 0;
  long long durationWithoutCuda_ = 0;
  // bool                              useCuda_ = false;
 
  void SearchWithCuda(const Cvb::ImagePlane &plane, Cvb::Rect<int> aoi,
                      Cvb::ShapeFinder2::PrecisionMode mode,
                      double relativeThreshold, int minimumThreshold,
                      int coarseLocality);
  void SearchWithoutCuda(const Cvb::ImagePlane &plane, Cvb::Rect<int> aoi,
                         Cvb::ShapeFinder2::PrecisionMode mode,
                         double relativeThreshold, int minimumThreshold,
                         int coarseLocality);
 
  // The custom stream handler to handle image processing while grabbing
  class CustomStreamHandler : public Cvb::SingleStreamHandler {
  public:
    static std::unique_ptr<CustomStreamHandler>
    Create(const Cvb::StreamPtr &stream, MainWidget &mainWidget) {
      return std::unique_ptr<CustomStreamHandler>(
          new CustomStreamHandler(stream, mainWidget));
    }
 
    ~CustomStreamHandler() { TryFinish(); }
 
  private:
    CustomStreamHandler(const Cvb::StreamPtr &stream, MainWidget &mainWidget)
        : Cvb::SingleStreamHandler(stream), mainWidget_(mainWidget) {}
 
    void HandleAsyncStream(const Cvb::StreamPtr &stream) override {
      auto waitResult = stream->WaitFor(std::chrono::milliseconds(10000));
      if (!waitResult.Image)
        return;
 
      if (mainWidget_.continuedSearch_)
        mainWidget_.Search();
    }
 
    MainWidget &mainWidget_;
  };
 
  std::shared_ptr<CustomStreamHandler> streamHandler_ = nullptr;
};