Polimago/QmlCookieClassification

This example requires Qt5 >= 5.9 setup for building.

You may build it with Ubuntu 18.04's default Qt5 after installing:

• qt5-default
• qtdeclarative5-dev
• qml-module-qtquick2
• qml-module-qtquick-controls
• qml-module-qtquick-dialogs

// ---------------------------------------------------------------------------
// ---------------------------------------------------------------------------

#include <iostream>

#include <QGuiApplication>
#include <QQmlApplicationEngine>
#include <QQmlContext>
#include <QQuickView>
#include <QIcon>

#include <cvb/ui/image_view_item.hpp>


#include "classification.hpp"
#include "result_model.hpp"

int main(int argc, char* argv[])
{
  try
  {
    QGuiApplication app(argc, argv);
    app.setWindowIcon(QIcon(":/qttutorial.png"));


    // setup QML interface objects
    Cvb::UI::ImageController imageController;
    ResultModel resultModel(imageController);

    // main search object
    Classification classification(resultModel);


    // register QML components for an image display
    Cvb::UI::ImageViewItem::Register();
    Cvb::UI::ImageLabelItem::Register();


    QQmlApplicationEngine engine;
    auto context = engine.rootContext();
    // create a controller object to communicate with QML
    context->setContextProperty("mainImage", &imageController);
    // create a result model to communicate with QML for overlays
    context->setContextProperty("resultModel", &resultModel);
    // create a classification object to communicate with QML (snap, count)
    context->setContextProperty("classification", &classification);

    // load main QML file
    engine.load(QUrl::fromLocalFile("../main.qml"));


    return app.exec();
  }
  catch (const std::exception& error)
  {
    std::cout << error.what() << std::endl;
  }
}

The classification class:

#include <QObject>
#include <QVariant>

#include <cvb/device_factory.hpp>
#include <cvb/async/single_stream_handler.hpp>
#include <cvb/polimago/search_predictor.hpp>
#include <cvb/polimago/classification_predictor.hpp>
#include <cvb/ui/image_view_item.hpp>
#include <cvb/utilities/stop_watch.hpp>

#include "result_model.hpp"


class Classification final:
  public QObject

{
  Q_OBJECT

  Q_PROPERTY(QUrl source READ Source WRITE SetSource NOTIFY NotifySource);
  Q_PROPERTY(QUrl searchPredictor READ SearchPredictor WRITE SetSearchPredictor NOTIFY NotifySearchPredictor);
  Q_PROPERTY(QUrl classificationPredictor READ ClassificationPredictor WRITE SetClassificationPredictor NOTIFY NotifyClassificationPredictor);


  public:

    Classification(ResultModel& model);


  public Q_SLOTS:


    void Snap();
    void Classify();

  private Q_SLOTS:

    void OnSourceChanged();
    void OnSearchPredictorChanged();
    void OnClassificationPredictorChnaged();

  private:

    QUrl Source() const
    {
      return source_;
    }

    void SetSource(const QUrl & source)
    {
      if (source_ == source)
        return;
      source_ = source;
      NotifySource();
    }

    QUrl SearchPredictor() const
    {
      return searchPredictor_;
    }

    void SetSearchPredictor(const QUrl & searchPredictor)
    {
      if (searchPredictor_ == searchPredictor)
        return;
      searchPredictor_ = searchPredictor;
      NotifySearchPredictor();
    }

    QUrl ClassificationPredictor() const
    {
      return classificationPredictor_;
    }

    void SetClassificationPredictor(const QUrl & classificationPredictor)
    {
      if (classificationPredictor_ == classificationPredictor)
        return;
      classificationPredictor_ = classificationPredictor;
      NotifyClassificationPredictor();
    }


    QUrl source_;
    Cvb::DevicePtr device_;
    Cvb::ImagePtr image_;

    QUrl searchPredictor_;
    std::unique_ptr<Cvb::Polimago::SearchPredictor> search_;

    QUrl classificationPredictor_;
    std::unique_ptr<Cvb::Polimago::ClassificationPredictor> classification_;


    ResultModel& controller_;

  Q_SIGNALS:

    void NotifySource();
    void NotifySearchPredictor();
    void NotifyClassificationPredictor();




};

#include "classification.hpp"

#include <QQmlEngine>
#include <QQmlComponent>

#include <cvb/ui/ui.hpp>
#include <cvb/area_2d.hpp>
#include <cvb/point_2d.hpp>




Classification::Classification(ResultModel& model)
  : QObject()
  , controller_(model)
{
  connect(this, &Classification::NotifySource, this, &Classification::OnSourceChanged);
  connect(this, &Classification::NotifySearchPredictor, this, &Classification::OnSearchPredictorChanged);
  connect(this, &Classification::NotifyClassificationPredictor, this, &Classification::OnClassificationPredictorChnaged);
  // open default example
  device_ = Cvb::DeviceFactory::Open(Cvb::InstallPath() + Cvb::String(CVB_LIT("drivers/CVMock.vin")), Cvb::AcquisitionStack::Vin); // CVMock.vin loads Cookies example emu
  search_ = Cvb::Polimago::SearchPredictor::Load(Cvb::InstallPath() + Cvb::String(CVB_LIT("tutorial/Polimago/Images/Cookies/Cookies.psc")));
  classification_ = Cvb::Polimago::ClassificationPredictor::Load(Cvb::InstallPath() + Cvb::String(CVB_LIT("tutorial/Polimago/Images/Cookies/CookieType.pcc")));
  Snap();
}

void Classification::Snap()
{
  if (!device_)
    return;

  controller_.Update({});
  auto waitResult = device_->Stream()->GetSnapshot();
  if (waitResult.Status != Cvb::WaitStatus::Ok)
    return;

  image_ = waitResult.Image;
  controller_.Refresh(image_);
}

void Classification::Classify()
{
  if(!image_ || !search_ || !classification_)
    return;

  auto redImage = image_->Plane(0).Map();

  const auto gridStep = 0.6;
  const auto threshold = 0.0;
  const auto locality = 1.0;

  std::vector<PolimagoResult> results;

  // 1. find items
  auto positions = search_->GridSearch(*redImage, redImage->Bounds(), gridStep, threshold, locality);
  auto end = std::remove_if(positions.begin(), positions.end(), [](const Cvb::Polimago::SearchResult &position)
  {
    return position.Quality() < .3;
  });

  // 2. classify the items
  std::transform(positions.begin(), end, std::back_inserter(results), [this](const Cvb::Polimago::SearchResult &pos)
  {
    auto result = classification_->Classify(*image_, Cvb::Point2D<int>{static_cast<int>(pos.X()), static_cast<int>(pos.Y())});
    return PolimagoResult(pos, result);
  });

  // 3. update to UI
  controller_.Update(results);
}

void Classification::OnSourceChanged()
{
  image_.reset();
  device_.reset();

  search_.reset();
  classification_.reset();

  auto fileName = source_.toLocalFile();
  auto path = Cvb::UI::QtToCvb(fileName);
  if (fileName.endsWith(".vin"))
  {
    device_ = Cvb::DeviceFactory::Open(path);
    Snap();
  }
  else if (fileName.endsWith(".bmp"))
  {
    image_ = Cvb::Image::Load(path);
    controller_.Update({});
    controller_.Refresh(image_);
  }

}

void Classification::OnSearchPredictorChanged()
{
  search_ = Cvb::Polimago::SearchPredictor::Load(Cvb::UI::QtToCvb(searchPredictor_.toLocalFile()));
}

void Classification::OnClassificationPredictorChnaged()
{
  classification_ = Cvb::Polimago::ClassificationPredictor::Load(Cvb::UI::QtToCvb(classificationPredictor_.toLocalFile()));
}

The polimago result:

#pragma once

#include <iomanip>
#include <mutex>
#include <vector>

#include <QAbstractListModel>
#include <QPoint>

#include <cvb/point_2d.hpp>
#include <cvb/polimago/classification_result.hpp>
#include <cvb/polimago/search_result.hpp>
#include <cvb/ui/ui.hpp>
#include <cvb/ui/image_view_item.hpp>


class PolimagoResult
{
public:
  PolimagoResult(const Cvb::Polimago::SearchResult &pos, const Cvb::Polimago::ClassificationResult &result)
    : position_(static_cast<float>(pos.X()), static_cast<float>(pos.Y()))
    , name_(Cvb::UI::CvbToQt(result.Name()))
    , posQuality_(pos.Quality())
    , colorQuality_(result.Quality())
  {
  }

  QPointF Position() const noexcept { return position_; }

  QString Name() const noexcept { return name_; }

  double PosQuality() const noexcept { return posQuality_; }

  double ColorQuality() const noexcept { return colorQuality_; }

private:
  QPointF position_;
  QString name_;
  double posQuality_;
  double colorQuality_;
};


class ResultModel
  : public QAbstractListModel
{
  Q_OBJECT


  friend class Classification;

  enum ResultRoles
  {
    LineText = Qt::UserRole,
    StartPosition = Qt::UserRole + 1,
    Quality
  };

  public:

    explicit ResultModel(Cvb::UI::ImageController &controller);

    void Update(const std::vector<PolimagoResult> &results)
    {
      results_ = results;
      layoutChanged();
    }

    void Refresh(const Cvb::ImagePtr &image)
    {
      controller_.Refresh(image);
    }

  private:

  

    QHash<int, QByteArray> roleNames() const override;

    int rowCount(const QModelIndex & = QModelIndex()) const override;

    QVariant data(const QModelIndex &index, int role = Qt::DisplayRole) const override;

    

  private:
    Cvb::UI::ImageController &controller_;
    std::vector<PolimagoResult> results_;


  Q_SIGNALS:

};

#include "result_model.hpp"

#include <QRect>


ResultModel::ResultModel(Cvb::UI::ImageController &controller)
  : QAbstractListModel()
  , controller_(controller)
{
}




QHash<int, QByteArray> ResultModel::roleNames() const
{
  QHash<int, QByteArray> names;

  names[LineText] = "lineText";
  names[StartPosition] = "startPosition";
  names[Quality] = "quality";

  return names;
}

int ResultModel::rowCount(const QModelIndex &) const
{
  return static_cast<int>(results_.size());
}


QVariant ResultModel::data(const QModelIndex &index, int role) const
{
  if (!index.isValid())
    return QVariant();

  const auto & result = results_[index.row()];
  
  switch(role)
  {
  default:
    return QVariant();

  case LineText:
    return result.Name();
  case StartPosition:
    return result.Position();
  case Quality:
    return result.PosQuality();
  }
}

The main QML:

import QtQuick 2.3
import CvbQuick 1.0 as CvbQuick
import QtQuick.Controls 1.3
import QtQuick.Layouts 1.2
import QtQuick.Dialogs 1.2

ApplicationWindow
{
    id: rootWin
    visible: true
    property int margin: 11
    width: 1080
    height: 720

    ColumnLayout
    {
        id: mainLayout
        anchors.fill: parent
        anchors.margins: margin

        CvbQuick.ImageView
        {
            id: view
            image: mainImage
            Layout.fillWidth: true
            Layout.fillHeight: true

            // Text result per line
            Repeater
            {
                model : resultModel
                delegate: Marker
                {
                    imageView : view
                    imageX : startPosition.x
                    imageY : startPosition.y
                    posQuality : quality
                    text : lineText

                }
            }
        }

        RowLayout
        {
          Button
          {
            text: "Load Image"
            onClicked: openImageDialog.open()
          }

          Button
          {
            text: "Load Search Predictor"
            onClicked: loadSearchPredictorDialog.open()
          }

          Button
          {
            text: "Load Classification Predictor"
            onClicked: loadClassificationPredictorDialog.open()
          }

          Button
          {
            id: btnSnap
            text: "Snap"
            onClicked: classification.Snap()
          }

          Button
          {
            text: "Classify"
            onClicked: classification.Classify()
          }


        }

        FileDialog
       {
         id: openImageDialog
         title: "Open Image"
         selectExisting: true
          nameFilters: [ "Image source files (*.bmp)", "Common Vision Blox driver files (*.vin)" ]
          onAccepted:
          {
            classification.source = openImageDialog.fileUrl;
            var path = classification.source.toString();
            if (path.endsWith(".bmp"))
              btnSnap.enabled = false
            else
              btnSnap.enabled = true
          }
        }

        FileDialog
       {
         id: loadSearchPredictorDialog
         title: "Load Search Predictor"
         selectExisting: true
          nameFilters: [ "Search classifier (*.psc)" ]
          onAccepted: classification.searchPredictor = loadSearchPredictorDialog.fileUrl;
        }

        FileDialog
       {
         id: loadClassificationPredictorDialog
         title: "Load Classification Predictor"
         selectExisting: true
          nameFilters: [ "Classification predictor (*.pcc)" ]
          onAccepted: classification.classificationPredictor = loadClassificationPredictorDialog.fileUrl;
        }
    }
}

The label:

import QtQuick 2.3
import CvbQuick 1.0 as CvbQuick

// Output result as overlay
CvbQuick.ImageLabel
{
  id: label
  property var text : ""
  property var posQuality : 0.0 
  labelScale : CvbQuick.LabelScale.Off

  Text
  {
    text: label.text + " (Q=" + String(label.posQuality.toFixed(2)) + ")"
    font.pointSize : 10
    font.bold: true
    color : "white"
    style: Text.Outline
    styleColor: "black"  
  }
}