Minos/QmlMinos

This example requires Qt5 >= 5.9 setup for building.

You may build it with Ubuntu 18.04's default Qt5 after installing:

• qt5-default
• qtdeclarative5-dev
• qml-module-qtquick2
• qml-module-qtquick-controls
• qml-module-qtquick-dialogs

// ---------------------------------------------------------------------------
// ---------------------------------------------------------------------------

#include <iostream>

#include <QGuiApplication>
#include <QQmlApplicationEngine>
#include <QQmlContext>
#include <QQuickView>
#include <QIcon>

#include <cvb/ui/image_view_item.hpp>
#include <cvb/minos/classifier.hpp>
#include <cvb/minos/search_result.hpp>

#include "minos_search.hpp"
#include "minos_result_model.hpp"

int main(int argc, char* argv[])
{
  try
  {
    QGuiApplication app(argc, argv);
    app.setWindowIcon(QIcon(":/qttutorial.png"));

    // open device
    auto device = Cvb::DeviceFactory::Open(Cvb::InstallPath() + Cvb::String(CVB_LIT("drivers/CVMock.vin")), Cvb::AcquisitionStack::Vin);

    // setup QML interface objects
    Cvb::UI::ImageController imageController;
    MinosResultModel minosResultModel(imageController);

    // main search object
    MinosSearch minosSearch(device->Stream(), minosResultModel);
    
  
    // register QML components for an image display
    Cvb::UI::ImageViewItem::Register();
    Cvb::UI::ImageLabelItem::Register();


    QQmlApplicationEngine engine;
    auto context = engine.rootContext();
    // create a controller object to communicate with QML
    context->setContextProperty("mainImage", &imageController);
    // create a Minos result model to communicate with QML for overlays
    context->setContextProperty("minosResultModel", &minosResultModel);
    // create a Minos search object to communicate with QML (grab, snap)
    context->setContextProperty("minosSearch", &minosSearch);

    // load main QML file
    engine.load(QUrl::fromLocalFile("../main.qml"));

    // do a first snap at startup
    minosSearch.Snap();

    return app.exec();
  }
  catch (const std::exception& error)
  {
    std::cout << error.what() << std::endl;
  }
}

The search class:

#include <QObject>
#include <QVariant>

#include <cvb/device_factory.hpp>
#include <cvb/async/single_stream_handler.hpp>
#include <cvb/minos/classifier.hpp>
#include <cvb/ui/image_view_item.hpp>
#include <cvb/utilities/stop_watch.hpp>

#include "minos_result_model.hpp"


class MinosSearch final: 
  public QObject,
  public Cvb::Async::SingleStreamHandler
{
  Q_OBJECT

  public:

    MinosSearch(const Cvb::StreamPtr & stream, MinosResultModel& model);

    ~MinosSearch();

    std::vector<MinosResultItem> Search(const Cvb::ImagePlane & plane);

  public Q_SLOTS:

    void StartGrab();
    void StopGrab();
    void Snap();


  private:

    void SearchLine(const Cvb::ImagePlane & plane, Cvb::Area2D aoi, std::vector<MinosResultItem> & resultList);

    void HandleAsyncStream(const Cvb::StreamPtr & stream) override;

    Cvb::StreamPtr stream_;
    Cvb::Minos::Classifier classifier_;
    MinosResultModel& model_;

    Cvb::Utilities::StopWatch stopWatch_;

};

#include "minos_search.hpp"

#include <QQmlEngine>
#include <QQmlComponent>

#include <cvb/ui/ui.hpp>
#include <cvb/area_2d.hpp>
#include <cvb/point_2d.hpp>


// Global params
const double FIRST_AREA_RADIUS = 8.0;
const double LINE_RADIUS = 4;
const double WORD_HEIGHT = 4;
const double WORD_WIDTH = 4;
const double LINE_STEP_RADIUS = 8;

const Cvb::Point2D<double> LINE_STEP = Cvb::Point2D<double>(0.0, 16.0);
const Cvb::Area2D OCR_AOI = Cvb::Area2D(-3.0, -3.0, 3.0, -3.0, -3.0, 3.0);

MinosSearch::MinosSearch(const Cvb::StreamPtr & stream, MinosResultModel& model)
  : QObject()
  , SingleStreamHandler(stream)
  , stream_(stream)
  , classifier_(Cvb::InstallPath() + Cvb::String(CVB_LIT("tutorial/Minos/Images/OCR/Training Set/Numbers.clf")))
  , model_(model)
  , stopWatch_(Cvb::StopWatchMode::MultiCPU)
{
}

MinosSearch::~MinosSearch()
{
  TryFinish();
}


void MinosSearch::StartGrab()
{
  Run();
}

void MinosSearch::StopGrab()
{
  Finish();
}

void MinosSearch::Snap()
{
  auto waitResult = stream_->GetTimedSnapshot(std::chrono::seconds(1));
  if (waitResult.Status != Cvb::WaitStatus::Ok)
    return;

  // Minos search in snap
  MinosResultData result;

  stopWatch_.Start();
  result.List = Search(waitResult.Image->Plane(0));
  model_.SetProcessingTime(stopWatch_.TimeSpan().count());

  result.Image = waitResult.Image;
  model_.PushData(result); 
  model_.Refresh();
}

// The Minos search
std::vector<MinosResultItem> MinosSearch::Search(const Cvb::ImagePlane & plane)
{
 
  auto width = static_cast<double>(plane.Parent().Width());
  auto height = static_cast<double>(plane.Parent().Height());

  // STEP 1 finding the top most line
  // create an area centered in horizontal direction
  // size of the area is FIRST_AREA_WIDTH = ImageHeight
  // the direction of the area is set to the bottom
  // create an area from these values
  Cvb::Area2D searchAoi(Cvb::Point2D<double>(width / 2 - FIRST_AREA_RADIUS, 0.0),
    Cvb::Point2D<double>(width / 2 + FIRST_AREA_RADIUS, 0.0),
    Cvb::Point2D<double>(width / 2 - FIRST_AREA_RADIUS, height - 1.0));

  // search for the first object
  auto searchResult = classifier_.Search(plane, Cvb::Minos::SearchMode::FindFirst, searchAoi);

  std::vector<MinosResultItem> resultList;
  if (searchResult.Quality() < 0.1)
    return resultList;

  // now we found the first line of the text

  // STEP 2 finding the beginning of a line
  // try to find the first char in the line
  // create an area centered around the first line
  // beginning at the left side of the image
  // size of the area is LINE_HEIGHT * dXPos
  // the direction of the area is set to the right
  // create an area from these values
  auto position = searchResult.Position();
  Cvb::Area2D lineAoi(Cvb::Point2D<double>(0.0, position.Y() - LINE_RADIUS),
    Cvb::Point2D<double>(0.0, position.Y() + LINE_RADIUS),
    Cvb::Point2D<double>(position.X(), position.Y() - LINE_RADIUS));



  // do a line by line search 
  SearchLine(plane, lineAoi, resultList);

  return resultList;
}

// Minos search per line
void MinosSearch::SearchLine(const Cvb::ImagePlane & plane, Cvb::Area2D lineAoi, std::vector<MinosResultItem> & resultList)
{
  // search for the first object
  auto searchResult = classifier_.Search(plane, Cvb::Minos::SearchMode::FindFirst, lineAoi);

  if (searchResult.Quality() == 0.0)
    return;

  // now we found the first char in the first line

  // STEP 3 read the string
  // save the x and y position of the line
  // try to read the first word
  // create an area left aligned the first char
  // size of WORD_WIDTH * WORD_HEIGHT
  // create an area from these values
  auto lineStart = searchResult.Position();
  Cvb::Area2D readAoi(Cvb::Point2D<double>(lineStart.X(), lineStart.Y() - WORD_HEIGHT / 2),
    Cvb::Point2D<double>(lineStart.X() + WORD_WIDTH / 2,lineStart.Y() - WORD_HEIGHT / 2),
    Cvb::Point2D<double>(lineStart.X(), lineStart.Y() + WORD_HEIGHT / 2));

  // read the word
  auto readResult = classifier_.Read(plane, readAoi, OCR_AOI);

  // save the result
  Cvb::StringStream stream;
  for (const auto c : readResult)
    stream << c.Name();
  MinosResultItem resultItem = { lineStart, stream.str() };
  resultList.push_back(resultItem);


  lineStart += LINE_STEP;


  // STEP 4 jump to the next line
  // try to read the first word
  // create an area left aligned the first char
  // size of WORD_WIDTH * WORD_HEIGHT
  // create an area from these values
  Cvb::Area2D nextLineAoi(Cvb::Point2D<double>(lineStart.X() - LINE_STEP_RADIUS, lineStart.Y() - LINE_STEP_RADIUS),
    Cvb::Point2D<double>(lineStart.X() + LINE_STEP_RADIUS, lineStart.Y() - LINE_STEP_RADIUS),
    Cvb::Point2D<double>(lineStart.X() - LINE_STEP_RADIUS, lineStart.Y() + LINE_STEP_RADIUS));

  // next line (recursion should - no to deep)
  SearchLine(plane, nextLineAoi, resultList);
}

void MinosSearch::HandleAsyncStream(const Cvb::StreamPtr & stream)
{
  auto waitResult = stream->WaitFor<Cvb::RingBufferImage>(std::chrono::seconds(1));
  if (waitResult.Status == Cvb::WaitStatus::Ok)
  {


    // Minos search in grab
    MinosResultData result;

    stopWatch_.Start();
    result.List = Search(waitResult.Image->Plane(0));
    model_.SetProcessingTime(stopWatch_.TimeSpan().count());

    result.Image = waitResult.Image;
    model_.PushData(result);

    // Tell the UI thread to redraw and present the results
    QMetaObject::invokeMethod(&model_, "Refresh", Qt::QueuedConnection);
  }
}

The minos search result:

#pragma once

#include <deque>
#include <mutex>

#include <QAbstractListModel>

#include <cvb/minos/search_result.hpp>
#include <cvb/ui/image_view_item.hpp>


struct MinosResultItem
{
  Cvb::Point2D<double> Position;
  Cvb::String Text;
};

struct MinosResultData
{
  std::vector<MinosResultItem> List;
  Cvb::ImagePtr Image;
};



class MinosResultModel
  : public QAbstractListModel
{
  Q_OBJECT

  Q_PROPERTY(double processingTime READ ProcessingTime NOTIFY NotifyProcessingTime);
  Q_PROPERTY(QString searchResultText READ SearchResultText NOTIFY NotifySearchResultText);

  friend class MinosSearch;

  enum MinosResultRoles
  {
    LineText = Qt::UserRole,
    StartPosition = Qt::UserRole + 1
  };

  public:

    explicit MinosResultModel(Cvb::UI::ImageController& imageController);

    

  private Q_SLOTS:

    void Refresh();

  private:

    QString SearchResultText() const;

    double ProcessingTime() const noexcept;

    void SetProcessingTime(double processingTime);

    void PushData(const MinosResultData & data);

    QHash<int, QByteArray> roleNames() const override;

    int rowCount(const QModelIndex & = QModelIndex()) const override;

    QVariant data(const QModelIndex &index, int role = Qt::DisplayRole) const override;

    void SetSearchResultText(const QString searchResultText);

  private:

    Cvb::UI::ImageController& imageController_;

    std::mutex resultMutex_;
    std::deque<MinosResultData> resultQue_;
    MinosResultData currentResult_;

    QString searchResultText_;

    double processingTime_ = std::numeric_limits<double>::quiet_NaN();

  Q_SIGNALS:

    void NotifyProcessingTime();
    void NotifySearchResultText();
};

#include "minos_result_model.hpp"

#include <QRect>

#include <cvb/ui/ui.hpp>

MinosResultModel::MinosResultModel(Cvb::UI::ImageController& imageController)
  : QAbstractListModel()
  , imageController_(imageController)
{
}




QHash<int, QByteArray> MinosResultModel::roleNames() const
{
  QHash<int, QByteArray> names;

  names[LineText] = "lineText";
  names[StartPosition] = "startPosition";

  return names;
}

int MinosResultModel::rowCount(const QModelIndex &) const
{
  return static_cast<int>(currentResult_.List.size());
}


QVariant MinosResultModel::data(const QModelIndex &index, int role) const
{
  if (!index.isValid())
    return QVariant();

  const auto & result = currentResult_.List[index.row()];
  switch (role)
  {

  default:
    return QVariant();

  case StartPosition:
    return QVariant(Cvb::UI::CvbToQt(result.Position));

  case LineText:
    return QVariant(Cvb::UI::CvbToQt(result.Text));
  }
}

QString MinosResultModel::SearchResultText() const
{
  return searchResultText_;
}

double MinosResultModel::ProcessingTime() const noexcept
{
  return processingTime_;
}

void MinosResultModel::SetProcessingTime(double processingTime)
{
  if (processingTime_ == processingTime)
    return;
  processingTime_ = processingTime;
  NotifyProcessingTime();
}

void MinosResultModel::PushData(const MinosResultData & data)
{
  std::unique_lock<std::mutex> guard(resultMutex_);
  resultQue_.push_back(data);
}

void MinosResultModel::Refresh()
{
  {
    std::unique_lock<std::mutex> guard(resultMutex_);
    if (!resultQue_.size())
      return;
    currentResult_ = resultQue_.back();
    resultQue_.clear();
  }
  layoutChanged();
  Cvb::StringStream stream;
  for (const auto & result : currentResult_.List)
    stream << result.Text << CVB_LIT("\n");
  SetSearchResultText(Cvb::UI::CvbToQt(stream.str()));
  imageController_.Refresh(currentResult_.Image);
}

void MinosResultModel::SetSearchResultText(const QString searchResultText)
{
  if (searchResultText_ == searchResultText)
    return;
  searchResultText_ = searchResultText;
  NotifySearchResultText();
}

The main QML:

import QtQuick 2.3
import CvbQuick 1.0 as CvbQuick
import QtQuick.Controls 1.2
import QtQuick.Layouts 1.3

ApplicationWindow
{
    id: rootWin
    visible: true
    property int margin: 11
    width: 1080
    height: 720

    ColumnLayout
    {
        id: mainLayout
        anchors.fill: parent
        anchors.margins: margin

        CvbQuick.ImageView
        {
            id: view
            image: mainImage
            Layout.fillWidth: true
            Layout.fillHeight: true

            // Text result per line
            Repeater
            {
                model : minosResultModel
                delegate: TextLabel
                {
                    imageView : view
                    imageX : startPosition.x
                    imageY : startPosition.y
                    text : lineText

                }
            }
        }

        RowLayout
        {

            TextArea
            {
              id: txtArea
              text: minosResultModel.searchResultText
              readOnly: true
              wrapMode: TextEdit.WrapAnywhere
              Layout.fillWidth: true
            }

            ColumnLayout
            {
              Layout.preferredWidth: 200

              RowLayout
              {

                Layout.alignment: Qt.AlignCenter

                Button
                {
                  text: "Snap"
                  onClicked: minosSearch.Snap()
                  enabled: !cbGrab.checked
                }

                CheckBox
                {
                  id: cbGrab
                  text: "Grab"
                  checked: false
                  onClicked:
                  {
                    if (checkedState === Qt.Checked)
                      return minosSearch.StartGrab()
                    else
                      return minosSearch.StopGrab()
                  }
                }
              }

              Label
              {
                id: txtProcessingTime
                text: "Processing time: " + String(minosResultModel.processingTime) + " ms"
                font.bold: true
                Layout.alignment: Qt.AlignCenter
              }

            }

        }
    }
}

The label overlay:

import QtQuick 2.3
import CvbQuick 1.0 as CvbQuick

// Output result as overlay
CvbQuick.ImageLabel
{
  id: label
  property var text : ""
  labelScale : CvbQuick.LabelScale.On

  Text
  {
    text: label.text
    color: "red"
    font.pointSize : 4
  }

}