Es++ Sample: ShapeMatcher


/******************************************************************************
 *
 * Copyright (c) 2017 Antillia.com TOSHIYUKI ARAI. ALL RIGHTS RESERVED.
 *
 * Redistribution and use in binarized and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of binarized code must retain the above copyright
 *    notice, this list of conditions, and the following disclaimer.
 *  
 * 2. The name of the author may not be used to endorse or promote products
 *    derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 
 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 
 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 
 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 *
 *  ShapeMatcher.cpp
 *
 *****************************************************************************/

#include <es++/Pair.h>
#include <es++/gtkmm-3.0/Application.h>
#include <es++/gtkmm-3.0/FileOpenDialog.h>
#include <es++/gtkmm-3.0/Label.h>
#include <es++/gtkmm-3.0/LabeledComboBox.h>
#include <es++/gtkmm-3.0/LabeledTrackBar.h>
#include <es++/gtkmm-3.0/PushButton.h>
#include <es++/opencv-4.0/OpenCVMainView.h>
#include <es++/opencv-4.0/OpenCVScaleComboBox.h>
#include <es++/opencv-4.0/OpenCVScrolledImageView.h>

using namespace Gtk;

namespace Es {

class MainView :public Es::OpenCVMainView {
private:
  /////////////////////////////////////////////////////
  //Inner classes start.
  class BinarizedImageView :public Es::OpenCVScrolledImageView {
  private:
    cv::Mat gray_image;
    cv::Mat binarized_image;

  public:
    BinarizedImageView()
    {
    }

    void loadImage(const std::string& filename, 
        int imageLoadingFlag= CV_LOAD_IMAGE_COLOR,
                int scaling_ratio=100)
    {
      OpenCVScrolledImageView::loadImage(filename, imageLoadingFlag,
                                scaling_ratio);
 
      cv::Mat& original_image = getOriginalImage();
      cv::cvtColor(original_image, gray_image, COLOR_BGR2GRAY);

      binarized_image  = gray_image.clone();
      scaleImage(binarized_image, scaling_ratio);
    }    

    cv::Mat& getBinarizedImage()
    {
      return binarized_image;
    }

    void binarize(int threshold_type, 
               int threshold_value, int scaling_ratio)
    {
      try {
        cv::threshold( gray_image, binarized_image, threshold_value, 
             THRESHOLD_VALUE_MAX, threshold_type );

      } catch (cv::Exception& ex) {
        //Sometimes we get an exception; I don't know the reason why it happens.
      }      

      scaleImage(binarized_image, scaling_ratio);
    }

    void rescale(int scaling_ratio)
    {
      OpenCVScrolledImageView::scaleImage(binarized_image, scaling_ratio);
    }

  };

  class MatchedImageView :public Es::OpenCVScrolledImageView {
  private:
    cv::Mat matched_image; 

  public:
    MatchedImageView()
    {
    }

    void loadImage(const std::string& filename,
                int imageLoadingFlag= CV_LOAD_IMAGE_COLOR,
                int scaling_ratio=100)
    {
      OpenCVScrolledImageView::loadImage(filename, imageLoadingFlag,
                                scaling_ratio);

      cv::Mat& original_image = getOriginalImage();

      matched_image  = original_image.clone();

      scaleImage(matched_image, scaling_ratio);
    }

    void rescale(int scaling_ratio)
    {
      OpenCVScrolledImageView::scaleImage(matched_image, scaling_ratio);
    }

    void setMatched(cv::Mat& mat, int scaling_ratio)
    {
      matched_image = mat.clone();
      scaleImage(matched_image, scaling_ratio);
    }
  };

  //Inner classes end.
  /////////////////////////////////////////////////////
  //
private:
  static const int        IMAGES_COUNT = 2;
  static const int        FIRST  = 0;
  static const int        SECOND = 1;
  std::string             filenames[IMAGES_COUNT];
  
  Es::Label               filepath;

  Es::HorizontalLayout   horiz_layout;
    Es::VerticalLayout     view_layout;
      Es::HorizontalLayout   binarized_view_layout;
        BinarizedImageView        binarized_images[IMAGES_COUNT];
      MatchedImageView       matched_image;

    Es::VerticalLayout      control_pane;
      Es::OpenCVScaleComboBox combobox;

      Es::LabeledComboBox     threshold_type_combobox;

      Es::LabeledTrackBar     threshold_value_trackbar;

      Es::LabeledTrackBar     minimum_size_trackbar;

      Es::LabeledTrackBar     maximum_size_trackbar;

      Es::PushButton          match_button;

  int                     threshold_type_index;  

  int                     threshold_value;

  int                     minimum_size;

  int                     maximum_size;

  static const int        FILEPATH_HEIGHT   = 30; 

  static const int        CONTROLPANE_WIDTH = 200;

  int                     scaling_ratio;

  static const int        loading_flag = IMREAD_COLOR; 

  Es::FileOpenDialog      file_dialog;

  static const int THRESHOLD_VALUE_MAX = 255;

public:
  //////////////////////////////////////////////
  //Constructor
  //
  MainView(Es::Application& applet,
          std::string& title,
          Es::Args& args)
  :OpenCVMainView(applet, title, 
                     //We don't use the defaultFilePulldown menu.
                     args.set(XmNuseDefaultFilePulldown, false) )
  ,scaling_ratio(100)
  ,binarized_view_layout(0)  //SPACING
  ,file_dialog(*this, Es::FileOpenDialog::IMAGE_FILES)
  {
   //Define File pulldown menu with multiple [Open File] pulldown menu items.
    Es::SigcMenuCallback file_menu_callbacks[] = {
      {"New",        sigc::mem_fun(*this, &MainView::file_new)},
      {"Open File1", sigc::mem_fun(*this, &MainView::file_open1)},
      {"Open File2", sigc::mem_fun(*this, &MainView::file_open2)},
      {"Save",       sigc::mem_fun(*this, &MainView::file_save)},
      {"Save_As",    sigc::mem_fun(*this, &MainView::file_save_as)},
      {"Quit",       sigc::mem_fun(*this, &MainView::file_quit)},
    };

    file_pulldown_append( file_menu_callbacks,
                         CountOf(file_menu_callbacks));

    int w = (int)args.get(XmNwidth);
    int h = (int)args.get(XmNheight);
    int ww = (w - CONTROLPANE_WIDTH)/2;
    int hh = h/2;

    int ratio = (int)args.get(XmNscalingRatio);
    scaling_ratio = OpenCVImageView::validateScale(ratio);

    Es::MainLayout& main_layout = get_main_layout();

    filepath.set_size_request(w, FILEPATH_HEIGHT);

    filepath.set_alignment(Gtk::ALIGN_START);

    main_layout.pack_start(filepath, FALSE,FALSE, 0);
    main_layout.pack_start(horiz_layout, FALSE, FALSE, 0);

    binarized_images[FIRST].set_size_request(ww, h/2);
    binarized_images[SECOND].set_size_request(ww, h/2);
    matched_image.set_size_request(w, h/2);

    control_pane.set_spacing(10);
    control_pane.set_size_request(CONTROLPANE_WIDTH, h);

    main_layout.pack_start(filepath, FALSE, FALSE, 0);
      main_layout.pack_start(horiz_layout);
        horiz_layout.pack_start(view_layout);
          view_layout.pack_start(binarized_view_layout);
            binarized_view_layout.pack_start(binarized_images[FIRST]);
            binarized_view_layout.pack_start(binarized_images[SECOND]);
          view_layout.pack_start(matched_image);
           
        horiz_layout.pack_start(control_pane);
        control_pane.pack_start(combobox, Gtk::PACK_SHRINK );
        match_button.set_label("Match");

        control_pane.pack_start(threshold_type_combobox, Gtk::PACK_SHRINK);
        control_pane.pack_start(threshold_value_trackbar, Gtk::PACK_SHRINK);
        control_pane.pack_start(minimum_size_trackbar, Gtk::PACK_SHRINK);
        control_pane.pack_start(maximum_size_trackbar, Gtk::PACK_SHRINK);
        control_pane.pack_start(match_button, Gtk::PACK_SHRINK);

    combobox.set_selection(scaling_ratio);
    combobox.set_changed_callback(sigc::mem_fun(*this, &MainView::scale_changed) );

    match_button.set_label("Match");
    match_button.set_clicked_callback(sigc::mem_fun(*this, 
                     &MainView::match_button_clicked) );

    //Create a threshold type comboBox in controlPane.
    const char* types[] = {
        "Binary",
        "Binary Inverted",
        "Threshold Truncated",
        "Threshold to Zero",
        "Threshold to Zero Inverted",
        //"Mask", 
        "Otsu", 
      };

    threshold_type_index  = 0;
    threshold_type_combobox.set_label("ThresholdType");

    threshold_type_combobox.append_items(types, CountOf(types));
    threshold_type_combobox.set_active_text(
                       types[threshold_type_index]);//Binary 

    threshold_type_combobox.set_changed_callback(sigc::mem_fun(*this, 
                     &MainView::threshold_type_changed) );

    threshold_value_trackbar.set_label("ThresholdValue:[0,255]");
    threshold_value = 60;
    threshold_value_trackbar.configure_adjuster(threshold_value,
                      0, 255);
    threshold_value_trackbar.set_value_changed_callback(
            sigc::mem_fun(*this, &MainView::threshold_value_changed) );


    minimum_size = 11;
    minimum_size_trackbar.set_label("MatchMinSize:[10,100]"); 
    minimum_size_trackbar.configure_adjuster(minimum_size,
                      0, 100);
    minimum_size_trackbar.set_value_changed_callback(
            sigc::mem_fun(*this, &MainView::minimum_size_changed) );

    maximum_size = 101;
      
    maximum_size_trackbar.set_label("MatchMaxSize:[100,400]"); 
    maximum_size_trackbar.configure_adjuster(maximum_size,
                      100, 400);
    maximum_size_trackbar.set_value_changed_callback(
            sigc::mem_fun(*this, &MainView::maximum_size_changed) );

    filenames[FIRST]  = "../../images/CatImage.png";
    filenames[SECOND] = "../../images/CatFace.png";

    for (int i = 0; i<IMAGES_COUNT; i++) { 
      binarized_images[i].loadImage(filenames[i], loading_flag, scaling_ratio);
      binarized_images[i].binarize(
                                getThresholdType(threshold_type_index),
                                threshold_value,
                                scaling_ratio );
    }

    //binarized_images[SECOND].loadImage(filenames[SECOND], loading_flag, scaling_ratio);

    matched_image.loadImage(filenames[FIRST], loading_flag, scaling_ratio);
/*
    binarized_images[FIRST].binarize(
                                getThresholdType(threshold_type_index),
                                threshold_value,
                                scaling_ratio );

    binarized_images[SECOND].binarize(
                                getThresholdType(threshold_type_index),
                                threshold_value,
                                scaling_ratio );
*/

    update_filepath();
    show_all();
  }

  void scale_changed()
  {
    scaling_ratio = combobox.get_selection();
    printf("scale_changed %d\n", scaling_ratio);
    binarized_images[FIRST].rescale(scaling_ratio);
    binarized_images[SECOND].rescale(scaling_ratio);
    matched_image.rescale(scaling_ratio);
  }

  void update_filepath()
  {
    std::string space = "    ";
    filepath.set_label(filenames[FIRST] + space +  filenames[SECOND]);
  }

  static int getThresholdType(int index) 
  {
    int n = 0;
    //We don't include the THRESH_OTSU type
    Pair<int, int> types[] = {
      {n++, THRESH_BINARY     },
      {n++, THRESH_BINARY_INV },
      {n++, THRESH_TRUNC      },
      {n++, THRESH_TOZERO     },
      {n++, THRESH_TOZERO_INV },
      {n++, THRESH_OTSU       }, 
    };
    int type = THRESH_BINARY; 
    if (index >= 0 && index <n) {
      type = types[index].second;
    } 
    return type;
  }

  void threshold_type_changed()
  {
    int prev_threshold_type = threshold_type_index;
   
    threshold_type_index = threshold_type_combobox.get_active_row_number();
    printf("threshold_type_index %d\n", threshold_type_index);
    
    if (threshold_type_index != prev_threshold_type) {
      printf("Detector updated call match\n");
      match_button_clicked();
    }
  }

  void threshold_value_changed()
  {
    threshold_value = threshold_value_trackbar.get_value();
    match_button_clicked();
  }
  
  void minimum_size_changed()
  {
    minimum_size = minimum_size_trackbar.get_value();
    match_button_clicked();
  }

  void maximum_size_changed()
  {
    maximum_size = maximum_size_trackbar.get_value();
    match_button_clicked();
  }

  //Matching operation. 
  void match_button_clicked()
  {
    //Detect connectedCommponentsWithStats on destImage 
    Mat labels, stats, centroids; 
    cv::Mat& originalImage = binarized_images[FIRST ].getOriginalImage();
    cv::Mat& srcBin = binarized_images[FIRST ].getBinarizedImage();
    cv::Mat& tmpBin = binarized_images[SECOND].getBinarizedImage();

    cv::connectedComponentsWithStats(srcBin,
            labels, stats, centroids);
     
    int tw = tmpBin.rows;
    int th = tmpBin.cols;
       
    cv::Mat destImage = originalImage.clone();
 
    const double MATCHING_THRESHOLD = 0.005;

    cv::Rect minimum(0, 0, 0, 0);
    double MIN_SIMILARITY = 1.0;
    bool found = false;
 
    for(int i=0; i < stats.rows; i++)  { 
      int x = stats.at<int>(Point(0, i)); 
      int y = stats.at<int>(Point(1, i)); 
      int w = stats.at<int>(Point(2, i)); 
      int h = stats.at<int>(Point(3, i)); 
      cv::Rect rect(x, y, w, h);

      cv::Mat rectOfInterest = srcBin(rect); 
      double similarity = cv::matchShapes(tmpBin, 
               rectOfInterest, CV_CONTOURS_MATCH_I1, 0);

      if ( (w >= minimum_size || h >= minimum_size ) && 
           (w <= maximum_size || h <= maximum_size )) { 
        if (similarity <= MIN_SIMILARITY) {
          MIN_SIMILARITY = similarity;       
          minimum = rect;
            printf("matching similarity=%.04f  x=%d y=%d w=%d h=%d\n", 
             similarity, x, y, w, h);
          found = true;
        }
      }
    } 

    if (found) {
      printf("found \n");
      cv::rectangle(destImage, minimum, CV_RGB(255, 0, 0), 3);
    }

    matched_image.setMatched(destImage, scaling_ratio); 

  }

  void file_open1()
  {
    int rc = file_dialog.popup();

    if (rc == Gtk::RESPONSE_OK) {
      filenames[FIRST] = file_dialog.get_filename();
      update_filepath();
      binarized_images[FIRST].loadImage(filenames[FIRST], loading_flag, scaling_ratio);
      binarized_images[FIRST].binarize(
                                getThresholdType(threshold_type_index),
                                threshold_value,
                                scaling_ratio );
    }
  }

  void file_open2()
  {
    int rc = file_dialog.popup();

    if (rc == Gtk::RESPONSE_OK) {
      filenames[SECOND] = file_dialog.get_filename();
      update_filepath();
      binarized_images[SECOND].loadImage(filenames[SECOND], loading_flag, scaling_ratio);
      binarized_images[SECOND].binarize(
                                getThresholdType(threshold_type_index),
                                threshold_value,
                                scaling_ratio );
    }
  }
};
}


int main(int argc, char** argv)
{
  Es::Environment env;
  try {
    Es::Application applet(argc, argv);

    std::string title(argv[0]);

    Es::Args args;
    args.set(XmNx, 20);
    args.set(XmNy, 20);
    args.set(XmNwidth, 900);
    args.set(XmNheight,460);
    args.set(XmNscalingRatio,60);
    Es::MainView mainv(applet, title, args);

    mainv.show();

    applet.run(mainv);

  } catch (Es::Exception& ex) {
    caught(ex);
  } catch (...) {
    printf("Get exception \n");
  }
  return 0;
}