Unverified Commit 5509985f authored by Jeff Niu's avatar Jeff Niu Committed by GitHub

Merge pull request #71 from Mogball/jeff/tracker_goturn

Goturn tracker
parents f48c9b11 0c70bde3
......@@ -245,3 +245,7 @@ FakesAssemblies/
# CLion proj files
.idea/
*.swp
# GOTURN tracker models
goturn.prototxt
goturn.caffemodel
......@@ -16,7 +16,7 @@ before_install:
install:
- sudo apt-get install qt5-default qtmultimedia5-dev -y -qq
- sudo apt-get install libfontconfig1 mesa-common-dev libglu1-mesa-dev libudev-dev libxi6 libsm6 libxrender1 libegl1-mesa -y -qq
- sudo apt-get install libfontconfig1 mesa-common-dev libglu1-mesa-dev libudev-dev libxi6 libsm6 libxrender1 libegl1-mesa libxine2 -y -qq
script:
- mkdir build
......
set(PYTHON_SCRIPT_DIR python_scripts)
macro(add_python_target target)
foreach (file ${target})
get_filename_component(file_name ${file} NAME)
configure_file(${file} ${PYTHON_SCRIPT_DIR}/${file_name} COPYONLY)
endforeach ()
endmacro()
set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/cmake)
include(Util)
include(Python)
if (APPLE)
# Apple hard sets these variables for system Python
unset(PYTHON_LIBRARY CACHE)
unset(PYTHON_INCLUDE_DIR CACHE)
unset(PYTHON_LIBRARIES CACHE)
endif()
endif ()
cmake_minimum_required(VERSION 3.1.0)
......@@ -31,7 +27,7 @@ find_package(Threads REQUIRED)
if (Qt5_POSITION_INDEPENDENT_CODE)
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
endif()
endif ()
## Third Party files
set(THIRD_PARTY_COMMON
......@@ -46,17 +42,17 @@ if (WIN32)
set(THIRD_PARTY
third-party/qextserialenumerator_win.cpp
third-party/qextserialport_win.cpp)
endif()
endif ()
if (APPLE)
set(THIRD_PARTY
third-party/qextserialenumerator_unix.cpp
third-party/qextserialport_unix.cpp)
endif()
endif ()
if (UNIX)
set(THIRD_PARTY
third-party/qextserialenumerator_unix.cpp
third-party/qextserialport_unix.cpp)
endif()
endif ()
file(GLOB_RECURSE SOURCE_FILES
${PROJECT_SOURCE_DIR}/code/*.cpp
......@@ -72,10 +68,12 @@ file(GLOB_RECURSE FORM_FILES
add_python_target(code/scripts/__init__.py)
add_python_target(code/scripts/robot.py)
# Include library directories
include_directories(${Qt5Widgets_INCLUDE_DIRS})
include_directories(${PYTHON_INCLUDE_DIRS})
include_directories(third-party)
# AutoMOC and AutoUIC for Qt compiler
set(CMAKE_INCLUDE_CURRENT_DIR ON)
set(CMAKE_AUTOMOC ON)
set(CMAKE_AUTOUIC ON)
......@@ -83,18 +81,38 @@ set(CMAKE_AUTOUIC ON)
get_target_property(QtCore_location Qt5::Core LOCATION)
add_executable(minotaur-cpp ${SOURCE_FILES} ${HEADER_FILES} ${FORM_FILES} ${THIRD_PARTY} ${THIRD_PARTY_COMMON})
target_compile_definitions(minotaur-cpp PRIVATE NDEBUG)
# Uncomment this line to compile in debug mode
option(NO_DEBUG "Build minotaur without debugging output" OFF)
if (NO_DEBUG)
log("Building without debugging")
target_compile_definitions(minotaur-cpp PRIVATE QT_NO_DEBUG)
target_compile_definitions(minotaur-cpp PRIVATE NDEBUG)
else ()
log("Building with debugging on")
endif ()
# Attempt to find GOTURN files
include(Goturn)
if (${GOTURN_FILES_FOUND})
target_compile_definitions(minotaur-cpp PRIVATE GOTURN_FOUND)
endif ()
find_package(OpenCV REQUIRED)
# Find and link OpenCV 3 package
find_package(OpenCV 3 REQUIRED)
include_directories(${OpenCV_INCLUDE_DIRS})
target_link_libraries(minotaur-cpp ${OpenCV_LIBS})
# Link Qt, Python, and Thread libraries
target_link_libraries(minotaur-cpp Qt5::Widgets Qt5::Core Qt5::Gui Qt5::Multimedia Qt5::MultimediaWidgets)
target_link_libraries(minotaur-cpp ${PYTHON_LIBRARIES})
target_link_libraries(minotaur-cpp "${CMAKE_THREAD_LIBS_INIT}")
# Include current directory to find Qt MOC files
target_include_directories(minotaur-cpp PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
# Add definition for location of python scripts
target_compile_definitions(minotaur-cpp PRIVATE PYTHON_SCRIPT_DIR="${PYTHON_SCRIPT_DIR}")
# Windows requires separate libraries
if (WIN32)
target_link_libraries(minotaur-cpp advapi32 user32 setupapi)
endif ()
set(GOTURN_PROTOTXT ${CMAKE_CURRENT_SOURCE_DIR}/goturn.prototxt)
set(GOTURN_CAFFEMODEL ${CMAKE_CURRENT_SOURCE_DIR}/goturn.caffemodel)
log("Looking for GOTURN files under ${CMAKE_CURRENT_SOURCE_DIR}")
if (EXISTS ${GOTURN_PROTOTXT} AND EXISTS ${GOTURN_CAFFEMODEL})
log("GOTURN files found")
set(GOTURN_FILES_FOUND TRUE)
configure_file(${GOTURN_PROTOTXT} goturn.prototxt COPYONLY)
configure_file(${GOTURN_CAFFEMODEL} goturn.caffemodel COPYONLY)
else ()
log("GOTURN not found")
set(GOTURN_FILES_FOUND FALSE)
endif ()
set(PYTHON_SCRIPT_DIR python_scripts)
macro(add_python_target target)
foreach (file ${target})
get_filename_component(file_name ${file} NAME)
configure_file(${file} ${PYTHON_SCRIPT_DIR}/${file_name} COPYONLY)
endforeach ()
endmacro()
function(log MESSAGE)
message(STATUS ${MESSAGE})
endfunction()
function(error MESSAGE)
message(FATAL_ERROR ${MESSAGE})
endfunction()
function(warning MESSAGE)
message(WARNING ${MESSAGE})
endfunction()
......@@ -6,8 +6,6 @@
#include <QAction>
#include <QDir>
#include "../video/squares.h"
Capture::Capture(QObject *parent)
: QObject(parent) {}
......@@ -66,11 +64,6 @@ void Converter::matDelete(void *mat) {
}
void Converter::queue(const cv::Mat &frame) {
#ifndef NDEBUG
if (m_frame.empty()) {
qDebug() << "OpenCV Image Converter dropped a frame";
}
#endif
m_frame = frame;
if (!m_timer.isActive()) {
m_timer.start(0, this);
......@@ -83,7 +76,7 @@ void Converter::process(cv::Mat frame) {
(m_display->height() - 20) / (double) frame.size().height
);
if (m_modifier) {
m_modifier->modify(&frame);
m_modifier->modify(frame);
}
cv::resize(frame, frame, cv::Size(), scale, scale, cv::INTER_AREA);
cv::cvtColor(frame, frame, CV_BGR2RGB);
......@@ -92,7 +85,16 @@ void Converter::process(cv::Mat frame) {
QImage::Format_RGB888, &matDelete, new cv::Mat(frame)
);
Q_ASSERT(image.constBits() == frame.data);
emit imageReady(image);
Q_EMIT imageReady(image);
}
void Converter::imageKeyEvent(int key) {
#ifndef NDEBUG
qDebug() << "Key pressed: " << key;
#endif
if (m_modifier) {
m_modifier->forwardKeyEvent(key);
}
}
void Converter::timerEvent(QTimerEvent *ev) {
......@@ -114,11 +116,6 @@ const QImage &ImageViewer::getImage() {
}
void ImageViewer::setImage(const QImage &img) {
#ifndef NDEBUG
if (m_img.isNull()) {
qDebug() << "OpenCV Image Viewer dropped a frame";
}
#endif
m_img = img;
if (m_img.size() != size()) {
setFixedSize(m_img.size());
......@@ -173,10 +170,13 @@ CameraDisplay::CameraDisplay(QWidget *parent, int camera_index)
QObject::connect(&m_capture, &Capture::matReady, &m_converter, &Converter::processFrame);
QObject::connect(&m_converter, &Converter::imageReady, m_image_viewer, &ImageViewer::setImage);
QObject::connect(this, &CameraDisplay::forwardKeyEvent, &m_converter, &Converter::imageKeyEvent);
connect(m_camera_list, SIGNAL(currentIndexChanged(int)), this, SLOT(selectedCameraChanged(int)));
connect(m_capture_btn, SIGNAL(clicked()), this, SLOT(captureAndSave()));
connect(m_effects_list, SIGNAL(currentIndexChanged(int)), this, SLOT(effectsChanged(int)));
connect(m_effects_list, SIGNAL(currentIndexChanged(int)), this, SLOT(effectsChanged(int)));
setFocusPolicy(Qt::FocusPolicy::StrongFocus);
}
CameraDisplay::~CameraDisplay() {
......@@ -215,6 +215,10 @@ void CameraDisplay::setVisible(bool visible) {
QDialog::setVisible(visible);
}
void CameraDisplay::keyPressEvent(QKeyEvent *event) {
Q_EMIT forwardKeyEvent(event->key());
}
void CameraDisplay::reject() {
pauseVideo();
QDialog::reject();
......@@ -236,12 +240,19 @@ void CameraDisplay::effectsChanged(int effect_index) {
void CameraDisplay::captureAndSave() {
QString file = QDir::currentPath() + QDir::separator() + "image" + QString::number(m_image_count) + ".png";
#ifndef NDEBUG
qDebug() << "Saving image " << m_image_count;
qDebug() << "Target: " << file;
if(m_image_viewer->getImage().save(file)) {
#endif
if (m_image_viewer->getImage().save(file)) {
++m_image_count;
#ifndef NDEBUG
qDebug() << "Image saved";
} else {
#endif
}
#ifndef NDEBUG
else {
qDebug() << "Failed to save image";
}
#endif
}
......@@ -60,6 +60,8 @@ public:
Q_SLOT void modifierChanged(int modifier_index);
Q_SLOT void imageKeyEvent(int key);
private:
static void matDelete(void *mat);
......@@ -116,12 +118,18 @@ protected:
void reject() override;
void keyPressEvent(QKeyEvent *event) override;
protected Q_SLOTS:
void selectedCameraChanged(int list_index);
void effectsChanged(int effect_index);
void captureAndSave();
void captureAndSave();
Q_SIGNALS:
void forwardKeyEvent(int);
private:
void pauseVideo();
......@@ -129,7 +137,7 @@ private:
QVBoxLayout *m_layout;
QComboBox *m_camera_list;
QComboBox *m_effects_list;
QPushButton *m_capture_btn;
QPushButton *m_capture_btn;
ImageViewer *m_image_viewer;
int m_camera;
......@@ -139,6 +147,6 @@ private:
Converter m_converter;
IThread m_capture_thread;
IThread m_converter_thread;
};
};
#endif //MINOTAUR_CPP_CAMERA_H
#include "modify.h"
#include "squares.h"
#include "tracker.h"
#include "shapedetect.h"
void VideoModifier::attachModifier(std::unique_ptr<VideoModifier> &ptr, int modifier) {
switch(modifier) {
switch (modifier) {
case SQUARES:
ptr.reset(new Squares);
break;
case SHAPEDETECT:
ptr.reset(new Shapedetect);
ptr.reset(new ShapeDetect);
break;
case OBJTRACK:
ptr.reset(new TrackerModifier);
break;
default:
ptr.release();
......@@ -20,4 +24,7 @@ void VideoModifier::addModifierList(QComboBox *list) {
list->addItem("None");
list->addItem("Square");
list->addItem("Shape Detector");
list->addItem("Obj Tracker");
}
void VideoModifier::forwardKeyEvent(int) {}
......@@ -13,13 +13,16 @@ public:
NONE = 0,
SQUARES = 1,
SHAPEDETECT = 2,
OBJTRACK = 3
};
static void attachModifier(std::unique_ptr<VideoModifier> &ptr, int modifier);
static void addModifierList(QComboBox *list);
virtual void modify(cv::Mat *img) = 0;
virtual void modify(cv::Mat &img) = 0;
virtual void forwardKeyEvent(int);
};
#endif //MINOTAUR_CPP_MODIFY_H
......@@ -156,12 +156,12 @@ static cv::Mat findShapes(const cv::Mat &src, std::vector<std::vector<cv::Point>
return dst;
}
void Shapedetect::modify(cv::Mat *img) {
std::vector<std::vector<cv::Point> > triangles;
std::vector<std::vector<cv::Point> > rectangles;
std::vector<std::vector<cv::Point> > circles;
void ShapeDetect::modify(cv::Mat &img) {
std::vector<std::vector<cv::Point>> triangles;
std::vector<std::vector<cv::Point>> rectangles;
std::vector<std::vector<cv::Point>> circles;
*img = findShapes(*img, triangles, rectangles, circles);
img = findShapes(img, triangles, rectangles, circles);
// Outline rectangles and triangles in blue
//drawShapes(*img, triangles);
//drawShapes(*img, rectangles);
......
......@@ -10,9 +10,9 @@
#include "modify.h"
class Shapedetect : public VideoModifier {
class ShapeDetect : public VideoModifier {
public:
void modify(cv::Mat *img) override;
void modify(cv::Mat &img) override;
};
......
......@@ -20,14 +20,14 @@ static double angle(Point pt1, Point pt2, Point pt0) {
// returns sequence of squares detected on the image.
// the sequence is stored in the specified memory storage
static void findSquares(cv::Mat *image, vector<vector<Point> > &squares) {
static void findSquares(cv::Mat &image, vector<vector<Point> > &squares) {
squares.clear();
Mat pyr, timg, gray0(image->size(), CV_8U), gray;
Mat pyr, timg, gray0(image.size(), CV_8U), gray;
// down-scale and upscale the image to filter out the noise
pyrDown(*image, pyr, Size(image->cols / 2, image->rows / 2));
pyrUp(pyr, timg, image->size());
pyrDown(image, pyr, Size(image.cols / 2, image.rows / 2));
pyrUp(pyr, timg, image.size());
vector<vector<Point>> contours;
// find squares in every color plane of the image
......@@ -93,15 +93,15 @@ static void findSquares(cv::Mat *image, vector<vector<Point> > &squares) {
}
// the function draws all the squares in the image
static void drawSquares(cv::Mat *image, const vector<vector<Point>> &squares) {
static void drawSquares(cv::Mat &image, const vector<vector<Point>> &squares) {
for (const auto &square : squares) {
const Point *p = &square[0];
auto n = static_cast<int>(square.size());
polylines(*image, &p, &n, 1, true, Scalar(0, 255, 0), 3);
polylines(image, &p, &n, 1, true, Scalar(0, 255, 0), 3);
}
}
void Squares::modify(cv::Mat *img) {
void Squares::modify(cv::Mat &img) {
vector<vector<Point>> squares;
findSquares(img, squares);
drawSquares(img, squares);
......
......@@ -5,7 +5,7 @@
class Squares : public VideoModifier {
public:
void modify(cv::Mat *img) override;
void modify(cv::Mat &img) override;
};
......
#include <opencv2/opencv.hpp>
#include "tracker.h"
#ifndef NDEBUG
#include <QDebug>
#endif
// CMake will try to find goturn.caffemodel and goturn.prototxt, which need
// to be added separtaely. If these are found, the GOTURN tracker model
// will be used instead of the MIL tracker.
#ifdef GOTURN_FOUND
#define TRACKER_TYPE Type::GOTURN
#else
#define TRACKER_TYPE Type::MIL
#endif
TrackerModifier::TrackerModifier() = default;
TrackerModifier::TrackerModifier()
: m_bounding_box(287, 23, 100, 100),
m_type(TRACKER_TYPE),
m_state(State::UNINITIALIZED) {
switch (m_type) {
case Type::BOOSTING:
m_tracker = cv::TrackerBoosting::create();
break;
case Type::MIL:
m_tracker = cv::TrackerMIL::create();
break;
case Type::KCF:
m_tracker = cv::TrackerKCF::create();
break;
case Type::TLD:
m_tracker = cv::TrackerTLD::create();
break;
case Type::MEDIAN_FLOW:
m_tracker = cv::TrackerMedianFlow::create();
break;
case Type::GOTURN:
#ifndef NDEBUG
qDebug() << "Using GOTURN tracker";
#endif
m_tracker = cv::TrackerGOTURN::create();
break;
default:
break;
}
}
TrackerModifier::~TrackerModifier() = default;
void TrackerModifier::modify(cv::Mat *img) {
void TrackerModifier::forwardKeyEvent(int key) {
#ifndef NDEBUG
qDebug() << "Key event received";
#endif
if (key == Qt::Key_A) {
#ifndef NDEBUG
qDebug() << "Switching to First Scan";
#endif
m_state = State::FIRST_SCAN;
} else if (key == Qt::Key_S) {
#ifndef NDEBUG
qDebug() << "Resetting tracker";
#endif
m_state = State::UNINITIALIZED;
m_bounding_box = {};
}
}
void TrackerModifier::modify(cv::Mat &img) {
if (m_state == State::UNINITIALIZED) {
return;
}
if (m_state == State::TRACKING) {
m_tracker->update(img, m_bounding_box);
} else if (m_state == State::FIRST_SCAN) {
m_bounding_box = cv::selectROI(img);
m_tracker->init(img, m_bounding_box);
m_state = State::TRACKING;
}
cv::rectangle(img, m_bounding_box, cv::Scalar(255, 0, 0));
}
......@@ -3,15 +3,39 @@
#include "modify.h"
#include <opencv2/tracking.hpp>
class TrackerModifier : public VideoModifier {
public:
TrackerModifier();
~TrackerModifier();
void modify(cv::Mat *img) override;
void modify(cv::Mat &img) override;
void forwardKeyEvent(int key) override;
private:
enum Type {
BOOSTING,
MIL,
KCF,
TLD,
MEDIAN_FLOW,
GOTURN
};
enum State {
UNINITIALIZED,
FIRST_SCAN,
TRACKING
};
cv::Ptr<cv::Tracker> m_tracker;
cv::Rect2d m_bounding_box;
int m_type;
int m_state;
};
#endif //MINOTAUR_CPP_TRACKER_H
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment