From 55dc2a9b9a5a349cb450b372553e9a958c9ac3b6 Mon Sep 17 00:00:00 2001 From: TimZnr Date: Wed, 23 Nov 2022 17:09:27 +0100 Subject: [PATCH 1/8] read certain file instead of random one --- AutonomousMode/Input/input.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AutonomousMode/Input/input.cpp b/AutonomousMode/Input/input.cpp index 6c269a2..7ae1321 100644 --- a/AutonomousMode/Input/input.cpp +++ b/AutonomousMode/Input/input.cpp @@ -22,7 +22,7 @@ Mat Input::readFile(String filePath) // Random shuffle std::random_shuffle(filenames.begin(), filenames.end()); - Mat image = imread(filenames[0], IMREAD_COLOR); + Mat image = imread(filePath, IMREAD_COLOR); if(image.empty()) { From 977f531f72887901d81452479e185c77c269abf3 Mon Sep 17 00:00:00 2001 From: TimZnr Date: Wed, 23 Nov 2022 17:10:04 +0100 Subject: [PATCH 2/8] New approach using a roi and drawContours method --- AutonomousMode/Spielwiese/spielwiese.cpp | 66 ++++++++++++++++++++---- 1 file changed, 55 insertions(+), 11 deletions(-) diff --git a/AutonomousMode/Spielwiese/spielwiese.cpp b/AutonomousMode/Spielwiese/spielwiese.cpp index 5b27b5b..a699702 100644 --- a/AutonomousMode/Spielwiese/spielwiese.cpp +++ b/AutonomousMode/Spielwiese/spielwiese.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -34,20 +35,63 @@ void sweep_em_all(int thresholdBinary, int videoHeight, int videoWidth, int gaus void in_depth_processing_chain(int thresholdBinary, int videoHeight, int videoWidth, int gaussKernelSize, int thresholdCanny1, int thresholdCanny2, int apertureSizeCanny) { - std::string outputFolder = "C:\\Users\\User\\Desktop\\temp"; Input input(videoHeight, videoWidth); - Mat image = input.readFile("C:\\Users\\User\\Desktop\\Studium\\02_Master_MSY\\2. Semester Winter 22 23\\Projekt\\Line-Following-Robot\\Test_data"); - imwrite(outputFolder + "\\01_input.jpg", image); - cvtColor(image, image, COLOR_BGR2GRAY); - imwrite(outputFolder + "\\02_color_convert.jpg", image); - GaussianBlur(image, image, Size(gaussKernelSize, gaussKernelSize), 0); - imwrite(outputFolder + "\\03_gauss.jpg", image); - threshold(image, image, thresholdBinary, 255, THRESH_BINARY); - imwrite(outputFolder + "\\04_threshold.jpg", image); - Canny(image, image, thresholdCanny1, thresholdCanny2, apertureSizeCanny); - imwrite(outputFolder + "\\05_canny.jpg", image); + std::string outputFolder = "C:\\Users\\User\\Desktop\\temp"; + std::string inputFolder = "C:\\Users\\User\\Desktop\\Studium\\02_Master_MSY\\2. Semester Winter 22 23\\Projekt\\Line-Following-Robot\\AutonomousMode\\Test_data"; + std::vector filenames; + cv::glob(inputFolder, filenames); + + //filenames.begin(), filenames.end() + int i = 0; + for(std::vector::iterator it = filenames.begin(); it != filenames.end(); it++) + { + std::string current_output = outputFolder + "\\" + to_string(i); + std::cout << current_output << std::endl; + const char* current_output_char = current_output.c_str(); + _mkdir(current_output_char); + + std::string inputFile = inputFolder + "\\image" + to_string(i+1) + ".jpeg"; + Mat original_image = input.readFile(inputFile); + imwrite(current_output + "\\00_input.jpg", original_image); + + Rect roi = Rect(0, original_image.rows*(7.5/12.0), original_image.cols, original_image.rows/12); + + Mat image = original_image(roi); + + imwrite(current_output + "\\01_roi.jpg", image); + cvtColor(image, image, COLOR_BGR2GRAY); + imwrite(current_output + "\\02_color_convert.jpg", image); + GaussianBlur(image, image, Size(gaussKernelSize, gaussKernelSize), 0); + imwrite(current_output + "\\03_gauss.jpg", image); + threshold(image, image, thresholdBinary, 255, THRESH_BINARY); + imwrite(current_output + "\\04_threshold.jpg", image); + + // Opening (reduces noise) + Mat kernel(5,5, CV_8UC1,1); + morphologyEx(image, image, 2, kernel); + imwrite(current_output + "\\05_opening.jpg", image); + + //Canny(image, image, thresholdCanny1, thresholdCanny2, apertureSizeCanny); + //imwrite(outputFolder + "\\06_canny.jpg", image); + + vector > contours; + vector hierarchy; + + findContours(image,contours, hierarchy, RETR_LIST, CHAIN_APPROX_SIMPLE); + + for( int i = 0; i< contours.size(); i++ ) // iterate through each contour. + { + double a = contourArea( contours[i],false); // Find the area of contour + if(a > 3500) + { + drawContours(original_image, contours, i, Scalar(0,255,255), 1, 8, hierarchy, 0, Point(0,original_image.rows*(7.5/12.0))); + } + } + imwrite(current_output + "\\06_contours.jpg", original_image); + i++; + } } From f208cda1ed2c86b4f851e906924dd44f72b1c94d Mon Sep 17 00:00:00 2001 From: Tim Date: Tue, 29 Nov 2022 18:10:16 +0100 Subject: [PATCH 3/8] Resolve mixing of std::string and cv::String, more work on new img processing approach. --- AutonomousMode/Input/input.cpp | 4 +-- AutonomousMode/Spielwiese/spielwiese.cpp | 45 +++++++++++++++++++----- 2 files changed, 38 insertions(+), 11 deletions(-) diff --git a/AutonomousMode/Input/input.cpp b/AutonomousMode/Input/input.cpp index 7ae1321..96b392a 100644 --- a/AutonomousMode/Input/input.cpp +++ b/AutonomousMode/Input/input.cpp @@ -15,8 +15,8 @@ Mat Input::readFile(String filePath) { std::srand(std::time(0)); // Read all .jpg files from the specified folder - std::string folder = filePath; - std::vector filenames; + cv::String folder = filePath; + std::vector filenames; cv::glob(folder, filenames); // Random shuffle diff --git a/AutonomousMode/Spielwiese/spielwiese.cpp b/AutonomousMode/Spielwiese/spielwiese.cpp index a699702..bd44bf1 100644 --- a/AutonomousMode/Spielwiese/spielwiese.cpp +++ b/AutonomousMode/Spielwiese/spielwiese.cpp @@ -1,4 +1,4 @@ -#include +//#include #include #include @@ -37,15 +37,15 @@ void in_depth_processing_chain(int thresholdBinary, int videoHeight, int videoWi { Input input(videoHeight, videoWidth); - std::string outputFolder = "C:\\Users\\User\\Desktop\\temp"; - std::string inputFolder = "C:\\Users\\User\\Desktop\\Studium\\02_Master_MSY\\2. Semester Winter 22 23\\Projekt\\Line-Following-Robot\\AutonomousMode\\Test_data"; + cv::String outputFolder = "C:\\Users\\tim-z\\Desktop\\temp"; + cv::String inputFolder = "C:\\Users\\tim-z\\Desktop\\Studium\\02_Master MSY\\2. Semester Winter 2022 2023\\Projekt\\Repo\\Line-Following-Robot\\AutonomousMode\\Test_data"; - std::vector filenames; + std::vector filenames; cv::glob(inputFolder, filenames); //filenames.begin(), filenames.end() int i = 0; - for(std::vector::iterator it = filenames.begin(); it != filenames.end(); it++) + for(std::vector::iterator it = filenames.begin(); it != filenames.end(); it++) { std::string current_output = outputFolder + "\\" + to_string(i); std::cout << current_output << std::endl; @@ -56,7 +56,8 @@ void in_depth_processing_chain(int thresholdBinary, int videoHeight, int videoWi Mat original_image = input.readFile(inputFile); imwrite(current_output + "\\00_input.jpg", original_image); - Rect roi = Rect(0, original_image.rows*(7.5/12.0), original_image.cols, original_image.rows/12); + Point roiOrigin(0, original_image.rows*(7.5/12.0)); + Rect roi = Rect(roiOrigin.x, roiOrigin.y, original_image.cols, original_image.rows/12); Mat image = original_image(roi); @@ -78,15 +79,41 @@ void in_depth_processing_chain(int thresholdBinary, int videoHeight, int videoWi vector > contours; vector hierarchy; + vector vectorOfRects; + vector vectorOfLeftEdges; findContours(image,contours, hierarchy, RETR_LIST, CHAIN_APPROX_SIMPLE); - + int amountOfValidRects = 0; for( int i = 0; i< contours.size(); i++ ) // iterate through each contour. { double a = contourArea( contours[i],false); // Find the area of contour if(a > 3500) { - drawContours(original_image, contours, i, Scalar(0,255,255), 1, 8, hierarchy, 0, Point(0,original_image.rows*(7.5/12.0))); + drawContours(original_image, contours, i, Scalar(0,255,255), 1, 8, hierarchy, 0, roiOrigin); + + Rect currentBoundingRect = boundingRect(contours[i]); + //Handle roi offset: + currentBoundingRect.x += roiOrigin.x; + currentBoundingRect.y += roiOrigin.y; + vectorOfRects.push_back(currentBoundingRect); + + rectangle(original_image, currentBoundingRect, Scalar(0,255,0)); + + // get mid-point of rect + Point midRect = Point(currentBoundingRect.x+currentBoundingRect.width/2, currentBoundingRect.y+currentBoundingRect.height/2); + + // Draw middle as small rect instead of circle because for whatever reasons drawing a circle doesnt work. + Rect testRect(Point(midRect.x-2, midRect.y-2), Point(midRect.x+2, midRect.y+2)); + rectangle(original_image, testRect, Scalar(0,0,255)); + // get the left edge of rect + // used as offset as raspicam is not + // mounted on mid of regbot + Point leftEdge(currentBoundingRect.x, currentBoundingRect.y+currentBoundingRect.height/2); + vectorOfLeftEdges.push_back(leftEdge); + + testRect = Rect(Point(leftEdge.x-2, leftEdge.y-2), Point(leftEdge.x+2, leftEdge.y+2)); + rectangle(original_image, testRect, Scalar(0,0,255)); + amountOfValidRects++; } } imwrite(current_output + "\\06_contours.jpg", original_image); @@ -98,7 +125,7 @@ void in_depth_processing_chain(int thresholdBinary, int videoHeight, int videoWi int main(void) { //Disable opencv logging messages - cv::utils::logging::setLogLevel(cv::utils::logging::LOG_LEVEL_WARNING); + //cv::utils::logging::setLogLevel(cv::utils::logging::LOG_LEVEL_WARNING); const int thresholdBinary = 140; const int videoHeight = 720; From b9965ee95674d664aa79109ed9b0e42d2d6c3194 Mon Sep 17 00:00:00 2001 From: Tim Zeuner Date: Wed, 30 Nov 2022 17:35:53 +0100 Subject: [PATCH 4/8] grab+retrieve to use cam on pi --- AutonomousMode/Input/input.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/AutonomousMode/Input/input.cpp b/AutonomousMode/Input/input.cpp index 96b392a..ca704d2 100644 --- a/AutonomousMode/Input/input.cpp +++ b/AutonomousMode/Input/input.cpp @@ -43,7 +43,8 @@ Mat Input::readWebcam() return Mat(); } - cap.read(image); + cap.grab(); + cap.retrieve(image); return image; } From 551d5f4bb30e9e7d3774ff851df5c8ef32a3554b Mon Sep 17 00:00:00 2001 From: Tim Zeuner Date: Wed, 30 Nov 2022 18:53:50 +0100 Subject: [PATCH 5/8] Set video size higher --- AutonomousMode/autonomous_mode_main.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/AutonomousMode/autonomous_mode_main.cpp b/AutonomousMode/autonomous_mode_main.cpp index 5fcb8c5..e4256e7 100644 --- a/AutonomousMode/autonomous_mode_main.cpp +++ b/AutonomousMode/autonomous_mode_main.cpp @@ -7,8 +7,8 @@ int main(void) cv::utils::logging::setLogLevel(cv::utils::logging::LOG_LEVEL_WARNING); const int thresholdBinary = 140; - const int videoHeight = 240; - const int videoWidth = 320; + const int videoHeight = 720; + const int videoWidth = 1280; const int gaussKernelSize = 21; const int thresholdCanny1 = 50; const int thresholdCanny2 = 100; @@ -20,4 +20,4 @@ int main(void) char a; std::cin >> a; lfr.endLoop(); -} \ No newline at end of file +} From e45f46c102b5b7c3ee1434bd546e6aaeca812928 Mon Sep 17 00:00:00 2001 From: Tim Zeuner Date: Wed, 30 Nov 2022 18:55:20 +0100 Subject: [PATCH 6/8] Proper error message, Check if grab() was successfull --- AutonomousMode/Input/input.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/AutonomousMode/Input/input.cpp b/AutonomousMode/Input/input.cpp index ca704d2..e2c5173 100644 --- a/AutonomousMode/Input/input.cpp +++ b/AutonomousMode/Input/input.cpp @@ -39,17 +39,19 @@ Mat Input::readWebcam() Mat image; if(!cap.isOpened()) { - cout << "Fehler"; + cout << "Video capture not opened" << std::endl; return Mat(); } - cap.grab(); + if(!cap.grab()) { + cout << "Could not grab frame from camera" << std::endl; + return Mat(); + } cap.retrieve(image); - return image; } void Input::freeWebcam() { this->cap.release(); -} \ No newline at end of file +} From 01f29ca9cd4ee46fd610ed9d295c7b8e75351cee Mon Sep 17 00:00:00 2001 From: Tim Zeuner Date: Wed, 30 Nov 2022 21:25:56 +0100 Subject: [PATCH 7/8] Choose between showing img and saving it --- AutonomousMode/autonomous_mode_main.cpp | 2 ++ AutonomousMode/lfr.cpp | 23 +++++++++++++++++------ AutonomousMode/lfr.h | 6 ++++++ 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/AutonomousMode/autonomous_mode_main.cpp b/AutonomousMode/autonomous_mode_main.cpp index e4256e7..fb17b25 100644 --- a/AutonomousMode/autonomous_mode_main.cpp +++ b/AutonomousMode/autonomous_mode_main.cpp @@ -15,6 +15,8 @@ int main(void) const int apertureSizeCanny = 3; LFR lfr(videoHeight, videoWidth, thresholdBinary, gaussKernelSize, thresholdCanny1, thresholdCanny2, apertureSizeCanny); + lfr.saveOutputFlag = true; + lfr.outputFileName = "/home/pi/Line-Following-Robot/AutonomousMode/tmp/test.jpg"; lfr.startLoop(); //To end the video stream, write any char in the console. char a; diff --git a/AutonomousMode/lfr.cpp b/AutonomousMode/lfr.cpp index 45b1058..dc6abd1 100644 --- a/AutonomousMode/lfr.cpp +++ b/AutonomousMode/lfr.cpp @@ -10,6 +10,10 @@ LFR::LFR(int videoHeight, int videoWidth, int thresholdBinary, int gaussKernelSi this->thresholdCanny1 = thresholdCanny1; this->thresholdCanny2 = thresholdCanny2; this->apertureSizeCanny = apertureSizeCanny; + + this->videoFlag = false; + this->saveOutputFlag = false; + this->outputFileName = ""; } LFR::~LFR() @@ -22,7 +26,7 @@ LFR::~LFR() void LFR::loop() { - namedWindow("Display window"); + if(this->videoFlag) {namedWindow("Display window");} while(iAmLooping) { Mat image = input.readWebcam(); @@ -32,11 +36,18 @@ void LFR::loop() { line( image, Point(lines[i][0], lines[i][1]), Point( lines[i][2], lines[i][3]), (0,0,255), 1, 8 ); - } - imshow("Display window", image); - char c = (char)waitKey(1); + } + if(this->videoFlag) + { + imshow("Display window", image); + char c = (char)waitKey(1); + } + if (this->saveOutputFlag && !(this->outputFileName.empty())) + { + imwrite(this->outputFileName, image); + } } - destroyWindow("Display window"); + if(this->videoFlag) {destroyWindow("Display window");} input.freeWebcam(); } @@ -51,4 +62,4 @@ void LFR::endLoop() iAmLooping = false; this->loopThread.join(); return; -} \ No newline at end of file +} diff --git a/AutonomousMode/lfr.h b/AutonomousMode/lfr.h index acbb0b5..c8ea635 100644 --- a/AutonomousMode/lfr.h +++ b/AutonomousMode/lfr.h @@ -39,5 +39,11 @@ public: void startLoop(); void endLoop(); + + + bool videoFlag; + bool saveOutputFlag; + + std::string outputFileName; }; \ No newline at end of file From 8c53f49935a2ee5dfe2d73c9454a23876d567640 Mon Sep 17 00:00:00 2001 From: Tim Zeuner Date: Wed, 30 Nov 2022 22:50:50 +0100 Subject: [PATCH 8/8] Implement approach prev tested in spielwiese --- AutonomousMode/Processing/processing.cpp | 40 ++++++++++++++++------- AutonomousMode/Processing/processing.h | 2 +- AutonomousMode/Utils/utils.h | 11 +++++++ AutonomousMode/lfr.cpp | 41 +++++++++++++----------- AutonomousMode/lfr.h | 3 ++ 5 files changed, 67 insertions(+), 30 deletions(-) diff --git a/AutonomousMode/Processing/processing.cpp b/AutonomousMode/Processing/processing.cpp index c7bc1f3..0dd3462 100644 --- a/AutonomousMode/Processing/processing.cpp +++ b/AutonomousMode/Processing/processing.cpp @@ -24,19 +24,37 @@ void Processing::processImage(Mat& inputPicture, int thresholdBinary, int gaussK // And one (the other one) to segment the lines. // No return value here as the input is passed by reference -> directly modified. cvtColor(inputPicture, inputPicture, COLOR_BGR2GRAY); - threshold(inputPicture, inputPicture, thresholdBinary, 255, THRESH_BINARY); GaussianBlur(inputPicture, inputPicture, Size(gaussKernelSize, gaussKernelSize), 0); - Canny(inputPicture, inputPicture, thresholdCanny1, thresholdCanny2, apertureSizeCanny); + threshold(inputPicture, inputPicture, thresholdBinary, 255, THRESH_BINARY); + + //Perform a opening + Mat kernel(5,5, CV_8UC1,1); + morphologyEx(inputPicture, inputPicture, 2, kernel); } -std::vector Processing::calculateLineSegments(const Mat& inputPicture) +FrameData Processing::calculateLineSegments(const Mat& inputPicture, const cv::Rect& roi) { - //See following link - //https://stackoverflow.com/questions/45322630/how-to-detect-lines-in-opencv - vector lines; - VectorOfLines linesInVectors; - HoughLinesP(inputPicture, lines, 1, CV_PI/360, 150, 0, 250); - //lines = linesInVectors.findMiddleLine(lines); - - return lines; + FrameData data; + cv::findContours(inputPicture, data.contours, RETR_LIST, CHAIN_APPROX_SIMPLE); + + //Delete the areas that are too small + auto iterator = data.contours.begin(); + while(iterator != data.contours.end()) + { + if (contourArea(*iterator) < 3500) + { + iterator = data.contours.erase(iterator); + } + else + { + Rect boundingBox = boundingRect(*iterator); + boundingBox.x += roi.x; + boundingBox.y += roi.y; + data.boundingBoxes.push_back(boundingBox); + data.middlePoints.push_back(Point(boundingBox.x+boundingBox.width/2, boundingBox.y+boundingBox.height/2)); + data.leftEdges.push_back(Point(boundingBox.x, boundingBox.y+boundingBox.height/2)); + ++iterator; + } + } + return data; } diff --git a/AutonomousMode/Processing/processing.h b/AutonomousMode/Processing/processing.h index bad9aae..27732f9 100644 --- a/AutonomousMode/Processing/processing.h +++ b/AutonomousMode/Processing/processing.h @@ -21,5 +21,5 @@ public: void processImage(Mat& inputPicture, int thresholdBinary, int gaussKernelSize, int thresholdCanny1, int thresholdCanny2, int apertureSizeCanny); - std::vector calculateLineSegments(const Mat& inputPicture); + FrameData calculateLineSegments(const Mat& inputPicture, const cv::Rect& roi); }; \ No newline at end of file diff --git a/AutonomousMode/Utils/utils.h b/AutonomousMode/Utils/utils.h index 05ea138..46de2a9 100644 --- a/AutonomousMode/Utils/utils.h +++ b/AutonomousMode/Utils/utils.h @@ -60,4 +60,15 @@ class VectorOfLines{ static double calcDistance(Point p0, Point p1); vector findMiddleLine(vector &lines); +}; + +class FrameData +{ +public: + std::vector> contours; + std::vector boundingBoxes; + std::vector leftEdges; + std::vector middlePoints; + + FrameData(): contours(), boundingBoxes(), leftEdges(), middlePoints() {} }; \ No newline at end of file diff --git a/AutonomousMode/lfr.cpp b/AutonomousMode/lfr.cpp index dc6abd1..ae95cf2 100644 --- a/AutonomousMode/lfr.cpp +++ b/AutonomousMode/lfr.cpp @@ -2,7 +2,7 @@ LFR::LFR(int videoHeight, int videoWidth, int thresholdBinary, int gaussKernelSize, int thresholdCanny1, int thresholdCanny2, int apertureSizeCanny) - : iAmLooping(false), input(videoHeight, videoWidth), processing(), controlModule(), interpreter(), intersectionHandler() + : iAmLooping(false), input(videoHeight, videoWidth), processing(), controlModule(), interpreter(), intersectionHandler(), roi() { this->iAmLooping = false; this->thresholdBinary = thresholdBinary; @@ -14,6 +14,10 @@ LFR::LFR(int videoHeight, int videoWidth, int thresholdBinary, int gaussKernelSi this->videoFlag = false; this->saveOutputFlag = false; this->outputFileName = ""; + + cv::Point roiOrigin(0, videoHeight*(7.5/12.0)); + roi = Rect(roiOrigin.x, roiOrigin.y, videoWidth, videoHeight/12); + } LFR::~LFR() @@ -29,23 +33,11 @@ void LFR::loop() if(this->videoFlag) {namedWindow("Display window");} while(iAmLooping) { - Mat image = input.readWebcam(); - processing.processImage(image, this->thresholdBinary, this->gaussKernelSize, this->thresholdCanny1, thresholdCanny2, this->apertureSizeCanny); - std::vector lines = processing.calculateLineSegments(image); - for( size_t i = 0; i < lines.size(); i++ ) - { - line( image, Point(lines[i][0], lines[i][1]), - Point( lines[i][2], lines[i][3]), (0,0,255), 1, 8 ); - } - if(this->videoFlag) - { - imshow("Display window", image); - char c = (char)waitKey(1); - } - if (this->saveOutputFlag && !(this->outputFileName.empty())) - { - imwrite(this->outputFileName, image); - } + Mat originalImage = input.readWebcam(); + Mat processedImage = originalImage; + processing.processImage(processedImage, this->thresholdBinary, this->gaussKernelSize, this->thresholdCanny1, thresholdCanny2, this->apertureSizeCanny); + FrameData data = processing.calculateLineSegments(processedImage, this->roi); + this->provideOutput(processedImage); } if(this->videoFlag) {destroyWindow("Display window");} input.freeWebcam(); @@ -63,3 +55,16 @@ void LFR::endLoop() this->loopThread.join(); return; } + +void LFR::provideOutput(const Mat& image) +{ + if(this->videoFlag) + { + imshow("Display window", image); + char c = (char)waitKey(1); + } + if (this->saveOutputFlag && !(this->outputFileName.empty())) + { + imwrite(this->outputFileName, image); + } +} diff --git a/AutonomousMode/lfr.h b/AutonomousMode/lfr.h index c8ea635..76460ec 100644 --- a/AutonomousMode/lfr.h +++ b/AutonomousMode/lfr.h @@ -30,6 +30,9 @@ class LFR int thresholdCanny1; int thresholdCanny2; int apertureSizeCanny; + cv::Rect roi; + + void provideOutput(const Mat& image); public: