From 50929e69452b292c36665378536adfcabd529e0a Mon Sep 17 00:00:00 2001 From: wangchongwu <759291707@qq.com> Date: Fri, 24 Jan 2025 18:12:36 +0800 Subject: [PATCH] =?UTF-8?q?=E5=AE=8C=E6=88=90BA=E6=A1=86=E6=9E=B6=EF=BC=8C?= =?UTF-8?q?=E5=BE=85=E4=BF=AE=E6=94=B9=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CMakeLists.txt | 7 + .../source/Src/Mosaic/featureMatch.cpp | 2 +- stitch/CMakeLists.txt | 16 +- stitch/src/Arith_BATask.cpp | 209 ++++++++++++++++-- stitch/src/Arith_BATask.h | 11 +- stitch/src/Arith_FeaMatch.cpp | 50 ++++- stitch/src/Arith_FeaMatch.h | 6 + 7 files changed, 271 insertions(+), 30 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8c0382b..7c4cad1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,6 +4,12 @@ project(stitch VERSION 0.1.0 LANGUAGES C CXX) SET(ArithStitchDir stitch) +IF(WIN32) + set(OpenCV_DIR "D:/opencv410_vc17") + set(CMAKE_TOOLCHAIN_FILE "D:/wangchongwu_gitea_2023/vcpkg-2025.01.13/scripts/buildsystems/vcpkg.cmake") +ELSE(WIN32) + set(OpenCV_DIR "/home/wcw/opencv-3.4.16/install/share/OpenCV") +ENDIF(WIN32) find_package(OpenCV REQUIRED) @@ -11,6 +17,7 @@ include_directories(${OpenCV_INCLUDE_DIRS} public_include ${ArithStitchDir}/src) + # 设置算法库目录 SET(LIB_STITCH GuideStitch) diff --git a/ref/AutoStitching/source/Src/Mosaic/featureMatch.cpp b/ref/AutoStitching/source/Src/Mosaic/featureMatch.cpp index 3cae021..7bd8870 100644 --- a/ref/AutoStitching/source/Src/Mosaic/featureMatch.cpp +++ b/ref/AutoStitching/source/Src/Mosaic/featureMatch.cpp @@ -455,7 +455,7 @@ void PointMatcher::drawMatches(int imgIndex1, int imgIndex2, vector poi CvFont font; double hScale = 1; double vScale = 1; - cvInitFont(&font,CV_FONT_HERSHEY_PLAIN, hScale,vScale,0,1); // + cvInitFont(&font,CV_FONT_HERSHEY_PLAIN, hScale,vScale,0,1); //���������� for (i = 0; i < pointSet1.size(); i ++) { Point2d tempPt1 = pointSet1[i]; diff --git a/stitch/CMakeLists.txt b/stitch/CMakeLists.txt index 741a97c..a59393c 100644 --- a/stitch/CMakeLists.txt +++ b/stitch/CMakeLists.txt @@ -10,13 +10,24 @@ if(MSVC) add_compile_options(/wd4996) endif() -find_package(OpenMP REQUIRED) +IF(WIN32) + set(OpenCV_DIR "D:/opencv410_vc17") + set(CMAKE_TOOLCHAIN_FILE "D:/wangchongwu_gitea_2023/vcpkg-2025.01.13/scripts/buildsystems/vcpkg.cmake") +ELSE(WIN32) + set(OpenCV_DIR "/home/wcw/opencv-3.4.16/install/share/OpenCV") +ENDIF(WIN32) + find_package(Ceres REQUIRED) include_directories(${CERES_INCLUDE_DIRS}) +find_package(OpenMP REQUIRED) +include_directories(${CERES_INCLUDE_DIRS}) + + + # 公共头文件（放置在最前面，所有后面定义的模块才都能找到） SET(ArithTrkPubInc ${CMAKE_SOURCE_DIR}/public_include) @@ -48,7 +59,8 @@ target_include_directories(${LIB_STITCH} PUBLIC target_link_libraries(${LIB_STITCH} OpenMP::OpenMP_CXX ${OpenCV_LIBS} - ${CERES_LIBRARIES}) + ${CERES_LIBRARIES} + ) # # gcc编译器要求0警告 if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") diff --git a/stitch/src/Arith_BATask.cpp b/stitch/src/Arith_BATask.cpp index 3beb07e..80f26f8 100644 --- a/stitch/src/Arith_BATask.cpp +++ b/stitch/src/Arith_BATask.cpp @@ -1,13 +1,52 @@ #include "Arith_BATask.h" #include "ceres/ceres.h" using namespace ceres; - -struct CostFunctor { +// 定义残差结构体 +struct HomographyResidual +{ + HomographyResidual(const cv::KeyPoint& keypoint_i, const cv::KeyPoint& keypoint_j) + : keypoint_i_(keypoint_i), keypoint_j_(keypoint_j) + { + } template - bool operator()(const T* const x, T* residual) const { - residual[0] = 10.0 - x[0]; + bool operator()(const T* const h_i, const T* const h_j, T* residual) const { + // 残差计算逻辑 + T H_i[9] = { h_i[0], h_i[1], h_i[2], + h_i[3], h_i[4], h_i[5], + h_i[6], h_i[7], T(1.0) }; + + T H_j[9] = { h_j[0], h_j[1], h_j[2], + h_j[3], h_j[4], h_j[5], + h_j[6], h_j[7], T(1.0) }; + + T p_i[3] = { T(keypoint_i_.pt.x), T(keypoint_i_.pt.y), T(1.0) }; + T p_j[3] = { T(keypoint_j_.pt.x), T(keypoint_j_.pt.y), T(1.0) }; + + T P_i[3] = { T(0), T(0), T(0) }; + T P_j[3] = { T(0), T(0), T(0) }; + + for (int row = 0; row < 3; row++) { + for (int col = 0; col < 3; col++) { + P_i[row] += H_i[row * 3 + col] * p_i[col]; + P_j[row] += H_j[row * 3 + col] * p_j[col]; + } + } + + P_i[0] /= P_i[2]; + P_i[1] /= P_i[2]; + P_j[0] /= P_j[2]; + P_j[1] /= P_j[2]; + + residual[0] = P_i[0] - P_j[0]; + residual[1] = P_i[1] - P_j[1]; + return true; } + + +private: + const cv::KeyPoint keypoint_i_; // 第 i 帧图像中的特征点 + const cv::KeyPoint keypoint_j_; // 第 j 帧图像中的特征点 }; @@ -15,7 +54,7 @@ struct CostFunctor { BA_Task::BA_Task(GeoStitcher * pGeoTrans) { _GeoStitcher = pGeoTrans; - _FeaMatcher = new FeatureMatcher(DetectorType::ORB, MatcherType::BF); + _FeaMatcher = new FeatureMatcher(DetectorType::SIFT, MatcherType::FLANN); _imgVec.reserve(100); } @@ -52,9 +91,6 @@ SINT32 BA_Task::addFrame(GD_VIDEO_FRAME_S img, FrameInfo para) // 缓存包围多边形 _polygon.push_back(warpRectWithH(H,cv::Size(img.u32Width,img.u32Height))); - - - return _imgVec.size(); } @@ -64,10 +100,6 @@ void BA_Task::setPanPara(PanInfo info) } - - - - void BA_Task::Test() { // google::InitGoogleLogging("ceres"); @@ -100,21 +132,124 @@ void BA_Task::Test() void BA_Task::optimizeBA() { + //for (int i = 0; i < _currMatrix.size(); i++) + //{ + // std::cout << "------------" << std::endl; + // std::cout << _origMatrix[i] << std::endl; + // std::cout << _currMatrix[i] << std::endl; + // std::cout << "------------" << std::endl; + //} + // 计算匹配性矩阵 CalMatchMat(0.3); + google::InitGoogleLogging("ceres"); + + // 将 cv::Mat 转换为 Ceres 需要的数组形式 + std::vector h_list(_origMatrix.size()); + + + for (int i = 0; i < _origMatrix.size(); i++) + { + h_list[i] = (double*)_origMatrix[i].data; + } + // 创建 Ceres 问题 + ceres::Problem problem; + + // 添加残差块 + for (int i = 0; i < _MatchMat.cols; i++) + { + for (int j = i + 1; j < _MatchMat.rows; j++) + { + int M = _MatchMat.at(i, j); // 获取匹配点对的数量 + if (M > 0) + { + // 添加匹配点对的残差块 + for (int m = 0; m < M; m++) + { + // 注意：这里不对，应该找匹配点！！ todo 节后完成 + cv::KeyPoint keypoint_i = _FeaPtVec[i][m]; + cv::KeyPoint keypoint_j = _FeaPtVec[j][m]; + + ceres::CostFunction* cost_function = + new ceres::AutoDiffCostFunction( + new HomographyResidual(keypoint_i, keypoint_j)); + problem.AddResidualBlock(cost_function, nullptr, h_list[i], h_list[j]); + } + } + } + } + + + + // 配置求解器 + ceres::Solver::Options options; + options.minimizer_progress_to_stdout = true; + //options.linear_solver_type = ceres::SPARSE_NORMAL_CHOLESKY; // 使用稀疏求解器 + options.minimizer_progress_to_stdout = true; + options.num_threads = 1; // 使用多线程 + ceres::Solver::Summary summary; + + // 求解 + ceres::Solve(options, &problem, &summary); + //// 将优化后的参数转换回 cv::Mat + for (int i = 0; i < _currMatrix.size(); i++) + { + std::cout << "------------" << std::endl; + std::cout << _origMatrix[i] << std::endl; + std::cout << _currMatrix[i] << std::endl; + std::cout << "------------" << std::endl; + _currMatrix[i].at(0, 0) = h_list[i][0]; + _currMatrix[i].at(0, 1) = h_list[i][1]; + _currMatrix[i].at(0, 2) = h_list[i][2]; + _currMatrix[i].at(1, 0) = h_list[i][3]; + _currMatrix[i].at(1, 1) = h_list[i][4]; + _currMatrix[i].at(1, 2) = h_list[i][5]; + _currMatrix[i].at(2, 0) = h_list[i][6]; + _currMatrix[i].at(2, 1) = h_list[i][7]; + _currMatrix[i].at(2, 2) = 1.0; // 固定 h_33 = 1 + } + + // 输出结果 + std::cout << summary.BriefReport() << std::endl; + + remap(); } +void BA_Task::remap() +{ + cv::Mat image(1000, 1000, CV_8UC3, cv::Scalar(0, 0, 0)); + + for (size_t i = 0; i < _imgVec.size(); i++) + { + cv::Mat imagetmp(1000, 1000, CV_8UC3, cv::Scalar(0, 0, 0)); + cv::warpPerspective(_imgVec[i], imagetmp, _currMatrix[i], imagetmp.size()); + + cv::Mat mask = cv::Mat::ones(_imgVec[i].size(), CV_8UC1) * 255; + cv::Mat warped_mask; + cv::warpPerspective(mask, warped_mask, _currMatrix[i], image.size()); + imagetmp.copyTo(image, warped_mask); + } + + imshow("remap", image); + cv::waitKey(0); +} + +//#define SHOW_MATCH SINT32 BA_Task::CalMatchMat(float fiou_thre) { _IOUMat = cv::Mat::zeros(_polygon.size(),_polygon.size(),CV_32FC1); + + _MatchMat = cv::Mat::zeros(_polygon.size(), _polygon.size(), CV_32SC1); + + // 先计算IOU矩阵 for (size_t i = 0; i < _polygon.size(); i++) { vector poly_i = _polygon[i]; - for (size_t j = 0; j < _polygon.size(); j++) + for (size_t j = i + 1; j < _polygon.size(); j++) { if (i == j) { @@ -133,32 +268,64 @@ SINT32 BA_Task::CalMatchMat(float fiou_thre) // IOU大于0.3以上才进行特征点匹配 for (size_t i = 0; i < _polygon.size(); i++) { - for (size_t j = 0; j < _polygon.size(); j++) + for (size_t j = i+1; j < _polygon.size(); j++) { if (i == j) { continue; } + // IOU满足条件才匹配特征点 if (_IOUMat.at(i,j) < fiou_thre) { continue; } - - - // IOU满足条件才匹配特征点 - cv::Mat image(1000, 1000, CV_8UC3, cv::Scalar(255, 255, 255)); +#ifdef SHOW_MATCH + cv::Mat image(1000, 1000, CV_8UC3, cv::Scalar(0, 0, 0)); + cv::Mat imagetmp(1000, 1000, CV_8UC3, cv::Scalar(0, 0, 0)); vector> tmpPoly; tmpPoly.push_back(_polygon[i]); tmpPoly.push_back(_polygon[j]); + + cv::warpPerspective(_imgVec[i], imagetmp, _origMatrix[i], imagetmp.size()); + + // 生成遮罩（全白图像，表示有效区域） + cv::Mat mask1 = cv::Mat::ones(_imgVec[i].size(), CV_8UC1) * 255; + cv::Mat warped_mask1; + cv::warpPerspective(mask1, warped_mask1, _origMatrix[i], image.size()); + + imagetmp.copyTo(image, warped_mask1); + + cv::warpPerspective(_imgVec[j], imagetmp, _origMatrix[j], imagetmp.size()); + cv::Mat mask2 = cv::Mat::ones(_imgVec[i].size(), CV_8UC1) * 255; + cv::Mat warped_mask2; + cv::warpPerspective(mask2, warped_mask2, _origMatrix[j], image.size()); + + imagetmp.copyTo(image, warped_mask2); + drawPolygons(image, tmpPoly); // 显示绘制结果 cv::imshow("Polygons", image); cv::waitKey(1); + +#endif std::vector matches; - _FeaMatcher->matchFeatures(_FeaDespVec[i],_FeaDespVec[j],matches); + //_FeaMatcher->matchFeatures(_FeaDespVec[i],_FeaDespVec[j],matches); + _FeaMatcher->matchFeatures_WithH(_FeaPtVec[i],_FeaDespVec[i], _FeaPtVec[j], _FeaDespVec[j], _origMatrix[i], _origMatrix[j], matches); + + // 图像特征匹配点对超过N对才认为有效 + if (matches.size() > 50) + { + _MatchMat.at(i, j) = matches.size(); + _MatchMat.at(j, i) = matches.size(); + } + else + { + continue; + } +#ifdef SHOW_MATCH // 可视化匹配结果 cv::Mat img_matches; cv::drawMatches( @@ -172,12 +339,12 @@ SINT32 BA_Task::CalMatchMat(float fiou_thre) cv::DrawMatchesFlags::NOT_DRAW_SINGLE_POINTS // 不绘制未匹配的点 ); - cv::resize(img_matches,img_matches,cv::Size(1280,512)); + cv::resize(img_matches, img_matches, cv::Size(1280, 512)); // 显示匹配结果 cv::imshow("Feature Matches", img_matches); cv::waitKey(0); - +#endif } } diff --git a/stitch/src/Arith_BATask.h b/stitch/src/Arith_BATask.h index 34b0c3f..e087ee0 100644 --- a/stitch/src/Arith_BATask.h +++ b/stitch/src/Arith_BATask.h @@ -38,11 +38,12 @@ public: // BA优化 void optimizeBA(); + // 使用H矩阵重投影所有帧 + void remap(); + private: SINT32 CalMatchMat(float fiou_thre);//计算匹配性矩阵(以IOU约束) - SINT32 GuideMatch(int i,int j);//外参指示下的匹配 - private: GeoStitcher* _GeoStitcher;//外参计算 FeatureMatcher* _FeaMatcher;//特征匹配 @@ -52,8 +53,10 @@ private: Mat_ _MatchMat;//配准点邻接表 Mat_ _IOUMat;//交汇邻接表 - vector _origMatrix;//初始H矩阵 - vector _currMatrix;//当前H矩阵 + vector> _origMatrix;//初始H矩阵 + vector> _currMatrix;//当前H矩阵 + + //超大空间，考虑文件cache todo！ private: diff --git a/stitch/src/Arith_FeaMatch.cpp b/stitch/src/Arith_FeaMatch.cpp index e01434e..471d6ed 100644 --- a/stitch/src/Arith_FeaMatch.cpp +++ b/stitch/src/Arith_FeaMatch.cpp @@ -1,4 +1,7 @@ #include "Arith_FeaMatch.h" +#include "Arith_GeoStitcher.h" +using std::vector; +using cv::KeyPoint; // 构造函数 FeatureMatcher::FeatureMatcher(DetectorType detectorType, MatcherType matcherType) @@ -27,6 +30,49 @@ void FeatureMatcher::matchFeatures(cv::Mat& descriptors1, cv::Mat& descriptors2, } } +void FeatureMatcher::matchFeatures_WithH(vector keypoints1, cv::Mat& descriptors1, vector keypoints2, cv::Mat& descriptors2, cv::Mat H1, cv::Mat H2, std::vector& matches) +{ + std::vector _matches; + if (matcherType_ == FLANN) + { + flannMatcher_->match(descriptors1, descriptors2, _matches); + } + else + { + bfMatcher_->match(descriptors1, descriptors2, _matches); + } + + int i, j; + double minimaDsit = 99999; + for (i = 0; i < _matches.size(); i++) + { + double dist = _matches[i].distance; + if (dist < minimaDsit) + { + minimaDsit = dist; + } + } + double fitedThreshold = minimaDsit * 10; + + for (size_t i = 0; i < _matches.size(); i++) + { + cv::DMatch mc = _matches[i]; + + if (mc.distance > fitedThreshold) + { + continue; + } + + auto warp_pt1 = warpPointWithH(H1, keypoints1[mc.queryIdx].pt); + auto warp_pt2 = warpPointWithH(H2, keypoints2[mc.trainIdx].pt); + + if (fabs(warp_pt1.x - warp_pt2.x) + fabs(warp_pt1.y - warp_pt2.y) < 15) + { + matches.push_back(mc); + } + } +} + // 计算单应性矩阵 cv::Mat FeatureMatcher::computeHomography(std::vector& keypoints1, std::vector& keypoints2, std::vector& matches, double ransacReprojThreshold) @@ -46,7 +92,7 @@ void FeatureMatcher::initDetector() switch (detectorType_) { case SIFT: - detector_ = cv::SIFT::create(100); + detector_ = cv::SIFT::create(500); break; case SURF: //detector_ = cv::xfeatures2d::SURF::create(); @@ -57,7 +103,7 @@ void FeatureMatcher::initDetector() #endif break; case ORB: - detector_ = cv::ORB::create(1000); + detector_ = cv::ORB::create(500); break; default: throw std::invalid_argument("Unsupported feature detector type"); diff --git a/stitch/src/Arith_FeaMatch.h b/stitch/src/Arith_FeaMatch.h index 2d71162..91bd9b2 100644 --- a/stitch/src/Arith_FeaMatch.h +++ b/stitch/src/Arith_FeaMatch.h @@ -35,6 +35,12 @@ public: // 匹配特征点 void matchFeatures(cv::Mat& descriptors1, cv::Mat& descriptors2, std::vector& matches); + + // 带初始H监督的匹配 + void matchFeatures_WithH(std::vector keypoints1, cv::Mat& descriptors1, std::vector keypoints2, cv::Mat& descriptors2, + cv::Mat H1, cv::Mat H2, std::vector& matches); + + // 计算单应性矩阵 cv::Mat computeHomography(std::vector& keypoints1, std::vector& keypoints2, std::vector& matches, double ransacReprojThreshold = 3.0);