diff --git a/3rdParty/debug/lib/ceres-debug.lib b/3rdParty/debug/lib/ceres-debug.lib new file mode 100644 index 0000000..4de6847 Binary files /dev/null and b/3rdParty/debug/lib/ceres-debug.lib differ diff --git a/3rdParty/debug/lib/gflags_debug.lib b/3rdParty/debug/lib/gflags_debug.lib new file mode 100644 index 0000000..6fffdf6 Binary files /dev/null and b/3rdParty/debug/lib/gflags_debug.lib differ diff --git a/3rdParty/debug/lib/gflags_nothreads_debug.lib b/3rdParty/debug/lib/gflags_nothreads_debug.lib new file mode 100644 index 0000000..4fd0634 Binary files /dev/null and b/3rdParty/debug/lib/gflags_nothreads_debug.lib differ diff --git a/3rdParty/debug/lib/glogd.lib b/3rdParty/debug/lib/glogd.lib new file mode 100644 index 0000000..47d41c6 Binary files /dev/null and b/3rdParty/debug/lib/glogd.lib differ diff --git a/3rdParty/debug/lib/opencv_world455d.lib b/3rdParty/debug/lib/opencv_world455d.lib new file mode 100644 index 0000000..f86c437 Binary files /dev/null and b/3rdParty/debug/lib/opencv_world455d.lib differ diff --git a/3rdParty/release/lib/ceres.lib b/3rdParty/release/lib/ceres.lib new file mode 100644 index 0000000..91e9ff4 Binary files /dev/null and b/3rdParty/release/lib/ceres.lib differ diff --git a/3rdParty/release/lib/gflags.lib b/3rdParty/release/lib/gflags.lib new file mode 100644 index 0000000..5998520 Binary files /dev/null and b/3rdParty/release/lib/gflags.lib differ diff --git a/3rdParty/release/lib/gflags_nothreads.lib b/3rdParty/release/lib/gflags_nothreads.lib new file mode 100644 index 0000000..ac92a8a Binary files /dev/null and b/3rdParty/release/lib/gflags_nothreads.lib differ diff --git a/3rdParty/release/lib/glog.lib b/3rdParty/release/lib/glog.lib new file mode 100644 index 0000000..17d3ad6 Binary files /dev/null and b/3rdParty/release/lib/glog.lib differ diff --git a/3rdParty/release/lib/opencv_world455.lib b/3rdParty/release/lib/opencv_world455.lib new file mode 100644 index 0000000..c35f130 Binary files /dev/null and b/3rdParty/release/lib/opencv_world455.lib differ diff --git a/CMakeLists.txt b/CMakeLists.txt index 61309fd..4c0a504 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -40,6 +40,9 @@ target_link_libraries(stitch_DJ ${OpenCV_LIBS} ${LIB_STITCH}) +add_executable(stitch_Fea "feaStitchTest.cpp") +target_link_libraries(stitch_Fea ${OpenCV_LIBS} ${LIB_STITCH}) + diff --git a/ProcDJ.cpp b/ProcDJ.cpp index 8034f84..9c78b39 100644 --- a/ProcDJ.cpp +++ b/ProcDJ.cpp @@ -215,8 +215,13 @@ void ProcDJVideo(vector videoPathList, vector srtPathL // 初始化 stitcher->Init(info); + mat_pan = stitcher->ExportPanMat(); + cap.release(); + cv::VideoWriter output; + output.open("D:/DJ_stitchVL.mp4", cv::VideoWriter::fourcc('H', '2', '6', '4'), 5, cv::Size(mat_pan.cols/8, mat_pan.rows/8), true); + for (size_t vid = 0; vid < videoPathList.size(); vid++) { printf("Proc %s\n", videoPathList[vid].c_str()); @@ -277,6 +282,7 @@ void ProcDJVideo(vector videoPathList, vector srtPathL { continue; } + std::cout << info.craft.stPos.B << " " << info.craft.stPos.L << " " << info.craft.stPos.H << " " << info.craft.stAtt.fYaw << " " << info.craft.stAtt.fPitch << " " << info.craft.stAtt.fRoll << " " << info.servoInfo.fServoAz << " " << info.servoInfo.fServoPt @@ -289,12 +295,14 @@ void ProcDJVideo(vector videoPathList, vector srtPathL tm.stop(); - mat_pan = stitcher->ExportPanMat(); + Mat pan_rgb, pan_rgb_ds; cv::cvtColor(mat_pan, pan_rgb, cv::COLOR_BGRA2BGR); cv::resize(pan_rgb, pan_rgb_ds, cv::Size(pan_rgb.cols / 8, pan_rgb.rows / 8)); + output.write(pan_rgb_ds); + imshow("pan_rgb", pan_rgb_ds); if (cv::waitKey(1) == 27) { @@ -312,7 +320,7 @@ void ProcDJVideo(vector videoPathList, vector srtPathL std::cout << "time opt:" << tm.getTimeMilli() << std::endl; - + output.release(); } } diff --git a/feaStitchTest.cpp b/feaStitchTest.cpp new file mode 100644 index 0000000..01ac412 --- /dev/null +++ b/feaStitchTest.cpp @@ -0,0 +1,181 @@ +#include +#include +#include +#include +#include +#include +#include "API_FeaStitch.h" +#include "PlatformDefine.h" +#include +#include "opencv2/opencv.hpp" +#include + +void ProcDJVideo(std::string videoPathList) +{ + auto stitcher = API_FeaStitch::Create(); + + + + GD_VIDEO_FRAME_S frame = { 0 };//输入帧 + GD_VIDEO_FRAME_S pan = { 0 };//输出全景 + + cv::Mat mat_pan;//全景显示 + + + cv::VideoCapture cap(videoPathList); + + // Get video properties + double fps = cap.get(cv::CAP_PROP_FPS); + int width = static_cast(cap.get(cv::CAP_PROP_FRAME_WIDTH)); + int height = static_cast(cap.get(cv::CAP_PROP_FRAME_HEIGHT)); + int frame_count = static_cast(cap.get(cv::CAP_PROP_FRAME_COUNT)); + + + // dsample + int nDownSample = 1; + + if (width > 3000) + { + nDownSample = 4; + } + + + + FrameInfo info = { 0 }; + info.nFrmID = 0; + info.camInfo.nFocus = 0; + info.camInfo.fPixelSize = 0; + + info.craft.stAtt.fYaw = 0; + info.craft.stAtt.fPitch = 0; + info.craft.stAtt.fRoll = 0; + + info.craft.stPos.B = 0; + info.craft.stPos.L = 0; + info.craft.stPos.H = 0; + + + info.nEvHeight = 0; + + info.servoInfo.fServoAz = 0; + info.servoInfo.fServoPt = 0; + + info.nWidth = width / nDownSample; + info.nHeight = height / nDownSample; + + cv::Mat mat; + + // Read a new frame + cap >> mat; + Mat mat_ds2; + + cv::resize(mat, mat_ds2, cv::Size(width / nDownSample, height / nDownSample)); + + frame.enPixelFormat = GD_PIXEL_FORMAT_RGB_PACKED; + frame.u32Width = mat_ds2.cols; + frame.u32Height = mat_ds2.rows; + frame.u64VirAddr[0] = mat_ds2.data; + + // 初始化 + stitcher->Init(frame,info); + + mat_pan = stitcher->ExportPanMat(); + Mat pan_rgb, pan_rgb_ds; + cv::cvtColor(mat_pan, pan_rgb, cv::COLOR_BGRA2BGR); + + cv::resize(pan_rgb, pan_rgb_ds, cv::Size(pan_rgb.cols / 2, pan_rgb.rows / 2)); + + imshow("pan_rgb", pan_rgb_ds); + cv::waitKey(0); + + cv::VideoWriter output; + output.open("D:/DJ_stitchVL.mp4", cv::VideoWriter::fourcc('H', '2', '6', '4'), 5, cv::Size(mat_pan.cols / 3, mat_pan.rows / 3), true); + + + int frmID = 0; + while (true) + { + cv::Mat mat; + + // Read a new frame + cap >> mat; + + // Check if frame is empty (end of video) + if (mat.empty()) { + std::cout << "End of video\n"; + cap.release(); + break; + } + + frmID++; + Mat mat_ds2; + + cv::resize(mat, mat_ds2, cv::Size(width / nDownSample, height / nDownSample)); + + + + FrameInfo info = { 0 }; + info.nFrmID = frmID; + info.nWidth = mat_ds2.cols; + info.nHeight = mat_ds2.rows; + + + frame.enPixelFormat = GD_PIXEL_FORMAT_RGB_PACKED; + frame.u32Width = mat_ds2.cols; + frame.u32Height = mat_ds2.rows; + frame.u64VirAddr[0] = mat_ds2.data; + + if (frmID % 100 != 0) + { + continue; + } + + + cv::TickMeter tm; + tm.start(); + // 基于外参的快拼 + stitcher->Run(frame, info); + + tm.stop(); + + printf("cost time:%f\n", tm.getTimeMilli()); + + + Mat pan_rgb, pan_rgb_ds; + cv::cvtColor(mat_pan, pan_rgb, cv::COLOR_BGRA2BGR); + + cv::resize(pan_rgb, pan_rgb_ds, cv::Size(pan_rgb.cols / 2, pan_rgb.rows / 2)); + + output.write(pan_rgb_ds); + + imshow("pan_rgb", pan_rgb_ds); + if (cv::waitKey(1) == 27) + { + break; + } + + } + + + + output.release(); + +} + + + + + + +using std::string; + +int main() +{ + + string videoPath = "F:/DJI_202504181507_016/DJI_20250418152649_0005_W.MP4"; + + ProcDJVideo(videoPath); + return 0; +} + + diff --git a/main.cpp b/main.cpp index 7e282ee..84ab125 100644 --- a/main.cpp +++ b/main.cpp @@ -268,9 +268,9 @@ void ProcessVL(string filePath,string outname) info.nHeight = IMAGE_HEIGHT_VL; - info.craft.stAtt.fYaw += gr.generate(); - info.craft.stAtt.fPitch += gr.generate(); - info.craft.stAtt.fRoll += gr.generate(); + //info.craft.stAtt.fYaw += gr.generate(); + //info.craft.stAtt.fPitch += gr.generate(); + //info.craft.stAtt.fRoll += gr.generate(); cv::Mat mat_src(IMAGE_HEIGHT_VL * 1.5, IMAGE_WIDTH_VL, CV_8UC1, pFrameVL); cv::Mat IMG; @@ -292,7 +292,7 @@ void ProcessVL(string filePath,string outname) mat_pan = cv::Mat(pan.u32Height, pan.u32Width, CV_8UC4, pan.u64VirAddr[0]); - output.open("D:/output.mp4", VideoWriter::fourcc('H', '2', '6', '4'), 25, Size(pan.u32Width, pan.u32Height), true); + output.open("D:/output.mp4", VideoWriter::fourcc('H', '2', '6', '4'), 5, Size(pan.u32Width/8, pan.u32Height/8), true); if (!output.isOpened()) { cout << "打开视频失败" << endl; @@ -320,18 +320,19 @@ void ProcessVL(string filePath,string outname) cout << "time:" << tm.getTimeMilli() << endl; - output.write(mat_pan); - // 接收帧 - //auto a = stitcher->ReceiveFrame(frame, info); } - cv::Mat res; - cv::resize(mat_pan, res, cv::Size(pan.u32Width / 4, pan.u32Height / 4)); - imshow("pan_opt", res); + Mat pan_rgb, pan_rgb_ds; + cv::cvtColor(mat_pan, pan_rgb, cv::COLOR_BGRA2BGR); + + cv::resize(pan_rgb, pan_rgb_ds, cv::Size(pan_rgb.cols / 8, pan_rgb.rows / 8)); + + output.write(pan_rgb_ds); + + imshow("pan_opt", pan_rgb_ds); - output.release(); waitKey(1); @@ -348,13 +349,22 @@ void ProcessVL(string filePath,string outname) cout << "time opt:" << tm.getTimeMilli() << endl; - cv::Mat res; - cv::resize(mat_pan, res, cv::Size(pan.u32Width / 4, pan.u32Height / 4)); - imshow("pan_opt", res); + Mat pan_rgb, pan_rgb_ds; + cv::cvtColor(mat_pan, pan_rgb, cv::COLOR_BGRA2BGR); + + cv::resize(pan_rgb, pan_rgb_ds, cv::Size(pan_rgb.cols / 8, pan_rgb.rows / 8)); + + for (int i = 0; i < 25; i++) + { + cv::putText(pan_rgb_ds, "BA opt", cv::Point(100, 40), 0, 1, cv::Scalar(255, 0, 0)); + output.write(pan_rgb_ds); + } + waitKey(1); + output.release(); } @@ -497,10 +507,10 @@ int main(int, char**) { //ProcessIR("F:/S729/22.xraw", "22"); //ProcessVL("Z:/729dataset/raw/vl_1920_1080_para40_y8/20241219153557_11.video", "20241219152643_1"); - //ProcessVL("F:/S729/20241219152917_4.video", "20241219152917_4"); + ProcessVL("F:/S729/20241219152917_4.video", "20241219152917_4"); //ProcessVL("F:/S729/20241219153515_10.video", "20241219153515_10"); // - ProcessVL("F:/S729/1.video", "1"); + //ProcessVL("F:/S729/20241219153557_11.video", "20241219153557_11"); //ProcessVL() diff --git a/stitch/include/Arith_FeaStitch.h b/stitch/include/Arith_FeaStitch.h new file mode 100644 index 0000000..3c14b52 --- /dev/null +++ b/stitch/include/Arith_FeaStitch.h @@ -0,0 +1,30 @@ +class FeaStitch : public API_FeaStitch +{ +public: + FeaStitch(); + ~FeaStitch(); + + GenPanInfo Init(GD_VIDEO_FRAME_S img, FrameInfo info); + void SetOutput(std::string filename, std::string outdir); + SINT32 Run(GD_VIDEO_FRAME_S img, FrameInfo para); + GD_VIDEO_FRAME_S ExportPanAddr(); + cv::Mat ExportPanMat(); + +private: + void SetBase(cv::Mat& img, FrameInfo& info, EdgeDirection edgeDirection); + EdgeDirection isNearEdge(cv::Mat& H); + void calculateInitialOffset(EdgeDirection direction, int& offsetX, int& offsetY); + float calculateIOU(const cv::Mat& H1, const cv::Mat& H2, const cv::Size& imgSize); + bool checkHContinuity(const cv::Mat& prevH, const cv::Mat& currH); + + FeatureMatcher* _FeaMatcher; + std::vector _paraVec; + std::vector _imgVec; + std::vector> _keypointsVec; + std::vector _descriptorsVec; + std::vector _currMatrix; + cv::Mat _panImage; + GenPanInfo _panPara; + SINT32 _totalFrameCnt; + SINT32 _failedFrameCnt; // 连续失败帧计数 +}; \ No newline at end of file diff --git a/stitch/src/API_FeaStitch.h b/stitch/src/API_FeaStitch.h new file mode 100644 index 0000000..d9b9652 --- /dev/null +++ b/stitch/src/API_FeaStitch.h @@ -0,0 +1,44 @@ +/*********版权所有（C）2025，武汉高德红外股份有限公司*************** +* 文件名称：API_FrontStitch.h +* 文件标识： +* 内容摘要：条带拼接算法（基于纯特征） +* 其它说明： +* 当前版本： +* 创建作者：04046wcw +* 创建日期：2025/05/14 +*******************************************************************/ +#pragma once + +#ifdef _WIN32 +#define STD_STITCH_API __declspec(dllexport) +#else +#define STD_STITCH_API __attribute__ ((visibility("default"))) +#endif + +#include "StitchStruct.h" + + + +// 视频帧连续的条带拼接 +class STD_STITCH_API API_FeaStitch +{ +public: + virtual ~API_FeaStitch() = default; + + // 初始化拼接 + virtual GenPanInfo Init(GD_VIDEO_FRAME_S img, FrameInfo info) = 0; + + // 运行拼接流程 + virtual SINT32 Run(GD_VIDEO_FRAME_S img, FrameInfo para) = 0; + + + // 获取全景图 + virtual GD_VIDEO_FRAME_S ExportPanAddr() = 0; + + + virtual cv::Mat ExportPanMat() = 0; + +public: + static API_FeaStitch* Create(std::string cachedir = "./cache"); + static void Destroy(API_FeaStitch* obj); +}; diff --git a/stitch/src/Arith_FeaMatch.cpp b/stitch/src/Arith_FeaMatch.cpp index e68c430..82d2327 100644 --- a/stitch/src/Arith_FeaMatch.cpp +++ b/stitch/src/Arith_FeaMatch.cpp @@ -136,4 +136,16 @@ void FeatureMatcher::initMatcher() bfMatcher_ = cv::BFMatcher::create(cv::NORM_HAMMING); } } +} + +void FeatureMatcher::matchKnn(cv::Mat& descriptors1, cv::Mat& descriptors2, std::vector>& matches, int k) +{ + if (matcherType_ == FLANN) + { + flannMatcher_->knnMatch(descriptors1, descriptors2, matches, k); + } + else + { + bfMatcher_->knnMatch(descriptors1, descriptors2, matches, k); + } } \ No newline at end of file diff --git a/stitch/src/Arith_FeaMatch.h b/stitch/src/Arith_FeaMatch.h index 9e92b6c..b27ba39 100644 --- a/stitch/src/Arith_FeaMatch.h +++ b/stitch/src/Arith_FeaMatch.h @@ -38,12 +38,13 @@ public: // 匹配特征点 void matchFeatures(cv::Mat& descriptors1, cv::Mat& descriptors2, std::vector& matches); + // K近邻匹配 + void matchKnn(cv::Mat& descriptors1, cv::Mat& descriptors2, std::vector>& matches, int k); // 带初始H监督的匹配 void matchFeatures_WithH(std::vector keypoints1, cv::Mat& descriptors1, std::vector keypoints2, cv::Mat& descriptors2, cv::Mat H1, cv::Mat H2, std::vector& matches); - // 计算单应性矩阵 cv::Mat computeHomography(std::vector& keypoints1, std::vector& keypoints2, std::vector& matches, double ransacReprojThreshold = 3.0); diff --git a/stitch/src/Arith_FeaStitch.cpp b/stitch/src/Arith_FeaStitch.cpp new file mode 100644 index 0000000..8f5d1d1 --- /dev/null +++ b/stitch/src/Arith_FeaStitch.cpp @@ -0,0 +1,407 @@ +#include "Arith_FeaStitch.h" +#include "FileCache.h" +#include "Arith_FeaMatch.h" +#include "Arith_Utils.h" +#include +#include + +API_FeaStitch* API_FeaStitch::Create(std::string cachedir) +{ + return new FeaStitch(); +} + + +void API_FeaStitch::Destroy(API_FeaStitch* obj) +{ + delete obj; +} + + + +FeaStitch::FeaStitch() +{ + _FeaMatcher = new FeatureMatcher(DetectorType::ORB, MatcherType::FLANN); +} + +FeaStitch::~FeaStitch() +{ + if (_FeaMatcher) + { + delete _FeaMatcher; + _FeaMatcher = nullptr; + } +} + +GenPanInfo FeaStitch::Init(GD_VIDEO_FRAME_S img, FrameInfo info) +{ + // 获取Mat + cv::Mat src = getRGBAMatFromGDFrame(img, img.u64VirAddr[0]); + + // 创建全景图 + _panPara.m_pan_width = info.nWidth * 2; + _panPara.m_pan_height = info.nHeight * 2; + + _panImage = cv::Mat::zeros(_panPara.m_pan_height, _panPara.m_pan_width, CV_8UC4); + + // 使用SetBase初始化，默认从中心开始 + SetBase(src, info, EdgeDirection::NONE); + + return _panPara; +} + + + +void FeaStitch::SetOutput(std::string filename, std::string outdir) +{ +} + +EdgeDirection FeaStitch::isNearEdge(cv::Mat& H) +{ + // 获取变换后的四个角点 + std::vector corners(4); + corners[0] = cv::Point2f(0, 0); + corners[1] = cv::Point2f(_imgVec.back().cols, 0); + corners[2] = cv::Point2f(_imgVec.back().cols, _imgVec.back().rows); + corners[3] = cv::Point2f(0, _imgVec.back().rows); + + std::vector transformed_corners; + cv::perspectiveTransform(corners, transformed_corners, H.clone()); + + // 检查是否有角点接近全景图边界 + const float margin = 200.0f; // 增加边界容差 + bool nearLeft = false, nearRight = false, nearTop = false, nearBottom = false; + int nearCount = 0; // 记录接近边界的角点数量 + + for (const auto& corner : transformed_corners) + { + if (corner.x < margin) + { + nearLeft = true; + nearCount++; + } + if (corner.x > _panPara.m_pan_width - margin) + { + nearRight = true; + nearCount++; + } + if (corner.y < margin) + { + nearTop = true; + nearCount++; + } + if (corner.y > _panPara.m_pan_height - margin) + { + nearBottom = true; + nearCount++; + } + } + + // 只有当至少两个角点接近边界时才触发重新定位 + if (nearCount < 2) + { + return EdgeDirection::NONE; + } + + // 判断出界方向 + if (nearLeft && nearTop) return EdgeDirection::TOP_LEFT; + if (nearRight && nearTop) return EdgeDirection::TOP_RIGHT; + if (nearLeft && nearBottom) return EdgeDirection::BOTTOM_LEFT; + if (nearRight && nearBottom) return EdgeDirection::BOTTOM_RIGHT; + if (nearLeft) return EdgeDirection::LEFT; + if (nearRight) return EdgeDirection::RIGHT; + if (nearTop) return EdgeDirection::TOP; + if (nearBottom) return EdgeDirection::BOTTOM; + + return EdgeDirection::NONE; +} + +void FeaStitch::calculateInitialOffset(EdgeDirection direction, int& offsetX, int& offsetY) +{ + // 默认偏移量（中心位置） + offsetX = _panPara.m_pan_width / 2; + offsetY = _panPara.m_pan_height / 2; + + // 根据图像尺寸动态计算margin + int margin = std::min(_panPara.m_pan_width, _panPara.m_pan_height) / 4; // 使用图像尺寸的1/4作为margin + margin = std::max(margin, 300); // 确保margin至少300像素 + + // 根据出界方向调整偏移量 + switch (direction) + { + case EdgeDirection::LEFT: + offsetX = _panPara.m_pan_width - margin; + break; + case EdgeDirection::RIGHT: + offsetX = margin; + break; + case EdgeDirection::TOP: + offsetY = _panPara.m_pan_height - margin; + break; + case EdgeDirection::BOTTOM: + offsetY = margin; + break; + case EdgeDirection::TOP_LEFT: + offsetX = _panPara.m_pan_width - margin; + offsetY = _panPara.m_pan_height - margin; + break; + case EdgeDirection::TOP_RIGHT: + offsetX = margin; + offsetY = _panPara.m_pan_height - margin; + break; + case EdgeDirection::BOTTOM_LEFT: + offsetX = _panPara.m_pan_width - margin; + offsetY = margin; + break; + case EdgeDirection::BOTTOM_RIGHT: + offsetX = margin; + offsetY = margin; + break; + default: + break; + } +} + +void FeaStitch::SetBase(cv::Mat& img, FrameInfo& info, EdgeDirection edgeDirection) +{ + // 清空所有缓存 + _paraVec.clear(); + _imgVec.clear(); + _keypointsVec.clear(); + _descriptorsVec.clear(); + _currMatrix.clear(); + _totalFrameCnt = 0; + _failedFrameCnt = 0; + // 清空全景图 + _panImage.setTo(0); + + // 计算初始偏移量 + int offsetX = _panPara.m_pan_width / 2; + int offsetY = _panPara.m_pan_height / 2; + + // 根据出界方向计算偏移量 + calculateInitialOffset(edgeDirection, offsetX, offsetY); + + // 初始化基准H矩阵 + float s = 0.3; + cv::Mat H0 = (cv::Mat_(3, 3) << + s, 0, offsetX, + 0, s, offsetY, + 0, 0, 1 + ); + + // 保存第一帧的H矩阵 + _currMatrix.push_back(H0); + + // 处理第一帧 + _paraVec.push_back(info); + _imgVec.push_back(img); + + // 提取特征点 + std::vector keypoints; + cv::Mat descriptors; + _FeaMatcher->extractFeatures(img, keypoints, descriptors); + + // 保存特征点和描述子 + _keypointsVec.push_back(keypoints); + _descriptorsVec.push_back(descriptors); + _totalFrameCnt = 1; + + // 将第一帧变换到全景图坐标系 + cv::Mat imagetmp(_panImage.size(), CV_8UC3, cv::Scalar(0, 0, 0)); + cv::warpPerspective(img, imagetmp, H0, imagetmp.size(), cv::INTER_LINEAR, cv::BORDER_TRANSPARENT); + + cv::Mat mask = cv::Mat::ones(img.size(), CV_8UC1); + cv::Mat warped_mask; + cv::warpPerspective(mask, warped_mask, H0, imagetmp.size(), cv::INTER_LINEAR); + imagetmp.copyTo(_panImage, warped_mask); +} + +float FeaStitch::calculateIOU(const cv::Mat& H1, const cv::Mat& H2, const cv::Size& imgSize) +{ + // 获取第一帧的四个角点 + std::vector corners1(4); + corners1[0] = cv::Point2f(0, 0); + corners1[1] = cv::Point2f(imgSize.width, 0); + corners1[2] = cv::Point2f(imgSize.width, imgSize.height); + corners1[3] = cv::Point2f(0, imgSize.height); + + // 获取第二帧的四个角点 + std::vector corners2 = corners1; + + // 变换角点 + std::vector transformed_corners1, transformed_corners2; + cv::perspectiveTransform(corners1, transformed_corners1, H1); + cv::perspectiveTransform(corners2, transformed_corners2, H2); + + // 计算两个多边形的面积 + float area1 = cv::contourArea(transformed_corners1); + float area2 = cv::contourArea(transformed_corners2); + + // 计算交集面积 + std::vector intersection; + cv::intersectConvexConvex(transformed_corners1, transformed_corners2, intersection); + float intersectionArea = cv::contourArea(intersection); + + // 计算并集面积 + float unionArea = area1 + area2 - intersectionArea; + + // 返回IOU + return intersectionArea / unionArea; +} + +bool FeaStitch::checkHContinuity(const cv::Mat& prevH, const cv::Mat& currH) +{ + // 计算两个H矩阵之间的差异 + cv::Mat diff = prevH - currH; + + // 计算差异矩阵的范数 + double norm = cv::norm(diff, cv::NORM_L2); + + // 计算相对变化率 + double relativeChange = norm / cv::norm(prevH, cv::NORM_L2); + + // 设置阈值 + const double MAX_RELATIVE_CHANGE = 0.3; // 最大允许30%的相对变化 + + return relativeChange <= MAX_RELATIVE_CHANGE; +} + +SINT32 FeaStitch::Run(GD_VIDEO_FRAME_S img, FrameInfo para) +{ + // 保存帧参数和数据 + _paraVec.push_back(para); + + cv::Mat src = getRGBAMatFromGDFrame(img, img.u64VirAddr[0]); + _imgVec.push_back(src); + + // 提取特征点 + std::vector keypoints; + cv::Mat descriptors; + _FeaMatcher->extractFeatures(src, keypoints, descriptors); + + // 保存特征点和描述子 + _keypointsVec.push_back(keypoints); + _descriptorsVec.push_back(descriptors); + _totalFrameCnt++; + + // 如果帧数小于2，无法进行拼接 + if (_totalFrameCnt < 2) + { + return _totalFrameCnt; + } + + // 获取当前帧和前一帧的数据 + std::vector& currKeypoints = _keypointsVec[_totalFrameCnt - 1]; + std::vector& prevKeypoints = _keypointsVec[_totalFrameCnt - 2]; + cv::Mat& currDescriptors = _descriptorsVec[_totalFrameCnt - 1]; + cv::Mat& prevDescriptors = _descriptorsVec[_totalFrameCnt - 2]; + + // 特征匹配 + std::vector matches; + _FeaMatcher->matchFeatures(prevDescriptors, currDescriptors, matches); + + // 提取匹配点对 + std::vector prevPoints, currPoints; + for (auto& match : matches) + { + prevPoints.push_back(prevKeypoints[match.queryIdx].pt); + currPoints.push_back(currKeypoints[match.trainIdx].pt); + } + + // 计算单应性矩阵 + cv::Mat H; + if (prevPoints.size() >= 4) + { + H = cv::findHomography(currPoints, prevPoints, cv::RANSAC, 3.0); + } + + if (H.empty()) + { + _failedFrameCnt++; + // 移除当前帧的数据 + _paraVec.pop_back(); + _imgVec.pop_back(); + _keypointsVec.pop_back(); + _descriptorsVec.pop_back(); + _totalFrameCnt--; + return _totalFrameCnt; + } + + // 更新当前H矩阵 + cv::Mat newH; + if (_currMatrix.empty()) + { + newH = H; + } + else + { + newH = _currMatrix.back() * H; + } + + // 检查H矩阵的连续性 + if (!_currMatrix.empty() && !checkHContinuity(_currMatrix.back(), newH)) + { + _failedFrameCnt++; + // 移除当前帧的数据 + _paraVec.pop_back(); + _imgVec.pop_back(); + _keypointsVec.pop_back(); + _descriptorsVec.pop_back(); + _totalFrameCnt--; + return _totalFrameCnt; + } + + // 检查是否接近边缘 + EdgeDirection edgeDirection = isNearEdge(newH); + if (edgeDirection != EdgeDirection::NONE && _totalFrameCnt > 3) + { + // 重新初始化全景图 + SetBase(src, para, edgeDirection); + return _totalFrameCnt; + } + + // 检查IOU，如果与上一帧重叠度太高则跳过 + if (!_currMatrix.empty()) + { + float iou = calculateIOU(_currMatrix.back(), newH, src.size()); + if (iou > 0.8f) // 如果重叠度超过80%，认为是无效帧 + { + _failedFrameCnt++; + // 移除当前帧的数据 + _paraVec.pop_back(); + _imgVec.pop_back(); + _keypointsVec.pop_back(); + _descriptorsVec.pop_back(); + _totalFrameCnt--; + return _totalFrameCnt; + } + } + + // 重置失败计数 + _failedFrameCnt = 0; + + // 保存新的H矩阵 + _currMatrix.push_back(newH); + + // 将当前帧变换到全景图坐标系 + cv::Mat imagetmp(_panImage.size(), CV_8UC3, cv::Scalar(0, 0, 0)); + cv::warpPerspective(src, imagetmp, newH, imagetmp.size(), cv::INTER_LINEAR, cv::BORDER_TRANSPARENT); + + cv::Mat mask = cv::Mat::ones(src.size(), CV_8UC1); + cv::Mat warped_mask; + cv::warpPerspective(mask, warped_mask, newH, imagetmp.size(), cv::INTER_LINEAR); + imagetmp.copyTo(_panImage, warped_mask); + + return _totalFrameCnt; +} + +GD_VIDEO_FRAME_S FeaStitch::ExportPanAddr() +{ + return GD_VIDEO_FRAME_S(); +} + +cv::Mat FeaStitch::ExportPanMat() +{ + return _panImage; +} + diff --git a/stitch/src/Arith_FeaStitch.h b/stitch/src/Arith_FeaStitch.h new file mode 100644 index 0000000..6955126 --- /dev/null +++ b/stitch/src/Arith_FeaStitch.h @@ -0,0 +1,72 @@ +#ifndef _FEASTITCH_H +#define _FEASTITCH_H +#include "API_FeaStitch.h" +#include "opencv2/opencv.hpp" +#include "Arith_BATask.h" +#include + +// 定义出界方向枚举 +enum class EdgeDirection +{ + NONE, // 未出界 + LEFT, // 左边界 + RIGHT, // 右边界 + TOP, // 上边界 + BOTTOM, // 下边界 + TOP_LEFT, // 左上角 + TOP_RIGHT, // 右上角 + BOTTOM_LEFT,// 左下角 + BOTTOM_RIGHT// 右下角 +}; + +class FeaStitch :public API_FeaStitch +{ +public: + FeaStitch(); + ~FeaStitch(); + + GenPanInfo Init(GD_VIDEO_FRAME_S img,FrameInfo info); + + // 当前帧设为拼接基准 + void SetBase(cv::Mat& img, FrameInfo& info, EdgeDirection edgeDirection = EdgeDirection::NONE); + + + void SetOutput(std::string filename, std::string outdir); + + SINT32 Run(GD_VIDEO_FRAME_S img, FrameInfo para); + +public: + + GD_VIDEO_FRAME_S ExportPanAddr(); + + cv::Mat ExportPanMat(); + + FeatureMatcher* _FeaMatcher;//特征匹配 + +private: + std::vector _paraVec; // 帧参数 + std::vector _imgVec; //图像缓存 + std::vector> _keypointsVec; // 特征点 + std::vector _descriptorsVec; // 描述子 + std::vector> _currMatrix;//当前H矩阵 + int _totalFrameCnt;//处理帧计数 + int _failedFrameCnt;//匹配失败计数 + +private: + GenPanInfo _panPara;//全景图配置 + cv::Mat _panImage; //全景图 + + // 检测图像是否接近边缘，返回出界方向 + EdgeDirection isNearEdge(cv::Mat& H); + + // 根据出界方向计算初始偏移量 + void calculateInitialOffset(EdgeDirection direction, int& offsetX, int& offsetY); + + float calculateIOU(const cv::Mat& H1, const cv::Mat& H2, const cv::Size& imgSize); + + bool checkHContinuity(const cv::Mat& prevH, const cv::Mat& currH); +}; + + + +#endif \ No newline at end of file diff --git a/stitch/src/Arith_UnderStitch.cpp b/stitch/src/Arith_UnderStitch.cpp index dfabfd7..1029a37 100644 --- a/stitch/src/Arith_UnderStitch.cpp +++ b/stitch/src/Arith_UnderStitch.cpp @@ -84,8 +84,8 @@ UPanInfo UnderStitch::InitMap(FrameInfo info) auto cur = warpPointWithH(H_0, ct_geo); // 计算平移到全景图固定点的平移量，从此处开始拼接 - int planX = panPara.m_pan_width/2 + 500; - int planY = panPara.m_pan_height - 500; + int planX = panPara.m_pan_width/2 + 400 ; + int planY = panPara.m_pan_height; panPara.map_shiftX = planX - (cur.x);//平移X @@ -126,7 +126,7 @@ UPanInfo UnderStitch::Init(FrameInfo info) { _panPara = InitMap(info); - _panImage = cv::Mat::zeros(_panPara.m_pan_height, _panPara.m_pan_width, CV_8UC4); + _panImage = cv::Mat(_panPara.m_pan_height, _panPara.m_pan_width, CV_8UC4, cv::Scalar(255, 255, 255, 255)); _panMask = cv::Mat::zeros(_panPara.m_pan_height, _panPara.m_pan_width, CV_8UC1); @@ -270,9 +270,11 @@ SINT32 UnderStitch::GeoStitch(GD_VIDEO_FRAME_S img, FrameInfo para) _googleProduct.ExportGeoPng(Orth_Image, info, _outDir + "/FrameTile"); } + //src = src.colRange(200, src.cols); // 利用H投影当前帧到全景 cv::Mat imagetmp(_panImage.size(), CV_8UC3, cv::Scalar(0, 0, 0)); + //cv::Mat imagetmp(_panImage.size(), CV_8UC3); cv::warpPerspective(src, imagetmp, H, imagetmp.size(), cv::INTER_LINEAR, cv::BORDER_TRANSPARENT); @@ -417,7 +419,7 @@ SINT32 UnderStitch::ProcessFrame(vector keys) } // 优化所有帧 - _BATask->OptFrame(keys, _H_pan); + //_BATask->OptFrame(keys, _H_pan); // 重投影所有帧到全景 _BlendTask->DirectMap(keys, _H_pan, _panImage,_panMask); diff --git a/stitch/src/FileCache.h b/stitch/src/FileCache.h index 19e131e..2f8a115 100644 --- a/stitch/src/FileCache.h +++ b/stitch/src/FileCache.h @@ -34,7 +34,7 @@ public: bool get(const KeyType& key, std::shared_ptr& pData); void set(const KeyType& key, std::shared_ptr pData); - + void clear(); // 新增清空缓存接口 size_t getMemSize(); private: @@ -183,3 +183,22 @@ size_t FileCache::getMemSize() std::lock_guard lock(mutex_); return cache_.size(); } + +template +void FileCache::clear() +{ + std::lock_guard lock(mutex_); + + // 清空内存缓存 + cache_.clear(); + lru_list_.clear(); + + // 删除磁盘缓存文件 + for (const auto& entry : fs::directory_iterator(cache_dir_)) + { + if (entry.path().extension() == ".cache") + { + fs::remove(entry.path()); + } + } +} diff --git a/stitch/src/StitchStruct.h b/stitch/src/StitchStruct.h index a8b8d48..071b959 100644 --- a/stitch/src/StitchStruct.h +++ b/stitch/src/StitchStruct.h @@ -109,3 +109,9 @@ struct cuda_Mem }; +// 通用全景图配置 +struct GenPanInfo +{ + int m_pan_width; + int m_pan_height; +}; diff --git a/stitch/src/Version.h b/stitch/src/Version.h index 8fa98b2..bbc3b0b 100644 --- a/stitch/src/Version.h +++ b/stitch/src/Version.h @@ -2,5 +2,5 @@ #pragma once #include -std::string BUILD_TIME = "BUILD_TIME 2025_04_30-15.05.00"; +std::string BUILD_TIME = "BUILD_TIME 2025_05_14-20.05.47"; std::string VERSION = "BUILD_VERSION 1.0.1";