devbox
编程挑战者
这个屌丝很懒，什么也没留下！
关注作者
热门标签
热门文章
当前位置:   article > 正文

yolov8 实例分割 onnx runtime C++部署_opencv onnxruntime yolov8

作者：编程挑战者 | 2024-01-29 22:42:39

赞

踩

opencv onnxruntime yolov8

如果第一次部署分割,建议先看这篇博客:

YOLOv5 实例分割 用 OPenCV DNN C++ 部署_爱钓鱼的歪猴的博客-CSDN博客

目录

Pre

一、OpenCV DNN C++ 部署

二、ONNX RUNTIME C++ 部署

yolov8_seg_utils.h

yolov8_seg_utils.cpp

yolov8_seg_onnx.h

yolov8_seg_onnx.cpp

main.cpp

CMakelist.txt

Pre

一定要知道，yolov8的输出与Yolov5 7.0 实例分割的输出不一样，

output0: float32[1,116,8400]。 116是4个box坐标信息+80个类别概率+32个mask系数

output1: float32[1,32,160,160]。一张图片一组mask原型。这个mask原型是相对于整个图像的。

一、OpenCV DNN C++ 部署

 所以output0需要装置一下，117->116,其他的与yolov5实例分割没啥区别：

  1. net.setInput(blob);
  2.     std::vector<cv::Mat> net_output_img;
  3.     vector<string> output_layer_names{ "output0","output1" };
  4.     net.forward(net_output_img, output_layer_names); //get outputs
  5.     std::vector<int> class_ids;// res-class_id
  6.     std::vector<float> confidences;// res-conf
  7.     std::vector<cv::Rect> boxes;// res-box
  8.     std::vector<vector<float>> picked_proposals;  //output0[:,:, 4 + _className.size():net_width]===> for mask
  9.     int net_width = _className.size() + 4 + _segChannels;
  10.  
  11. //转置
  12.     Mat output0=Mat( Size(net_output_img[0].size[2], net_output_img[0].size[1]), CV_32F, (float*)net_output_img[0].data).t();  //[bs,116,8400]=>[bs,8400,116]
  13.  
  14.     int rows = output0.rows;
  15.     float* pdata = (float*)output0.data;
  16.     for (int r = 0; r < rows; ++r) {
  17.             cv::Mat scores(1, _className.size(), CV_32FC1, pdata + 4);
  18.             Point classIdPoint;
  19.             double max_class_socre;
  20.             minMaxLoc(scores, 0, &max_class_socre, 0, &classIdPoint);
  21.             max_class_socre = (float)max_class_socre;
  22.             if (max_class_socre >= _classThreshold) {
  23.                 vector<float> temp_proto(pdata + 4 + _className.size(), pdata + net_width);
  24.                 picked_proposals.push_back(temp_proto);
  25.                 //rect [x,y,w,h]
  26.                 float x = (pdata[0] - params[2]) / params[0];
  27.                 float y = (pdata[1] - params[3]) / params[1];
  28.                 float w = pdata[2] / params[0];
  29.                 float h = pdata[3] / params[1];
  30.                 int left = MAX(int(x - 0.5 * w + 0.5), 0);
  31.                 int top = MAX(int(y - 0.5 * h + 0.5), 0);
  32.                 class_ids.push_back(classIdPoint.x);
  33.                 confidences.push_back(max_class_socre);
  34.                 boxes.push_back(Rect(left, top, int(w + 0.5), int(h + 0.5)));
  35.             }
  36.             pdata += net_width;//next line
  37.     }
  38.     //NMS
  39.     vector<int> nms_result;
  40.     NMSBoxes(boxes, confidences, _classThreshold, _nmsThreshold, nms_result);
  41.     std::vector<vector<float>> temp_mask_proposals;
  42.     Rect holeImgRect(0, 0, srcImg.cols, srcImg.rows);
  43.     for (int i = 0; i < nms_result.size(); ++i) {
  44.  
  45.         int idx = nms_result[i];
  46.         OutputSeg result;
  47.         result.id = class_ids[idx];
  48.         result.confidence = confidences[idx];
  49.         result.box = boxes[idx] & holeImgRect;
  50.         temp_mask_proposals.push_back(picked_proposals[idx]);
  51.         output.push_back(result);
  52.     }
  53.     MaskParams mask_params;
  54.     mask_params.params = params;
  55.     mask_params.srcImgShape = srcImg.size();
  56.     for (int i = 0; i < temp_mask_proposals.size(); ++i) {
  57.         GetMask2(Mat(temp_mask_proposals[i]).t(), net_output_img[1], output[i], mask_params);
  58.     }

但是我read模型的时候报错：ERROR during processing node with 2 inputs and 1 outputs: [Reshape]:(/model.22/dfl/Reshape_output_0) from domain='ai.onnx'

应该是OpenCV版本需要升级到4.7。之前部署检测的时候也报这个错：yolov8 OpenCV DNN 部署 推理报错_爱钓鱼的歪猴的博客-CSDN博客

我不想升级OpenCV了，所以选择用ONNX RUNTIME 部署

二、ONNX RUNTIME C++ 部署

yolov8n-seg  CPU 推理耗时150ms左右， 速度比yolov5n-seg慢一点(100ms)，但是检测效果好一些，具体来说，同样是我穿上这件外套，yolov8n-seg给出的置信度更高！

yolov8_seg_utils.h

  1. #pragma once
  2. #include<iostream>
  3. #include <numeric>
  4. #include<opencv2/opencv.hpp>
  5.  
  6. #define YOLO_P6 false //是否使用P6模型
  7. #define ORT_OLD_VISON 12  //ort1.12.0 之前的版本为旧版本API
  8. struct OutputSeg {
  9.     int id;             //结果类别id
  10.     float confidence;   //结果置信度
  11.     cv::Rect box;       //矩形框
  12.     cv::Mat boxMask;       //矩形框内mask，节省内存空间和加快速度
  13. };
  14. struct MaskParams {
  15.     int segChannels = 32;
  16.     int segWidth = 160;
  17.     int segHeight = 160;
  18.     int netWidth = 640;
  19.     int netHeight = 640;
  20.     float maskThreshold = 0.5;
  21.     cv::Size srcImgShape;
  22.     cv::Vec4d params;
  23.  
  24. };
  25. bool CheckParams(int netHeight, int netWidth, const int* netStride, int strideSize);
  26. void DrawPred(cv::Mat& img, std::vector<OutputSeg> result, std::vector<std::string> classNames, std::vector<cv::Scalar> color);
  27. void LetterBox(const cv::Mat& image, cv::Mat& outImage,
  28.     cv::Vec4d& params, //[ratio_x,ratio_y,dw,dh]
  29.     const cv::Size& newShape = cv::Size(640, 640),
  30.     bool autoShape = false,
  31.     bool scaleFill = false,
  32.     bool scaleUp = true,
  33.     int stride = 32,
  34.     const cv::Scalar& color = cv::Scalar(114, 114, 114));
  35. void GetMask(const cv::Mat& maskProposals, const cv::Mat& maskProtos, std::vector<OutputSeg>& output, const MaskParams& maskParams);
  36. void GetMask2(const cv::Mat& maskProposals, const cv::Mat& maskProtos, OutputSeg& output, const MaskParams& maskParams);

yolov8_seg_utils.cpp

  1. #pragma once
  2. #include "yolov8_seg_utils.h"
  3. using namespace cv;
  4. using namespace std;
  5. bool CheckParams(int netHeight, int netWidth, const int* netStride, int strideSize) {
  6.     if (netHeight % netStride[strideSize - 1] != 0 || netWidth % netStride[strideSize - 1] != 0)
  7.     {
  8.         cout << "Error:_netHeight and _netWidth must be multiple of max stride " << netStride[strideSize - 1] << "!" << endl;
  9.         return false;
  10.     }
  11.     return true;
  12. }
  13.  
  14. void LetterBox(const cv::Mat& image, cv::Mat& outImage, cv::Vec4d& params, const cv::Size& newShape,
  15.     bool autoShape, bool scaleFill, bool scaleUp, int stride, const cv::Scalar& color)
  16. {
  17.     if (false) {
  18.         int maxLen = MAX(image.rows, image.cols);
  19.         outImage = Mat::zeros(Size(maxLen, maxLen), CV_8UC3);
  20.         image.copyTo(outImage(Rect(0, 0, image.cols, image.rows)));
  21.         params[0] = 1;
  22.         params[1] = 1;
  23.         params[3] = 0;
  24.         params[2] = 0;
  25.     }
  26.  
  27.     cv::Size shape = image.size();
  28.     float r = std::min((float)newShape.height / (float)shape.height,
  29.         (float)newShape.width / (float)shape.width);
  30.     if (!scaleUp)
  31.         r = std::min(r, 1.0f);
  32.  
  33.     float ratio[2]{ r, r };
  34.     int new_un_pad[2] = { (int)std::round((float)shape.width * r),(int)std::round((float)shape.height * r) };
  35.  
  36.     auto dw = (float)(newShape.width - new_un_pad[0]);
  37.     auto dh = (float)(newShape.height - new_un_pad[1]);
  38.  
  39.     if (autoShape)
  40.     {
  41.         dw = (float)((int)dw % stride);
  42.         dh = (float)((int)dh % stride);
  43.     }
  44.     else if (scaleFill)
  45.     {
  46.         dw = 0.0f;
  47.         dh = 0.0f;
  48.         new_un_pad[0] = newShape.width;
  49.         new_un_pad[1] = newShape.height;
  50.         ratio[0] = (float)newShape.width / (float)shape.width;
  51.         ratio[1] = (float)newShape.height / (float)shape.height;
  52.     }
  53.  
  54.     dw /= 2.0f;
  55.     dh /= 2.0f;
  56.  
  57.     if (shape.width != new_un_pad[0] && shape.height != new_un_pad[1])
  58.     {
  59.         cv::resize(image, outImage, cv::Size(new_un_pad[0], new_un_pad[1]));
  60.     }
  61.     else {
  62.         outImage = image.clone();
  63.     }
  64.  
  65.     int top = int(std::round(dh - 0.1f));
  66.     int bottom = int(std::round(dh + 0.1f));
  67.     int left = int(std::round(dw - 0.1f));
  68.     int right = int(std::round(dw + 0.1f));
  69.     params[0] = ratio[0];
  70.     params[1] = ratio[1];
  71.     params[2] = left;
  72.     params[3] = top;
  73.     cv::copyMakeBorder(outImage, outImage, top, bottom, left, right, cv::BORDER_CONSTANT, color);
  74. }
  75.  
  76. void GetMask(const cv::Mat& maskProposals, const cv::Mat& maskProtos, std::vector<OutputSeg>& output, const MaskParams& maskParams) {
  77.     //cout << maskProtos.size << endl;
  78.  
  79.     int seg_channels = maskParams.segChannels;
  80.     int net_width = maskParams.netWidth;
  81.     int seg_width = maskParams.segWidth;
  82.     int net_height = maskParams.netHeight;
  83.     int seg_height = maskParams.segHeight;
  84.     float mask_threshold = maskParams.maskThreshold;
  85.     Vec4f params = maskParams.params;
  86.     Size src_img_shape = maskParams.srcImgShape;
  87.  
  88.     Mat protos = maskProtos.reshape(0, { seg_channels,seg_width * seg_height });
  89.  
  90.     Mat matmul_res = (maskProposals * protos).t();
  91.     Mat masks = matmul_res.reshape(output.size(), { seg_width,seg_height });
  92.     vector<Mat> maskChannels;
  93.     split(masks, maskChannels);
  94.     for (int i = 0; i < output.size(); ++i) {
  95.         Mat dest, mask;
  96.         //sigmoid
  97.         cv::exp(-maskChannels[i], dest);
  98.         dest = 1.0 / (1.0 + dest);
  99.  
  100.         Rect roi(int(params[2] / net_width * seg_width), int(params[3] / net_height * seg_height), int(seg_width - params[2] / 2), int(seg_height - params[3] / 2));
  101.         dest = dest(roi);
  102.         resize(dest, mask, src_img_shape, INTER_NEAREST);
  103.  
  104.         //crop
  105.         Rect temp_rect = output[i].box;
  106.         mask = mask(temp_rect) > mask_threshold;
  107.         output[i].boxMask = mask;
  108.     }
  109. }
  110.  
  111. void GetMask2(const Mat& maskProposals, const Mat& mask_protos, OutputSeg& output, const MaskParams& maskParams) {
  112.     int seg_channels = maskParams.segChannels;
  113.     int net_width = maskParams.netWidth;
  114.     int seg_width = maskParams.segWidth;
  115.     int net_height = maskParams.netHeight;
  116.     int seg_height = maskParams.segHeight;
  117.     float mask_threshold = maskParams.maskThreshold;
  118.     Vec4f params = maskParams.params;
  119.     Size src_img_shape = maskParams.srcImgShape;
  120.  
  121.     Rect temp_rect = output.box;
  122.     //crop from mask_protos
  123.     int rang_x = floor((temp_rect.x * params[0] + params[2]) / net_width * seg_width);
  124.     int rang_y = floor((temp_rect.y * params[1] + params[3]) / net_height * seg_height);
  125.     int rang_w = ceil(((temp_rect.x + temp_rect.width) * params[0] + params[2]) / net_width * seg_width) - rang_x;
  126.     int rang_h = ceil(((temp_rect.y + temp_rect.height) * params[1] + params[3]) / net_height * seg_height) - rang_y;
  127.  
  128.     //如果下面的 mask_protos(roi_rangs).clone()位置报错，说明你的output.box数据不对，或者矩形框就1个像素的，开启下面的注释部分防止报错。
  129.     rang_w = MAX(rang_w, 1);
  130.     rang_h = MAX(rang_h, 1);
  131.     if (rang_x + rang_w > seg_width) {
  132.         if (seg_width - rang_x > 0)
  133.             rang_w = seg_width - rang_x;
  134.         else
  135.             rang_x -= 1;
  136.     }
  137.     if (rang_y + rang_h > seg_height) {
  138.         if (seg_height - rang_y > 0)
  139.             rang_h = seg_height - rang_y;
  140.         else
  141.             rang_y -= 1;
  142.     }
  143.  
  144.     vector<Range> roi_rangs;
  145.     roi_rangs.push_back(Range(0, 1));
  146.     roi_rangs.push_back(Range::all());
  147.     roi_rangs.push_back(Range(rang_y, rang_h + rang_y));
  148.     roi_rangs.push_back(Range(rang_x, rang_w + rang_x));
  149.  
  150.     //crop
  151.     Mat temp_mask_protos = mask_protos(roi_rangs).clone();
  152.     Mat protos = temp_mask_protos.reshape(0, { seg_channels,rang_w * rang_h });
  153.     Mat matmul_res = (maskProposals * protos).t();
  154.     Mat masks_feature = matmul_res.reshape(1, { rang_h,rang_w });
  155.     Mat dest, mask;
  156.  
  157.     //sigmoid
  158.     cv::exp(-masks_feature, dest);
  159.     dest = 1.0 / (1.0 + dest);
  160.  
  161.     int left = floor((net_width / seg_width * rang_x - params[2]) / params[0]);
  162.     int top = floor((net_height / seg_height * rang_y - params[3]) / params[1]);
  163.     int width = ceil(net_width / seg_width * rang_w / params[0]);
  164.     int height = ceil(net_height / seg_height * rang_h / params[1]);
  165.  
  166.     resize(dest, mask, Size(width, height), INTER_NEAREST);
  167.     mask = mask(temp_rect - Point(left, top)) > mask_threshold;
  168.     output.boxMask = mask;
  169.  
  170. }
  171.  
  172. void DrawPred(Mat& img, vector<OutputSeg> result, std::vector<std::string> classNames, vector<Scalar> color) {
  173.     Mat mask = img.clone();
  174.     for (int i = 0; i < result.size(); i++) {
  175.         int left, top;
  176.         left = result[i].box.x;
  177.         top = result[i].box.y;
  178.         int color_num = i;
  179.         rectangle(img, result[i].box, color[result[i].id], 2, 8);
  180.         if(result[i].boxMask.rows&& result[i].boxMask.cols>0)
  181.             mask(result[i].box).setTo(color[result[i].id], result[i].boxMask);
  182.         string label = classNames[result[i].id] + ":" + to_string(result[i].confidence);
  183.         int baseLine;
  184.         Size labelSize = getTextSize(label, FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine);
  185.         top = max(top, labelSize.height);
  186.         //rectangle(frame, Point(left, top - int(1.5 * labelSize.height)), Point(left + int(1.5 * labelSize.width), top + baseLine), Scalar(0, 255, 0), FILLED);
  187.         putText(img, label, Point(left, top), FONT_HERSHEY_SIMPLEX, 1, color[result[i].id], 2);
  188.     }
  189.     addWeighted(img, 0.5, mask, 0.5, 0, img); //add mask to src
  190. //    imshow("1", img);
  191.     //imwrite("out.bmp", img);
  192. //    waitKey();
  193.     //destroyAllWindows();
  194.  
  195. }

yolov8_seg_onnx.h

  1. #pragma once
  2. #include <iostream>
  3. #include<memory>
  4. #include <opencv2/opencv.hpp>
  5. #include "yolov8_seg_utils.h"
  6. #include<onnxruntime_cxx_api.h>
  7.  
  8. //#include <tensorrt_provider_factory.h>  //if use OrtTensorRTProviderOptionsV2
  9. //#include <onnxruntime_c_api.h>
  10.  
  11.  
  12. class Yolov8SegOnnx {
  13. public:
  14.     Yolov8SegOnnx() :_OrtMemoryInfo(Ort::MemoryInfo::CreateCpu(OrtAllocatorType::OrtDeviceAllocator, OrtMemType::OrtMemTypeCPUOutput)) {};
  15.     ~Yolov8SegOnnx() {};// delete _OrtMemoryInfo;
  16.  
  17.  
  18. public:
  19.     /** \brief Read onnx-model
  20.     * \param[in] modelPath:onnx-model path
  21.     * \param[in] isCuda:if true,use Ort-GPU,else run it on cpu.
  22.     * \param[in] cudaID:if isCuda==true,run Ort-GPU on cudaID.
  23.     * \param[in] warmUp:if isCuda==true,warm up GPU-model.
  24.     */
  25.     bool ReadModel(const std::string& modelPath, bool isCuda = false, int cudaID = 0, bool warmUp = true);
  26.  
  27.     /** \brief  detect.
  28.     * \param[in] srcImg:a 3-channels image.
  29.     * \param[out] output:detection results of input image.
  30.     */
  31.     bool OnnxDetect(cv::Mat& srcImg, std::vector<OutputSeg>& output);
  32.     /** \brief  detect,batch size= _batchSize
  33.     * \param[in] srcImg:A batch of images.
  34.     * \param[out] output:detection results of input images.
  35.     */
  36.     bool OnnxBatchDetect(std::vector<cv::Mat>& srcImg, std::vector<std::vector<OutputSeg>>& output);
  37.  
  38. private:
  39.  
  40.     template <typename T>
  41.     T VectorProduct(const std::vector<T>& v)
  42.     {
  43.         return std::accumulate(v.begin(), v.end(), 1, std::multiplies<T>());
  44.     };
  45.     int Preprocessing(const std::vector<cv::Mat>& SrcImgs, std::vector<cv::Mat>& OutSrcImgs, std::vector<cv::Vec4d>& params);
  46. #if(defined YOLO_P6 && YOLO_P6==true)
  47.     //const float _netAnchors[4][6] = { { 19,27, 44,40, 38,94 },{ 96,68, 86,152, 180,137 },{ 140,301, 303,264, 238,542 },{ 436,615, 739,380, 925,792 } };
  48.     const int _netWidth = 1280;  //ONNX图片输入宽度
  49.     const int _netHeight = 1280; //ONNX图片输入高度
  50.     const int _segWidth = 320;  //_segWidth=_netWidth/mask_ratio
  51.     const int _segHeight = 320;
  52.     const int _segChannels = 32;
  53.  
  54. #else
  55.     //const float _netAnchors[3][6] = { { 10,13, 16,30, 33,23 },{ 30,61, 62,45, 59,119 },{ 116,90, 156,198, 373,326 } };
  56.     const int _netWidth = 640;   //ONNX-net-input-width
  57.     const int _netHeight = 640;  //ONNX-net-input-height
  58.     const int _segWidth = 160;    //_segWidth=_netWidth/mask_ratio
  59.     const int _segHeight = 160;
  60.     const int _segChannels = 32;
  61.  
  62. #endif // YOLO_P6
  63.  
  64.     int _batchSize = 1;  //if multi-batch,set this
  65.     bool _isDynamicShape = false;//onnx support dynamic shape
  66.  
  67.  
  68.     float _classThreshold = 0.5;
  69.     float _nmsThreshold = 0.45;
  70.     float _maskThreshold = 0.5;
  71.  
  72.  
  73.     //ONNXRUNTIME
  74.     Ort::Env _OrtEnv = Ort::Env(OrtLoggingLevel::ORT_LOGGING_LEVEL_ERROR, "Yolov5-Seg");
  75.     Ort::SessionOptions _OrtSessionOptions = Ort::SessionOptions();
  76.     Ort::Session* _OrtSession = nullptr;
  77.     Ort::MemoryInfo _OrtMemoryInfo;
  78. #if ORT_API_VERSION < ORT_OLD_VISON
  79.  
  80.     char* _inputName, * _output_name0, * _output_name1;
  81. #else
  82.     std::shared_ptr<char> _inputName, _output_name0,_output_name1;
  83. #endif
  84.  
  85.     std::vector<char*> _inputNodeNames; //输入节点名
  86.     std::vector<char*> _outputNodeNames;//输出节点名
  87.  
  88.     size_t _inputNodesNum = 0;        //输入节点数
  89.     size_t _outputNodesNum = 0;       //输出节点数
  90.  
  91.     ONNXTensorElementDataType _inputNodeDataType; //数据类型
  92.     ONNXTensorElementDataType _outputNodeDataType;
  93.     std::vector<int64_t> _inputTensorShape; //输入张量shape
  94.  
  95.     std::vector<int64_t> _outputTensorShape;
  96.     std::vector<int64_t> _outputMaskTensorShape;
  97. public:
  98.     std::vector<std::string> _className = {
  99.         "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light",
  100.         "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow",
  101.         "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee",
  102.         "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard",
  103.         "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple",
  104.         "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch",
  105.         "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", "cell phone",
  106.         "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", "scissors", "teddy bear",
  107.         "hair drier", "toothbrush"
  108.     };
  109.  
  110.  
  111.  
  112. };

yolov8_seg_onnx.cpp

  1. #include "yolov8_seg_onnx.h"
  2. using namespace std;
  3. using namespace cv;
  4. using namespace cv::dnn;
  5. using namespace Ort;
  6.  
  7. bool Yolov8SegOnnx::ReadModel(const std::string& modelPath, bool isCuda, int cudaID, bool warmUp) {
  8.     if (_batchSize < 1) _batchSize = 1;
  9.     try
  10.     {
  11.         std::vector<std::string> available_providers = GetAvailableProviders();
  12.         auto cuda_available = std::find(available_providers.begin(), available_providers.end(), "CUDAExecutionProvider");
  13.  
  14.         if (isCuda && (cuda_available == available_providers.end()))
  15.         {
  16.             std::cout << "Your ORT build without GPU. Change to CPU." << std::endl;
  17.             std::cout << "************* Infer model on CPU! *************" << std::endl;
  18.         }
  19.         else if (isCuda && (cuda_available != available_providers.end()))
  20.         {
  21.             std::cout << "************* Infer model on GPU! *************" << std::endl;
  22. //#if ORT_API_VERSION < ORT_OLD_VISON
  23. //			OrtCUDAProviderOptions cudaOption;
  24. //			cudaOption.device_id = cudaID;
  25. //			_OrtSessionOptions.AppendExecutionProvider_CUDA(cudaOption);
  26. //#else
  27. //			OrtStatus* status = OrtSessionOptionsAppendExecutionProvider_CUDA(_OrtSessionOptions, cudaID);
  28. //#endif
  29.         }
  30.         else
  31.         {
  32.             std::cout << "************* Infer model on CPU! *************" << std::endl;
  33.         }
  34.         //
  35.         _OrtSessionOptions.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_EXTENDED);
  36.  
  37. #ifdef _WIN32
  38.         std::wstring model_path(modelPath.begin(), modelPath.end());
  39.         _OrtSession = new Ort::Session(_OrtEnv, model_path.c_str(), _OrtSessionOptions);
  40. #else
  41.         _OrtSession = new Ort::Session(_OrtEnv, modelPath.c_str(), _OrtSessionOptions);
  42. #endif
  43.  
  44.         Ort::AllocatorWithDefaultOptions allocator;
  45.         //init input
  46.         _inputNodesNum = _OrtSession->GetInputCount();
  47. #if ORT_API_VERSION < ORT_OLD_VISON
  48.         _inputName = _OrtSession->GetInputName(0, allocator);
  49.         _inputNodeNames.push_back(_inputName);
  50. #else
  51.         _inputName = std::move(_OrtSession->GetInputNameAllocated(0, allocator));
  52.         _inputNodeNames.push_back(_inputName.get());
  53. #endif
  54.  
  55.         Ort::TypeInfo inputTypeInfo = _OrtSession->GetInputTypeInfo(0);
  56.         auto input_tensor_info = inputTypeInfo.GetTensorTypeAndShapeInfo();
  57.         _inputNodeDataType = input_tensor_info.GetElementType();
  58.         _inputTensorShape = input_tensor_info.GetShape();
  59.  
  60.         if (_inputTensorShape[0] == -1)
  61.         {
  62.             _isDynamicShape = true;
  63.             _inputTensorShape[0] = _batchSize;
  64.  
  65.         }
  66.         if (_inputTensorShape[2] == -1 || _inputTensorShape[3] == -1) {
  67.             _isDynamicShape = true;
  68.             _inputTensorShape[2] = _netHeight;
  69.             _inputTensorShape[3] = _netWidth;
  70.         }
  71.         //init output
  72.         _outputNodesNum = _OrtSession->GetOutputCount();
  73.         if (_outputNodesNum != 2) {
  74.             cout << "This model has " << _outputNodesNum << "output, which is not a segmentation model.Please check your model name or path!" << endl;
  75.             return false;
  76.         }
  77. #if ORT_API_VERSION < ORT_OLD_VISON
  78.         _output_name0 = _OrtSession->GetOutputName(0, allocator);
  79.         _output_name1 = _OrtSession->GetOutputName(1, allocator);
  80. #else
  81.         _output_name0 = std::move(_OrtSession->GetOutputNameAllocated(0, allocator));
  82.         _output_name1 = std::move(_OrtSession->GetOutputNameAllocated(1, allocator));
  83. #endif
  84.         Ort::TypeInfo type_info_output0(nullptr);
  85.         Ort::TypeInfo type_info_output1(nullptr);
  86.         bool flag = false;
  87. #if ORT_API_VERSION < ORT_OLD_VISON
  88.         flag = strcmp(_output_name0, _output_name1) < 0;
  89. #else
  90.         flag = strcmp(_output_name0.get(), _output_name1.get()) < 0;
  91. #endif
  92.         if (flag)  //make sure "output0" is in front of  "output1"
  93.         {
  94.             type_info_output0 = _OrtSession->GetOutputTypeInfo(0);  //output0
  95.             type_info_output1 = _OrtSession->GetOutputTypeInfo(1);  //output1
  96. #if ORT_API_VERSION < ORT_OLD_VISON
  97.             _outputNodeNames.push_back(_output_name0);
  98.             _outputNodeNames.push_back(_output_name1);
  99. #else
  100.             _outputNodeNames.push_back(_output_name0.get());
  101.             _outputNodeNames.push_back(_output_name1.get());
  102. #endif
  103.  
  104.         }
  105.         else {
  106.             type_info_output0 = _OrtSession->GetOutputTypeInfo(1);  //output0
  107.             type_info_output1 = _OrtSession->GetOutputTypeInfo(0);  //output1
  108. #if ORT_API_VERSION < ORT_OLD_VISON
  109.             _outputNodeNames.push_back(_output_name1);
  110.             _outputNodeNames.push_back(_output_name0);
  111. #else
  112.             _outputNodeNames.push_back(_output_name1.get());
  113.             _outputNodeNames.push_back(_output_name0.get());
  114. #endif
  115.         }
  116.  
  117.         auto tensor_info_output0 = type_info_output0.GetTensorTypeAndShapeInfo();
  118.         _outputNodeDataType = tensor_info_output0.GetElementType();
  119.         _outputTensorShape = tensor_info_output0.GetShape();
  120.         auto tensor_info_output1 = type_info_output1.GetTensorTypeAndShapeInfo();
  121.         //_outputMaskNodeDataType = tensor_info_output1.GetElementType(); //the same as output0
  122.         //_outputMaskTensorShape = tensor_info_output1.GetShape();
  123.         //if (_outputTensorShape[0] == -1)
  124.         //{
  125.         //	_outputTensorShape[0] = _batchSize;
  126.         //	_outputMaskTensorShape[0] = _batchSize;
  127.         //}
  128.         //if (_outputMaskTensorShape[2] == -1) {
  129.         //	//size_t ouput_rows = 0;
  130.         //	//for (int i = 0; i < _strideSize; ++i) {
  131.         //	//	ouput_rows += 3 * (_netWidth / _netStride[i]) * _netHeight / _netStride[i];
  132.         //	//}
  133.         //	//_outputTensorShape[1] = ouput_rows;
  134.  
  135.         //	_outputMaskTensorShape[2] = _segHeight;
  136.         //	_outputMaskTensorShape[3] = _segWidth;
  137.         //}
  138.         //warm up
  139.         if (isCuda && warmUp) {
  140.             //draw run
  141.             cout << "Start warming up" << endl;
  142.             size_t input_tensor_length = VectorProduct(_inputTensorShape);
  143.             float* temp = new float[input_tensor_length];
  144.             std::vector<Ort::Value> input_tensors;
  145.             std::vector<Ort::Value> output_tensors;
  146.             input_tensors.push_back(Ort::Value::CreateTensor<float>(
  147.                 _OrtMemoryInfo, temp, input_tensor_length, _inputTensorShape.data(),
  148.                 _inputTensorShape.size()));
  149.             for (int i = 0; i < 3; ++i) {
  150.                 output_tensors = _OrtSession->Run(Ort::RunOptions{ nullptr },
  151.                     _inputNodeNames.data(),
  152.                     input_tensors.data(),
  153.                     _inputNodeNames.size(),
  154.                     _outputNodeNames.data(),
  155.                     _outputNodeNames.size());
  156.             }
  157.  
  158.             delete[]temp;
  159.         }
  160.     }
  161.     catch (const std::exception&) {
  162.         return false;
  163.     }
  164.  
  165. }
  166.  
  167. int Yolov8SegOnnx::Preprocessing(const std::vector<cv::Mat>& srcImgs, std::vector<cv::Mat>& outSrcImgs, std::vector<cv::Vec4d>& params) {
  168.     outSrcImgs.clear();
  169.     Size input_size = Size(_netWidth, _netHeight);
  170.     for (int i = 0; i < srcImgs.size(); ++i) {
  171.         Mat temp_img = srcImgs[i];
  172.         Vec4d temp_param = { 1,1,0,0 };
  173.         if (temp_img.size() != input_size) {
  174.             Mat borderImg;
  175.             LetterBox(temp_img, borderImg, temp_param, input_size, false, false, true, 32);
  176.             //cout << borderImg.size() << endl;
  177.             outSrcImgs.push_back(borderImg);
  178.             params.push_back(temp_param);
  179.         }
  180.         else {
  181.             outSrcImgs.push_back(temp_img);
  182.             params.push_back(temp_param);
  183.         }
  184.     }
  185.  
  186.     int lack_num = srcImgs.size() % _batchSize;
  187.     if (lack_num != 0) {
  188.         for (int i = 0; i < lack_num; ++i) {
  189.             Mat temp_img = Mat::zeros(input_size, CV_8UC3);
  190.             Vec4d temp_param = { 1,1,0,0 };
  191.             outSrcImgs.push_back(temp_img);
  192.             params.push_back(temp_param);
  193.         }
  194.     }
  195.     return 0;
  196.  
  197. }
  198. bool Yolov8SegOnnx::OnnxDetect(cv::Mat& srcImg, std::vector<OutputSeg>& output) {
  199.     std::vector<cv::Mat> input_data = { srcImg };
  200.     std::vector<std::vector<OutputSeg>> tenp_output;
  201.     if (OnnxBatchDetect(input_data, tenp_output)) {
  202.         output = tenp_output[0];
  203.         return true;
  204.     }
  205.     else return false;
  206. }
  207. bool Yolov8SegOnnx::OnnxBatchDetect(std::vector<cv::Mat>& srcImgs, std::vector<std::vector<OutputSeg>>& output) {
  208.     vector<Vec4d> params;
  209.     vector<Mat> input_images;
  210.     cv::Size input_size(_netWidth, _netHeight);
  211.     //preprocessing
  212.     Preprocessing(srcImgs, input_images, params);
  213.     cv::Mat blob = cv::dnn::blobFromImages(input_images, 1 / 255.0, input_size, Scalar(0, 0, 0), true, false);
  214.  
  215.     int64_t input_tensor_length = VectorProduct(_inputTensorShape);
  216.     std::vector<Ort::Value> input_tensors;
  217.     std::vector<Ort::Value> output_tensors;
  218.     input_tensors.push_back(Ort::Value::CreateTensor<float>(_OrtMemoryInfo, (float*)blob.data, input_tensor_length, _inputTensorShape.data(), _inputTensorShape.size()));
  219.  
  220.     output_tensors = _OrtSession->Run(Ort::RunOptions{ nullptr },
  221.         _inputNodeNames.data(),
  222.         input_tensors.data(),
  223.         _inputNodeNames.size(),
  224.         _outputNodeNames.data(),
  225.         _outputNodeNames.size()
  226.     );
  227.  
  228.     //post-process
  229.  
  230.     int net_width = _className.size() + 4 + _segChannels;
  231.     float* all_data = output_tensors[0].GetTensorMutableData<float>();
  232.     _outputTensorShape = output_tensors[0].GetTensorTypeAndShapeInfo().GetShape();
  233.     _outputMaskTensorShape = output_tensors[1].GetTensorTypeAndShapeInfo().GetShape();
  234.     vector<int> mask_protos_shape = { 1,(int)_outputMaskTensorShape[1],(int)_outputMaskTensorShape[2],(int)_outputMaskTensorShape[3] };
  235.     int mask_protos_length = VectorProduct(mask_protos_shape);
  236.     int64_t one_output_length = VectorProduct(_outputTensorShape) / _outputTensorShape[0];
  237.     for (int img_index = 0; img_index < srcImgs.size(); ++img_index) {
  238.         Mat output0 = Mat(Size((int)_outputTensorShape[2], (int)_outputTensorShape[1]), CV_32F, all_data).t();  //[bs,116,8400]=>[bs,8400,116]
  239.         all_data += one_output_length;
  240.         float* pdata = (float*)output0.data;
  241.         int rows = output0.rows;
  242.         std::vector<int> class_ids;//\BD\E1\B9\FBid\CA\FD\D7\E9
  243.         std::vector<float> confidences;//\BD\E1\B9\FBÿ\B8\F6id\B6\D4Ӧ\D6\C3\D0Ŷ\C8\CA\FD\D7\E9
  244.         std::vector<cv::Rect> boxes;//ÿ\B8\F6id\BE\D8\D0ο\F2
  245.         std::vector<vector<float>> picked_proposals;  //output0[:,:, 5 + _className.size():net_width]===> for mask
  246.         for (int r = 0; r < rows; ++r) {    //stride
  247.                 cv::Mat scores(1, _className.size(), CV_32F, pdata +4);
  248.                 Point classIdPoint;
  249.                 double max_class_socre;
  250.                 minMaxLoc(scores, 0, &max_class_socre, 0, &classIdPoint);
  251.                 max_class_socre = (float)max_class_socre;
  252.                 if (max_class_socre >= _classThreshold) {
  253.                     vector<float> temp_proto(pdata + 4 + _className.size(), pdata + net_width);
  254.                     picked_proposals.push_back(temp_proto);
  255.                     //rect [x,y,w,h]
  256.                     float x = (pdata[0] - params[img_index][2]) / params[img_index][0];  //x
  257.                     float y = (pdata[1] - params[img_index][3]) / params[img_index][1];  //y
  258.                     float w = pdata[2] / params[img_index][0];  //w
  259.                     float h = pdata[3] / params[img_index][1];  //h
  260.                     int left = MAX(int(x - 0.5 * w + 0.5), 0);
  261.                     int top = MAX(int(y - 0.5 * h + 0.5), 0);
  262.                     class_ids.push_back(classIdPoint.x);
  263.                     confidences.push_back(max_class_socre );
  264.                     boxes.push_back(Rect(left, top, int(w + 0.5), int(h + 0.5)));
  265.                 }
  266.                 pdata += net_width;//\CF\C2һ\D0\D0
  267.         }
  268.  
  269.         vector<int> nms_result;
  270.         cv::dnn::NMSBoxes(boxes, confidences, _classThreshold, _nmsThreshold, nms_result);
  271.         std::vector<vector<float>> temp_mask_proposals;
  272.         cv::Rect holeImgRect(0, 0, srcImgs[img_index].cols, srcImgs[img_index].rows);
  273.         std::vector<OutputSeg> temp_output;
  274.         for (int i = 0; i < nms_result.size(); ++i) {
  275.             int idx = nms_result[i];
  276.             OutputSeg result;
  277.             result.id = class_ids[idx];
  278.             result.confidence = confidences[idx];
  279.             result.box = boxes[idx] & holeImgRect;
  280.             temp_mask_proposals.push_back(picked_proposals[idx]);
  281.             temp_output.push_back(result);
  282.         }
  283.  
  284.         MaskParams mask_params;
  285.         mask_params.params = params[img_index];
  286.         mask_params.srcImgShape = srcImgs[img_index].size();
  287.         Mat mask_protos = Mat(mask_protos_shape, CV_32F, output_tensors[1].GetTensorMutableData<float>() + img_index * mask_protos_length);
  288.         for (int i = 0; i < temp_mask_proposals.size(); ++i) {
  289.             GetMask2(Mat(temp_mask_proposals[i]).t(), mask_protos, temp_output[i], mask_params);
  290.         }
  291.  
  292.         //******************** ****************
  293.         // \C0ϰ汾\B5ķ\BD\B0\B8\A3\AC\C8\E7\B9\FB\C9\CF\C3\E6\D4ڿ\AA\C6\F4\CE\D2ע\CA͵IJ\BF\B7\D6֮\BA\F3\BB\B9һֱ\B1\A8\B4\ED\A3\AC\BD\A8\D2\E9ʹ\D3\C3\D5\E2\B8\F6\A1\A3
  294.         // If the GetMask2() still reports errors , it is recommended to use GetMask().
  295.         // Mat mask_proposals;
  296.         //for (int i = 0; i < temp_mask_proposals.size(); ++i)
  297.         //	mask_proposals.push_back(Mat(temp_mask_proposals[i]).t());
  298.         //GetMask(mask_proposals, mask_protos, output, mask_params);
  299.         //*****************************************************/
  300.         output.push_back(temp_output);
  301.  
  302.     }
  303.  
  304.     if (output.size())
  305.         return true;
  306.     else
  307.         return false;
  308. }

main.cpp

  1. #include <iostream>
  2. #include <opencv2/opencv.hpp>
  3.  
  4. #include "yolov8_seg_onnx.h"
  5. #include "yolov8_seg_utils.h"
  6. #include <sys/time.h>
  7.  
  8. using namespace std;
  9. using namespace cv;
  10.  
  11. int main()
  12. {
  13.     string model_path = "/home/jason/PycharmProjects/pytorch_learn/yolo/ultralytics-main-yolov8/yolov8n-seg.onnx";
  14.  
  15.     Yolov8SegOnnx test;
  16.  
  17.     if(test.ReadModel(model_path,false,0,false))
  18.         cout << "read net ok!\n";
  19.     else
  20.         cout << "read net err!\n";
  21.  
  22.  
  23.     vector<Scalar> color;
  24.     srand(time(0));
  25.  
  26.     for (int i=0; i<80; i++){
  27.         int b = rand() % 256;
  28.         int g = rand() % 256;
  29.         int r = rand() % 256;
  30.         color.push_back(Scalar(b,g,r));
  31.     }
  32.  
  33.  
  34.  
  35.     struct timeval t1, t2;
  36.     double timeuse;
  37.     VideoCapture capture(0);
  38.     Mat frame;
  39.     vector<OutputSeg> output;
  40.  
  41.  
  42.  
  43.     while(1){
  44.         capture >> frame;
  45.         output.clear();
  46.  
  47.         gettimeofday(&t1, NULL);
  48.         bool find = test.OnnxDetect(frame, output);
  49.         gettimeofday(&t2, NULL);
  50.  
  51.         if (find)
  52.             DrawPred(frame, output, test._className, color);
  53.         else
  54.             cout << "not find!\n";
  55.  
  56.  
  57.         timeuse = (t2.tv_sec - t1.tv_sec) + (double)(t2.tv_usec - t1.tv_usec)/1000000; // s
  58.         timeuse *= 1000; // ms
  59.         string label = "TimeUse: " + to_string(timeuse);
  60.         putText(frame, label, Point(30,30), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0,0,255), 2, 8);
  61.  
  62.         imshow("result", frame);
  63.         if(waitKey(1)=='q') break;
  64.     }
  65.  
  66.     return 0;
  67. }

CMakelist.txt

  1. find_package(OpenCV 4 REQUIRED)
  2. include_directories(/home/jason/下载/onnxruntime-linux-x64-1.14.1/include)
  3. include_directories(./include )
  4. aux_source_directory(./src SOURCES_LIST)
  5.  
  6. add_executable(${PROJECT_NAME} ${SOURCES_LIST})
  7. target_link_libraries(${PROJECT_NAME} ${OpenCV_LIBS} "/home/jason/下载/onnxruntime-linux-x64-1.14.1/lib/libonnxruntime.so")

声明：本文内容由网友自发贡献，不代表【wpsshop博客】立场，版权归原作者所有，本站不承担相应法律责任。如您发现有侵权的内容，请联系我们。转载请注明出处：https://www.wpsshop.cn/article/detail/45466
推荐阅读
相关标签

Copyright © 2003-2013 www.wpsshop.cn 版权所有，并保留所有权利。

  

闽ICP备14008679号