feat: 集成 fastdeploy，启用 paddle2onnx ocr

.gitmodules

@@ -1,9 +1,6 @@
 [submodule "test"]
 	path = test
 	url = https://github.com/MaaAssistantArknights/MaaTestSet.git
-[submodule "3rdparty/VisionOCR"]
-	path = 3rdparty/VisionOCR
-	url = https://github.com/MaaAssistantArknights/VisionOCR.git
 [submodule "src/MeoAsstMac"]
 	path = src/MeoAsstMac
 	url = https://github.com/MaaAssistantArknights/MeoAsstMac.git

3rdparty/include/PaddleOCR/exports.h

@@ -1,37 +0,0 @@
-#pragma once
-
-// The way how the function is called
-#if !defined(OCR_CALL)
-#if defined(_WIN32)
-#define OCR_CALL __stdcall
-#else
-#define OCR_CALL
-#endif /* _WIN32 */
-#endif /* OCR_CALL */
-
-// The function exported symbols
-#if defined _WIN32 || defined __CYGWIN__
-#define OCR_IMPORT __declspec(dllimport)
-#define OCR_EXPORT __declspec(dllexport)
-#define OCR_LOCAL
-#else
-#if __GNUC__ >= 4
-#define OCR_IMPORT __attribute__ ((visibility ("default")))
-#define OCR_EXPORT __attribute__ ((visibility ("default")))
-#define OCR_LOCAL  __attribute__ ((visibility ("hidden")))
-#else
-#define OCR_IMPORT
-#define OCR_EXPORT
-#define OCR_LOCAL
-#endif
-#endif
-
-#ifdef OCR_EXPORTS // defined if we are building the DLL (instead of using it)
-#define OCRAPI_PORT OCR_EXPORT
-#else
-#define OCRAPI_PORT OCR_IMPORT
-#endif // OCR_EXPORTS
-
-#define OCRAPI OCRAPI_PORT OCR_CALL
-
-#define OCRLOCAL OCR_LOCAL OCR_CALL

3rdparty/include/PaddleOCR/paddle_ocr.h

@@ -1,56 +0,0 @@
-#pragma once
-
-#include "exports.h"
-
-struct paddle_ocr_t;
-typedef int OCR_ERROR;
-typedef unsigned char uint8_t;
-
-#define OCR_SUCCESS 0
-#define OCR_FAILURE 1
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-    OCRAPI_PORT paddle_ocr_t* OCR_CALL PaddleOcrCreate(
-        const char* det_model_dir, const char* rec_model_dir,
-        const char* char_list_file, const char* cls_model_dir);
-
-    void OCRAPI PaddleOcrDestroy(paddle_ocr_t* ocr_ptr);
-
-    OCR_ERROR OCRAPI PaddleOcrDet(
-        paddle_ocr_t* ocr_ptr, const uint8_t* encode_buf, size_t encode_buf_size,
-        int* out_boxes, size_t* out_boxes_size,
-        double* out_times, size_t* out_times_size);
-
-    OCR_ERROR OCRAPI PaddleOcrDetWithData(
-        paddle_ocr_t* ocr_ptr, int rows, int cols, int type, void* data,
-        int* out_boxes, size_t* out_boxes_size,
-        double* out_times, size_t* out_times_size);
-
-    OCR_ERROR OCRAPI PaddleOcrRec(
-        paddle_ocr_t* ocr_ptr, const uint8_t* encode_buf, size_t encode_buf_size,
-        char** out_strs, float* out_scores, size_t* out_size,
-        double* out_times, size_t* out_times_size);
-
-    OCR_ERROR OCRAPI PaddleOcrRecWithData(
-        paddle_ocr_t* ocr_ptr, int rows, int cols, int type, void* data,
-        char** out_strs, float* out_scores, size_t* out_size,
-        double* out_times, size_t* out_times_size);
-
-    OCR_ERROR OCRAPI PaddleOcrSystem(
-        paddle_ocr_t* ocr_ptr, const uint8_t* encode_buf, size_t encode_buf_size,
-        bool with_cls,
-        int* out_boxes, char** out_strs, float* out_scores, size_t* out_size,
-        double* out_times, size_t* out_times_size);
-
-    OCR_ERROR OCRAPI PaddleOcrSystemWithData(
-        paddle_ocr_t* ocr_ptr, int rows, int cols, int type, void* data,
-        bool with_cls,
-        int* out_boxes, char** out_strs, float* out_scores, size_t* out_size,
-        double* out_times, size_t* out_times_size);
-
-#ifdef __cplusplus
-}
-#endif

3rdparty/include/fastdeploy/backends/backend.h

@@ -0,0 +1,74 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <iostream>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "fastdeploy/backends/common/multiclass_nms.h"
+#include "fastdeploy/core/fd_tensor.h"
+#include "fastdeploy/core/fd_type.h"
+
+namespace fastdeploy {
+
+/*! @brief Information of Tensor
+ */
+struct TensorInfo {
+  std::string name;        ///< Name of tensor
+  std::vector<int> shape;  ///< Shape of tensor
+  FDDataType dtype;        ///< Data type of tensor
+
+  friend std::ostream& operator<<(std::ostream& output,
+                                  const TensorInfo& info) {
+    output << "TensorInfo(name: " << info.name << ", shape: [";
+    for (size_t i = 0; i < info.shape.size(); ++i) {
+      if (i == info.shape.size() - 1) {
+        output << info.shape[i];
+      } else {
+        output << info.shape[i] << ", ";
+      }
+    }
+    output << "], dtype: " << Str(info.dtype) << ")";
+    return output;
+  }
+};
+
+class BaseBackend {
+ public:
+  bool initialized_ = false;
+
+  BaseBackend() {}
+  virtual ~BaseBackend() = default;
+
+  virtual bool Initialized() const { return initialized_; }
+
+  virtual int NumInputs() const = 0;
+  virtual int NumOutputs() const = 0;
+  virtual TensorInfo GetInputInfo(int index) = 0;
+  virtual TensorInfo GetOutputInfo(int index) = 0;
+  virtual std::vector<TensorInfo> GetInputInfos() = 0;
+  virtual std::vector<TensorInfo> GetOutputInfos() = 0;
+  virtual bool Infer(std::vector<FDTensor>& inputs,
+                     std::vector<FDTensor>* outputs) = 0;
+  virtual std::unique_ptr<BaseBackend> Clone(void *stream = nullptr,
+                                             int device_id = -1) {
+    FDERROR << "Clone no support" << std::endl;
+    return nullptr;
+  }
+};
+
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/backends/common/multiclass_nms.h

@@ -0,0 +1,45 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+#include <map>
+#include <string>
+#include <vector>
+
+namespace fastdeploy {
+namespace backend {
+struct MultiClassNMS {
+  int64_t background_label = -1;
+  int64_t keep_top_k = -1;
+  float nms_eta;
+  float nms_threshold = 0.7;
+  int64_t nms_top_k;
+  bool normalized;
+  float score_threshold;
+
+  std::vector<int32_t> out_num_rois_data;
+  std::vector<int32_t> out_index_data;
+  std::vector<float> out_box_data;
+  void FastNMS(const float* boxes, const float* scores, const int& num_boxes,
+               std::vector<int>* keep_indices);
+  int NMSForEachSample(const float* boxes, const float* scores, int num_boxes,
+                       int num_classes,
+                       std::map<int, std::vector<int>>* keep_indices);
+  void Compute(const float* boxes, const float* scores,
+               const std::vector<int64_t>& boxes_dim,
+               const std::vector<int64_t>& scores_dim);
+};
+}  // namespace backend
+
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/backends/ort/ops/multiclass_nms.h

@@ -0,0 +1,81 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <map>
+
+#ifndef NON_64_PLATFORM
+#include "onnxruntime_cxx_api.h"  // NOLINT
+
+namespace fastdeploy {
+
+struct MultiClassNmsKernel {
+ protected:
+  int64_t background_label = -1;
+  int64_t keep_top_k = -1;
+  float nms_eta;
+  float nms_threshold = 0.7;
+  int64_t nms_top_k;
+  bool normalized;
+  float score_threshold;
+  Ort::CustomOpApi ort_;
+
+ public:
+  MultiClassNmsKernel(Ort::CustomOpApi ort, const OrtKernelInfo* info)
+      : ort_(ort) {
+    GetAttribute(info);
+  }
+
+  void GetAttribute(const OrtKernelInfo* info);
+
+  void Compute(OrtKernelContext* context);
+  void FastNMS(const float* boxes, const float* scores, const int& num_boxes,
+               std::vector<int>* keep_indices);
+  int NMSForEachSample(const float* boxes, const float* scores, int num_boxes,
+                       int num_classes,
+                       std::map<int, std::vector<int>>* keep_indices);
+};
+
+struct MultiClassNmsOp
+    : Ort::CustomOpBase<MultiClassNmsOp, MultiClassNmsKernel> {
+  void* CreateKernel(Ort::CustomOpApi api, const OrtKernelInfo* info) const {
+    return new MultiClassNmsKernel(api, info);
+  }
+
+  const char* GetName() const { return "MultiClassNMS"; }
+
+  size_t GetInputTypeCount() const { return 2; }
+
+  ONNXTensorElementDataType GetInputType(size_t index) const {
+    return ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT;
+  }
+
+  size_t GetOutputTypeCount() const { return 3; }
+
+  ONNXTensorElementDataType GetOutputType(size_t index) const {
+    if (index == 0) {
+      return ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT;
+    }
+    return ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32;
+  }
+
+  const char* GetExecutionProviderType() const {
+    return "CPUExecutionProvider";
+  }
+};
+
+}  // namespace fastdeploy
+
+#endif

3rdparty/include/fastdeploy/backends/ort/ort_backend.h

@@ -0,0 +1,98 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <iostream>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "fastdeploy/backends/backend.h"
+#include "onnxruntime_cxx_api.h"  // NOLINT
+
+namespace fastdeploy {
+
+struct OrtValueInfo {
+  std::string name;
+  std::vector<int64_t> shape;
+  ONNXTensorElementDataType dtype;
+};
+
+struct OrtBackendOption {
+  // -1 means default
+  // 0: ORT_DISABLE_ALL
+  // 1: ORT_ENABLE_BASIC
+  // 2: ORT_ENABLE_EXTENDED
+  // 99: ORT_ENABLE_ALL (enable some custom optimizations e.g bert)
+  int graph_optimization_level = -1;
+  int intra_op_num_threads = -1;
+  int inter_op_num_threads = -1;
+  // 0: ORT_SEQUENTIAL
+  // 1: ORT_PARALLEL
+  int execution_mode = -1;
+  bool use_gpu = false;
+  int gpu_id = 0;
+  void* external_stream_ = nullptr;
+
+  // inside parameter, maybe remove next version
+  bool remove_multiclass_nms_ = false;
+  std::map<std::string, std::string> custom_op_info_;
+};
+
+class OrtBackend : public BaseBackend {
+ public:
+  OrtBackend() {}
+  virtual ~OrtBackend() = default;
+
+  void BuildOption(const OrtBackendOption& option);
+
+  bool InitFromPaddle(const std::string& model_file,
+                      const std::string& params_file,
+                      const OrtBackendOption& option = OrtBackendOption(),
+                      bool verbose = false);
+
+  bool InitFromOnnx(const std::string& model_file,
+                    const OrtBackendOption& option = OrtBackendOption(),
+                    bool from_memory_buffer = false);
+
+  bool Infer(std::vector<FDTensor>& inputs,
+             std::vector<FDTensor>* outputs) override;
+
+  int NumInputs() const override { return inputs_desc_.size(); }
+
+  int NumOutputs() const override { return outputs_desc_.size(); }
+
+  TensorInfo GetInputInfo(int index) override;
+  TensorInfo GetOutputInfo(int index) override;
+  std::vector<TensorInfo> GetInputInfos() override;
+  std::vector<TensorInfo> GetOutputInfos() override;
+  static std::vector<OrtCustomOp*> custom_operators_;
+  void InitCustomOperators();
+
+ private:
+  Ort::Env env_;
+  Ort::Session session_{nullptr};
+  Ort::SessionOptions session_options_;
+  std::shared_ptr<Ort::IoBinding> binding_;
+  std::vector<OrtValueInfo> inputs_desc_;
+  std::vector<OrtValueInfo> outputs_desc_;
+#ifndef NON_64_PLATFORM
+  Ort::CustomOpDomain custom_op_domain_ = Ort::CustomOpDomain("Paddle");
+#endif
+  OrtBackendOption option_;
+  void CopyToCpu(const Ort::Value& value, FDTensor* tensor,
+                 const std::string& name);
+};
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/backends/ort/utils.h

@@ -0,0 +1,39 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <iostream>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "fastdeploy/backends/backend.h"
+#include "onnxruntime_cxx_api.h"  // NOLINT
+
+namespace fastdeploy {
+
+// Convert FDDataType to OrtDataType
+ONNXTensorElementDataType GetOrtDtype(const FDDataType& fd_dtype);
+
+// Convert OrtDataType to FDDataType
+FDDataType GetFdDtype(const ONNXTensorElementDataType& ort_dtype);
+
+// Create Ort::Value
+// is_backend_cuda specify if the onnxruntime use CUDAExectionProvider
+// While is_backend_cuda = true, and tensor.device = Device::GPU
+// Will directly share the cuda data in tensor to OrtValue
+Ort::Value CreateOrtValue(FDTensor& tensor, bool is_backend_cuda = false);
+
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/backends/rknpu/rknpu2/rknpu2_backend.h

@@ -0,0 +1,103 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#pragma once
+
+#include "fastdeploy/backends/backend.h"
+#include "fastdeploy/core/fd_tensor.h"
+#include "rknn_api.h" // NOLINT
+#include "fastdeploy/backends/rknpu/rknpu2/rknpu2_config.h"
+#include <cstring>
+#include <iostream>
+#include <memory>
+#include <string>
+#include <vector>
+
+namespace fastdeploy {
+struct RKNPU2BackendOption {
+  rknpu2::CpuName cpu_name = rknpu2::CpuName::RK3588;
+
+  // The specification of NPU core setting.It has the following choices :
+  // RKNN_NPU_CORE_AUTO : Referring to automatic mode, meaning that it will
+  // select the idle core inside the NPU.
+  // RKNN_NPU_CORE_0 : Running on the NPU0 core
+  // RKNN_NPU_CORE_1: Runing on the NPU1 core
+  // RKNN_NPU_CORE_2: Runing on the NPU2 core
+  // RKNN_NPU_CORE_0_1: Running on both NPU0 and NPU1 core simultaneously.
+  // RKNN_NPU_CORE_0_1_2: Running on both NPU0, NPU1 and NPU2 simultaneously.
+  rknpu2::CoreMask core_mask = rknpu2::CoreMask::RKNN_NPU_CORE_AUTO;
+};
+
+class RKNPU2Backend : public BaseBackend {
+ public:
+  RKNPU2Backend() = default;
+
+  virtual ~RKNPU2Backend();
+
+  // RKNN API
+  bool LoadModel(void* model);
+
+  bool GetSDKAndDeviceVersion();
+
+  bool SetCoreMask(rknpu2::CoreMask& core_mask) const;
+
+  bool GetModelInputOutputInfos();
+
+  // BaseBackend API
+  void BuildOption(const RKNPU2BackendOption& option);
+
+  bool InitFromRKNN(const std::string& model_file,
+                    const RKNPU2BackendOption& option = RKNPU2BackendOption());
+
+  int NumInputs() const override {
+    return static_cast<int>(inputs_desc_.size());
+  }
+
+  int NumOutputs() const override {
+    return static_cast<int>(outputs_desc_.size());
+  }
+
+  TensorInfo GetInputInfo(int index) override;
+  TensorInfo GetOutputInfo(int index) override;
+  std::vector<TensorInfo> GetInputInfos() override;
+  std::vector<TensorInfo> GetOutputInfos() override;
+  bool Infer(std::vector<FDTensor>& inputs,
+             std::vector<FDTensor>* outputs) override;
+
+ private:
+  // The object of rknn context.
+  rknn_context ctx{};
+  // The structure rknn_sdk_version is used to indicate the version
+  // information of the RKNN SDK.
+  rknn_sdk_version sdk_ver{};
+  // The structure rknn_input_output_num represents the number of
+  // input and output Tensor
+  rknn_input_output_num io_num{};
+  std::vector<TensorInfo> inputs_desc_;
+  std::vector<TensorInfo> outputs_desc_;
+
+  rknn_tensor_attr* input_attrs_ = nullptr;
+  rknn_tensor_attr* output_attrs_ = nullptr;
+
+  rknn_tensor_mem** input_mems_;
+  rknn_tensor_mem** output_mems_;
+
+  bool infer_init = false;
+
+  RKNPU2BackendOption option_;
+
+  static void DumpTensorAttr(rknn_tensor_attr& attr);
+  static FDDataType RknnTensorTypeToFDDataType(rknn_tensor_type type);
+  static rknn_tensor_type FDDataTypeToRknnTensorType(FDDataType type);
+};
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/backends/rknpu/rknpu2/rknpu2_config.h

@@ -0,0 +1,37 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#pragma once
+
+namespace fastdeploy {
+namespace rknpu2 {
+typedef enum _rknpu2_cpu_name {
+  RK356X = 0, /* run on RK356X. */
+  RK3588 = 1, /* default,run on RK3588. */
+  UNDEFINED,
+} CpuName;
+
+/*! RKNPU2 core mask for mobile device. */
+typedef enum _rknpu2_core_mask {
+  RKNN_NPU_CORE_AUTO = 0,  //< default, run on NPU core randomly.
+  RKNN_NPU_CORE_0 = 1,    //< run on NPU core 0.
+  RKNN_NPU_CORE_1 = 2,    //< run on NPU core 1.
+  RKNN_NPU_CORE_2 = 4,    //< run on NPU core 2.
+  RKNN_NPU_CORE_0_1 =
+      RKNN_NPU_CORE_0 | RKNN_NPU_CORE_1,  //< run on NPU core 1 and core 2.
+  RKNN_NPU_CORE_0_1_2 =
+      RKNN_NPU_CORE_0_1 | RKNN_NPU_CORE_2,  //< run on NPU core 1 and core 2.
+  RKNN_NPU_CORE_UNDEFINED,
+} CoreMask;
+}  // namespace rknpu2
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/core/allocate.h

@@ -0,0 +1,60 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#pragma once
+
+#include <memory>
+#include <new>
+#include <numeric>
+#include <string>
+#include <vector>
+
+#include "fastdeploy/utils/utils.h"
+
+namespace fastdeploy {
+
+class FASTDEPLOY_DECL FDHostAllocator {
+ public:
+  bool operator()(void** ptr, size_t size) const;
+};
+
+class FASTDEPLOY_DECL FDHostFree {
+ public:
+  void operator()(void* ptr) const;
+};
+
+#ifdef WITH_GPU
+
+class FASTDEPLOY_DECL FDDeviceAllocator {
+ public:
+  bool operator()(void** ptr, size_t size) const;
+};
+
+class FASTDEPLOY_DECL FDDeviceFree {
+ public:
+  void operator()(void* ptr) const;
+};
+
+class FASTDEPLOY_DECL FDDeviceHostAllocator {
+ public:
+  bool operator()(void** ptr, size_t size) const;
+};
+
+class FASTDEPLOY_DECL FDDeviceHostFree {
+ public:
+  void operator()(void* ptr) const;
+};
+
+#endif
+
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/core/config.h

@@ -0,0 +1,72 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#pragma once
+
+#ifndef FASTDEPLOY_LIB
+/* #undef FASTDEPLOY_LIB */
+#endif
+
+#ifndef ENABLE_PADDLE_FRONTEND
+/* #undef ENABLE_PADDLE_FRONTEND */
+#endif
+
+#ifndef ENABLE_ORT_BACKEND
+#define ENABLE_ORT_BACKEND
+#endif
+
+#ifndef ENABLE_PADDLE_BACKEND
+#define ENABLE_PADDLE_BACKEND
+#endif
+
+#ifndef ENABLE_POROS_BACKEND
+/* #undef ENABLE_POROS_BACKEND */
+#endif
+
+#ifndef ENABLE_OPENVINO_BACKEND
+#define ENABLE_OPENVINO_BACKEND
+#endif
+
+#ifndef WITH_GPU
+/* #undef WITH_GPU */
+#endif
+
+#ifndef ENABLE_TRT_BACKEND
+/* #undef ENABLE_TRT_BACKEND */
+#endif
+
+#ifndef ENABLE_VISION
+#define ENABLE_VISION
+#endif
+
+#ifndef ENABLE_FLYCV
+/* #undef ENABLE_FLYCV */
+#endif
+
+#ifndef ENABLE_TEXT
+#define ENABLE_TEXT
+#endif
+
+#ifndef ENABLE_OPENCV_CUDA
+/* #undef ENABLE_OPENCV_CUDA */
+#endif
+
+#ifdef ENABLE_VISION
+#ifndef ENABLE_VISION_VISUALIZE
+#define ENABLE_VISION_VISUALIZE
+#endif
+#endif
+
+#ifndef ENABLE_FDTENSOR_FUNC
+#define ENABLE_FDTENSOR_FUNC
+#endif

3rdparty/include/fastdeploy/core/fd_tensor.h

@@ -0,0 +1,147 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#pragma once
+
+#include <iostream>
+#include <numeric>
+#include <string>
+#include <vector>
+
+#include "fastdeploy/core/allocate.h"
+#include "fastdeploy/core/fd_type.h"
+
+namespace fastdeploy {
+
+struct FASTDEPLOY_DECL FDTensor {
+  // std::vector<int8_t> data;
+  void* buffer_ = nullptr;
+  std::vector<int64_t> shape = {0};
+  std::string name = "";
+  FDDataType dtype = FDDataType::INT8;
+
+  // This use to skip memory copy step
+  // the external_data_ptr will point to the user allocated memory
+  // user has to maintain the memory, allocate and release
+  void* external_data_ptr = nullptr;
+  // The internal data will be on CPU
+  // Some times, the external data is on the GPU, and we are going to use
+  // GPU to inference the model
+  // so we can skip data transfer, which may improve the efficience
+  Device device = Device::CPU;
+  // By default the device id of FDTensor is -1, which means this value is
+  // invalid, and FDTensor is using the same device id as Runtime.
+  int device_id = -1;
+
+  // Whether the data buffer is in pinned memory, which is allocated
+  // with cudaMallocHost()
+  bool is_pinned_memory = false;
+
+  // if the external data is not on CPU, we use this temporary buffer
+  // to transfer data to CPU at some cases we need to visit the
+  // other devices' data
+  std::vector<int8_t> temporary_cpu_buffer;
+
+  // Get data buffer pointer
+  void* MutableData();
+
+  void* Data();
+
+  bool IsShared() {
+    return external_data_ptr != nullptr;
+  }
+
+  void StopSharing();
+
+  const void* Data() const;
+
+  // Use this data to get the tensor data to process
+  // Since the most senario is process data in CPU
+  // this function will return a pointer to cpu memory
+  // buffer.
+  // If the original data is on other device, the data
+  // will copy to cpu store in `temporary_cpu_buffer`
+  const void* CpuData() const;
+
+  // Set user memory buffer for Tensor, the memory is managed by
+  // the user it self, but the Tensor will share the memory with user
+  // So take care with the user buffer
+  void SetExternalData(const std::vector<int64_t>& new_shape,
+                       const FDDataType& data_type, void* data_buffer,
+                       const Device& new_device = Device::CPU);
+
+  // Expand the shape of a Tensor. Insert a new axis that will appear
+  // at the `axis` position in the expanded Tensor shape.
+  void ExpandDim(int64_t axis = 0);
+
+  // Squeeze the shape of a Tensor. Erase the axis that will appear
+  // at the `axis` position in the squeezed Tensor shape.
+  void Squeeze(int64_t axis = 0);
+
+  // Initialize Tensor
+  // Include setting attribute for tensor
+  // and allocate cpu memory buffer
+  void Allocate(const std::vector<int64_t>& new_shape,
+                const FDDataType& data_type,
+                const std::string& tensor_name = "",
+                const Device& new_device = Device::CPU);
+
+  // Total size of tensor memory buffer in bytes
+  int Nbytes() const;
+
+  // Total number of elements in this tensor
+  int Numel() const;
+
+  // Get shape of FDTensor
+  std::vector<int64_t> Shape() const { return shape; }
+
+  // Get dtype of FDTensor
+  FDDataType Dtype() const { return dtype; }
+
+  void Resize(size_t nbytes);
+
+  void Resize(const std::vector<int64_t>& new_shape);
+
+  void Resize(const std::vector<int64_t>& new_shape,
+              const FDDataType& data_type, const std::string& tensor_name = "",
+              const Device& new_device = Device::CPU);
+
+  // Debug function
+  // Use this function to print shape, dtype, mean, max, min
+  // prefix will also be printed as tag
+  void PrintInfo(const std::string& prefix = "TensorInfo: ") const;
+
+  bool ReallocFn(size_t nbytes);
+
+  void FreeFn();
+
+  FDTensor() {}
+  explicit FDTensor(const std::string& tensor_name);
+  // Deep copy
+  FDTensor(const FDTensor& other);
+  // Move constructor
+  FDTensor(FDTensor&& other);
+
+  // Deep copy assignment
+  FDTensor& operator=(const FDTensor& other);
+  // Move assignment
+  FDTensor& operator=(FDTensor&& other);
+
+  ~FDTensor() { FreeFn(); }
+
+  static void CopyBuffer(void* dst, const void* src, size_t nbytes,
+                         const Device& device = Device::CPU,
+                        bool is_pinned_memory = false);
+};
+
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/core/fd_type.h

@@ -0,0 +1,80 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#pragma once
+
+#include <ostream>
+#include <sstream>
+#include <string>
+
+#include "fastdeploy/core/config.h"
+#include "fastdeploy/utils/utils.h"
+
+namespace fastdeploy {
+
+enum FASTDEPLOY_DECL Device { CPU, GPU, RKNPU, IPU, TIMVX};
+
+FASTDEPLOY_DECL std::string Str(const Device& d);
+
+enum FASTDEPLOY_DECL FDDataType {
+  BOOL,
+  INT16,
+  INT32,
+  INT64,
+  FP16,
+  FP32,
+  FP64,
+  UNKNOWN1,
+  UNKNOWN2,
+  UNKNOWN3,
+  UNKNOWN4,
+  UNKNOWN5,
+  UNKNOWN6,
+  UNKNOWN7,
+  UNKNOWN8,
+  UNKNOWN9,
+  UNKNOWN10,
+  UNKNOWN11,
+  UNKNOWN12,
+  UNKNOWN13,
+  UINT8,
+  INT8
+};
+
+FASTDEPLOY_DECL std::ostream& operator<<(std::ostream& out, const Device& d);
+
+FASTDEPLOY_DECL std::ostream& operator<<(std::ostream& out,
+                                         const FDDataType& fdt);
+
+FASTDEPLOY_DECL std::string Str(const FDDataType& fdt);
+
+FASTDEPLOY_DECL int32_t FDDataTypeSize(const FDDataType& data_dtype);
+
+template <typename PlainType>
+struct FASTDEPLOY_DECL TypeToDataType {
+  static const FDDataType dtype;
+};
+
+/*! Deep learning model format */
+enum ModelFormat {
+  AUTOREC,      ///< Auto recognize the model format by model file name
+  PADDLE,       ///< Model with paddlepaddle format
+  ONNX,         ///< Model with ONNX format
+  RKNN,         ///< Model with RKNN format
+  TORCHSCRIPT,  ///< Model with TorchScript format
+};
+
+FASTDEPLOY_DECL std::ostream& operator<<(std::ostream& out,
+                                         const ModelFormat& format);
+
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/core/float16.h

@@ -0,0 +1,665 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <stdint.h>
+
+#include <cmath>
+#include <iostream>
+#include <limits>
+
+#if !defined(_WIN32)
+#define FD_ALIGN(x) __attribute__((aligned(x)))
+#else
+#define FD_ALIGN(x) __declspec(align(x))
+#endif
+
+namespace fastdeploy {
+
+struct FD_ALIGN(2) float16 {
+ public:
+  uint16_t x;
+
+  // The following defaulted special class member functions
+  // are added to make float16 pass the std::is_trivial test
+  float16() = default;
+  float16(const float16& o) = default;
+  float16& operator=(const float16& o) = default;
+  float16(float16&& o) = default;
+  float16& operator=(float16&& o) = default;
+  ~float16() = default;
+
+// Constructors
+
+#ifdef FD_WITH_NATIVE_FP16
+  // __fp16 is a native half precision data type for arm cpu,
+  // float16_t is an alias for __fp16
+  inline explicit float16(const float16_t& h) {
+    x = *reinterpret_cast<const uint16_t*>(&h);
+  }
+#endif
+
+  inline explicit float16(float val) {
+#if defined(FD_WITH_NATIVE_FP16)
+    float32x4_t tmp = vld1q_dup_f32(&val);
+    float16_t res = vget_lane_f16(vcvt_f16_f32(tmp), 0);
+    x = *reinterpret_cast<uint16_t*>(&res);
+
+#elif defined(__F16C__)
+    x = _cvtss_sh(val, 0);
+
+#else
+    // Conversion routine adapted from
+    // http://stackoverflow.com/questions/1659440/32-bit-to-16-bit-floating-point-conversion
+    Bits v, s;
+    v.f = val;
+    uint32_t sign = v.si & sigN;
+    v.si ^= sign;
+    sign >>= shiftSign;  // logical shift
+    s.si = mulN;
+    s.si = s.f * v.f;  // correct subnormals
+    v.si ^= (s.si ^ v.si) & -(minN > v.si);
+    v.si ^= (infN ^ v.si) & -((infN > v.si) & (v.si > maxN));
+    v.si ^= (nanN ^ v.si) & -((nanN > v.si) & (v.si > infN));
+    v.ui >>= shift;  // logical shift
+    v.si ^= ((v.si - maxD) ^ v.si) & -(v.si > maxC);
+    v.si ^= ((v.si - minD) ^ v.si) & -(v.si > subC);
+    x = v.ui | sign;
+
+#endif
+  }
+
+  inline explicit float16(bool b) : x(b ? 0x3c00 : 0) {}
+
+  template <class T>
+  inline explicit float16(const T& val)
+      : x(float16(static_cast<float>(val)).x) {}
+
+// Assignment operators
+
+#ifdef FD_WITH_NATIVE_FP16
+  inline float16& operator=(const float16_t& rhs) {
+    x = *reinterpret_cast<const uint16_t*>(&rhs);
+    return *this;
+  }
+#endif
+
+  inline float16& operator=(bool b) {
+    x = b ? 0x3c00 : 0;
+    return *this;
+  }
+
+  inline float16& operator=(int8_t val) {
+    x = float16(val).x;
+    return *this;
+  }
+
+  inline float16& operator=(uint8_t val) {
+    x = float16(val).x;
+    return *this;
+  }
+
+  inline float16& operator=(int16_t val) {
+    x = float16(val).x;
+    return *this;
+  }
+
+  inline float16& operator=(uint16_t val) {
+    x = float16(val).x;
+    return *this;
+  }
+
+  inline float16& operator=(int32_t val) {
+    x = float16(val).x;
+    return *this;
+  }
+
+  inline float16& operator=(uint32_t val) {
+    x = float16(val).x;
+    return *this;
+  }
+
+  inline float16& operator=(int64_t val) {
+    x = float16(val).x;
+    return *this;
+  }
+
+  inline float16& operator=(uint64_t val) {
+    x = float16(val).x;
+    return *this;
+  }
+
+  inline float16& operator=(float val) {
+    x = float16(val).x;
+    return *this;
+  }
+
+  inline float16& operator=(double val) {
+    x = float16(val).x;
+    return *this;
+  }
+
+// Conversion opertors
+#ifdef FD_WITH_NATIVE_FP16
+  HOSTDEVICE inline explicit operator float16_t() const {
+    return *reinterpret_cast<const float16_t*>(this);
+  }
+#endif
+
+  inline operator float() const {
+#if defined(FD_WITH_NATIVE_FP16)
+    float16x4_t res = vld1_dup_f16(reinterpret_cast<const float16_t*>(this));
+    return vgetq_lane_f32(vcvt_f32_f16(res), 0);
+
+#elif defined(__F16C__)
+    return _cvtsh_ss(this->x);
+
+#else
+    // Conversion routine adapted from
+    // http://stackoverflow.com/questions/1659440/32-bit-to-16-bit-floating-point-conversion
+    Bits v;
+    v.ui = this->x;
+    int32_t sign = v.si & sigC;
+    v.si ^= sign;
+    sign <<= shiftSign;
+    v.si ^= ((v.si + minD) ^ v.si) & -(v.si > subC);
+    v.si ^= ((v.si + maxD) ^ v.si) & -(v.si > maxC);
+    Bits s;
+    s.si = mulC;
+    s.f *= v.si;
+    int32_t mask = -(norC > v.si);
+    v.si <<= shift;
+    v.si ^= (s.si ^ v.si) & mask;
+    v.si |= sign;
+    return v.f;
+
+#endif
+  }
+
+  inline explicit operator bool() const { return (x & 0x7fff) != 0; }
+
+  inline explicit operator int8_t() const {
+    return static_cast<int8_t>(static_cast<float>(*this));
+  }
+
+  inline explicit operator uint8_t() const {
+    return static_cast<uint8_t>(static_cast<float>(*this));
+  }
+
+  inline explicit operator int16_t() const {
+    return static_cast<int16_t>(static_cast<float>(*this));
+  }
+
+  inline explicit operator uint16_t() const {
+    return static_cast<uint16_t>(static_cast<float>(*this));
+  }
+
+  inline explicit operator int32_t() const {
+    return static_cast<int32_t>(static_cast<float>(*this));
+  }
+
+  inline explicit operator uint32_t() const {
+    return static_cast<uint32_t>(static_cast<float>(*this));
+  }
+
+  inline explicit operator int64_t() const {
+    return static_cast<int64_t>(static_cast<float>(*this));
+  }
+
+  inline explicit operator uint64_t() const {
+    return static_cast<uint64_t>(static_cast<float>(*this));
+  }
+
+  inline operator double() const {
+    return static_cast<double>(static_cast<float>(*this));
+  }
+
+  inline bool operator>(const float& other) const {
+    return this->operator float() > other;
+  }
+
+  inline bool operator>(const double& other) const {
+    return this->operator double() > other;
+  }
+
+  inline bool operator<(const float& other) const {
+    return this->operator float() > other;
+  }
+
+  inline bool operator<(const double& other) const {
+    return this->operator double() > other;
+  }
+
+  template <typename T,
+            typename std::enable_if<!std::is_same<T, float16>::value,
+                                    bool>::type = true>
+  inline float16& operator+=(const T& other) {
+    *this = float16(static_cast<T>(*this) + other);
+    return *this;
+  }
+
+ private:
+  union Bits {
+    float f;
+    int32_t si;
+    uint32_t ui;
+  };
+
+  static const int shift = 13;
+  static const int shiftSign = 16;
+
+  static const int32_t infN = 0x7F800000;
+  static const int32_t maxN = 0x477FE000;  // max flt16 as flt32
+  static const int32_t minN = 0x38800000;  // min flt16 normal as flt32
+  static const int32_t sigN = 0x80000000;  // sign bit
+
+  static constexpr int32_t infC = infN >> shift;
+  static constexpr int32_t nanN = (infC + 1)
+                                  << shift;  // minimum flt16 nan as float32
+  static constexpr int32_t maxC = maxN >> shift;
+  static constexpr int32_t minC = minN >> shift;
+  static constexpr int32_t sigC = sigN >> shiftSign;
+
+  static const int32_t mulN = 0x52000000;  // (1 << 23) / minN
+  static const int32_t mulC = 0x33800000;  // minN / (1 << (23 - shift))
+  static const int32_t subC = 0x003FF;     // max flt32 subnormal downshifted
+  static const int32_t norC = 0x00400;     // min flt32 normal downshifted
+
+  static constexpr int32_t maxD = infC - maxC - 1;
+  static constexpr int32_t minD = minC - subC - 1;
+};
+
+// Arithmetic operators for float16 on ARMv8.2-A CPU
+#if defined(FD_WITH_NATIVE_FP16)
+inline float16 operator+(const float16& a, const float16& b) {
+  float16 res;
+  asm volatile(
+      "ld1 {v0.h}[0], [%[a_ptr]]\n"
+      "ld1 {v1.h}[0], [%[b_ptr]]\n"
+      "fadd h0, h0, h1\n"
+      "st1 {v0.h}[0], [%[res_ptr]]\n"
+      :  // outputs
+      :  // inputs
+      [a_ptr] "r"(&(a.x)), [b_ptr] "r"(&(b.x)),
+      [res_ptr] "r"(&(res.x))
+      :  // clobbers
+      "memory", "v0", "v1");
+  return res;
+}
+
+inline float16 operator-(const float16& a, const float16& b) {
+  float16 res;
+  asm volatile(
+      "ld1 {v0.h}[0], [%[a_ptr]]\n"
+      "ld1 {v1.h}[0], [%[b_ptr]]\n"
+      "fsub h0, h0, h1\n"
+      "st1 {v0.h}[0], [%[res_ptr]]\n"
+      :  // outputs
+      :  // inputs
+      [a_ptr] "r"(&(a.x)), [b_ptr] "r"(&(b.x)),
+      [res_ptr] "r"(&(res.x))
+      :  // clobbers
+      "memory", "v0", "v1");
+  return res;
+}
+
+inline float16 operator*(const float16& a, const float16& b) {
+  float16 res;
+  asm volatile(
+      "ld1 {v0.h}[0], [%[a_ptr]]\n"
+      "ld1 {v1.h}[0], [%[b_ptr]]\n"
+      "fmul h0, h0, h1\n"
+      "st1 {v0.h}[0], [%[res_ptr]]\n"
+      :  // outputs
+      :  // inputs
+      [a_ptr] "r"(&(a.x)), [b_ptr] "r"(&(b.x)),
+      [res_ptr] "r"(&(res.x))
+      :  // clobbers
+      "memory", "v0", "v1");
+  return res;
+}
+
+inline float16 operator/(const float16& a, const float16& b) {
+  float16 res;
+  asm volatile(
+      "ld1 {v0.h}[0], [%[a_ptr]]\n"
+      "ld1 {v1.h}[0], [%[b_ptr]]\n"
+      "fdiv h0, h0, h1\n"
+      "st1 {v0.h}[0], [%[res_ptr]]\n"
+      :  // outputs
+      :  // inputs
+      [a_ptr] "r"(&(a.x)), [b_ptr] "r"(&(b.x)),
+      [res_ptr] "r"(&(res.x))
+      :  // clobbers
+      "memory", "v0", "v1");
+  return res;
+}
+
+inline float16 operator-(const float16& a) {
+  float16 res;
+  asm volatile(
+      "ld1 {v0.h}[0], [%[a_ptr]]\n"
+      "fneg h0, h0\n"
+      "st1 {v0.h}[0], [%[res_ptr]]\n"
+      :  // outputs
+      :  // inputs
+      [a_ptr] "r"(&(a.x)),
+      [res_ptr] "r"(&(res.x))
+      :  // clobbers
+      "memory", "v0");
+  return res;
+}
+
+inline float16& operator+=(float16& a, const float16& b) {  // NOLINT
+  a = a + b;
+  return a;
+}
+
+inline float16& operator-=(float16& a, const float16& b) {  // NOLINT
+  a = a - b;
+  return a;
+}
+
+inline float16& operator*=(float16& a, const float16& b) {  // NOLINT
+  a = a * b;
+  return a;
+}
+
+inline float16& operator/=(float16& a, const float16& b) {  // NOLINT
+  a = a / b;
+  return a;
+}
+
+inline bool operator==(const float16& a, const float16& b) {
+  uint16_t res;
+  asm volatile(
+      "ld1 {v0.h}[0], [%[a_ptr]]\n"
+      "ld1 {v1.h}[0], [%[b_ptr]]\n"
+      "fcmeq h0, h0, h1\n"
+      "st1 {v0.h}[0], [%[res_ptr]]\n"
+      :  // outputs
+      :  // inputs
+      [a_ptr] "r"(&(a.x)), [b_ptr] "r"(&(b.x)),
+      [res_ptr] "r"(&res)
+      :  // clobbers
+      "memory", "v0", "v1");
+  return (res & 0xffff) != 0;
+}
+
+inline bool operator!=(const float16& a, const float16& b) { return !(a == b); }
+
+inline bool operator<(const float16& a, const float16& b) {
+  uint16_t res;
+  asm volatile(
+      "ld1 {v1.h}[0], [%[a_ptr]]\n"
+      "ld1 {v0.h}[0], [%[b_ptr]]\n"
+      "fcmgt h0, h0, h1\n"
+      "st1 {v0.h}[0], [%[res_ptr]]\n"
+      :  // outputs
+      :  // inputs
+      [a_ptr] "r"(&(a.x)), [b_ptr] "r"(&(b.x)),
+      [res_ptr] "r"(&res)
+      :  // clobbers
+      "memory", "v0", "v1");
+  return (res & 0xffff) != 0;
+}
+
+inline bool operator<=(const float16& a, const float16& b) {
+  uint16_t res;
+  asm volatile(
+      "ld1 {v1.h}[0], [%[a_ptr]]\n"
+      "ld1 {v0.h}[0], [%[b_ptr]]\n"
+      "fcmge h0, h0, h1\n"
+      "st1 {v0.h}[0], [%[res_ptr]]\n"
+      :  // outputs
+      :  // inputs
+      [a_ptr] "r"(&(a.x)), [b_ptr] "r"(&(b.x)),
+      [res_ptr] "r"(&res)
+      :  // clobbers
+      "memory", "v0", "v1");
+  return (res & 0xffff) != 0;
+}
+
+inline bool operator>(const float16& a, const float16& b) {
+  uint16_t res;
+  asm volatile(
+      "ld1 {v0.h}[0], [%[a_ptr]]\n"
+      "ld1 {v1.h}[0], [%[b_ptr]]\n"
+      "fcmgt h0, h0, h1\n"
+      "st1 {v0.h}[0], [%[res_ptr]]\n"
+      :  // outputs
+      :  // inputs
+      [a_ptr] "r"(&(a.x)), [b_ptr] "r"(&(b.x)),
+      [res_ptr] "r"(&res)
+      :  // clobbers
+      "memory", "v0", "v1");
+  return (res & 0xffff) != 0;
+}
+
+inline bool operator>=(const float16& a, const float16& b) {
+  uint16_t res;
+  asm volatile(
+      "ld1 {v0.h}[0], [%[a_ptr]]\n"
+      "ld1 {v1.h}[0], [%[b_ptr]]\n"
+      "fcmge h0, h0, h1\n"
+      "st1 {v0.h}[0], [%[res_ptr]]\n"
+      :  // outputs
+      :  // inputs
+      [a_ptr] "r"(&(a.x)), [b_ptr] "r"(&(b.x)),
+      [res_ptr] "r"(&res)
+      :  // clobbers
+      "memory", "v0", "v1");
+  return (res & 0xffff) != 0;
+#else
+inline float16 operator+(const float16& a, const float16& b) {
+  return float16(static_cast<float>(a) + static_cast<float>(b));
+}
+
+inline float16 operator-(const float16& a, const float16& b) {
+  return float16(static_cast<float>(a) - static_cast<float>(b));
+}
+
+inline float16 operator*(const float16& a, const float16& b) {
+  return float16(static_cast<float>(a) * static_cast<float>(b));
+}
+
+inline float16 operator/(const float16& a, const float16& b) {
+  return float16(static_cast<float>(a) / static_cast<float>(b));
+}
+
+inline float16 operator-(const float16& a) {
+  float16 res;
+  res.x = a.x ^ 0x8000;
+  return res;
+}
+
+inline float16& operator+=(float16& a, const float16& b) {  // NOLINT
+  a = float16(static_cast<float>(a) + static_cast<float>(b));
+  return a;
+}
+
+inline float16& operator-=(float16& a, const float16& b) {  // NOLINT
+  a = float16(static_cast<float>(a) - static_cast<float>(b));
+  return a;
+}
+
+inline float16& operator*=(float16& a, const float16& b) {  // NOLINT
+  a = float16(static_cast<float>(a) * static_cast<float>(b));
+  return a;
+}
+
+inline float16& operator/=(float16& a, const float16& b) {  // NOLINT
+  a = float16(static_cast<float>(a) / static_cast<float>(b));
+  return a;
+}
+
+inline bool operator==(const float16& a, const float16& b) {
+  return static_cast<float>(a) == static_cast<float>(b);
+}
+
+inline bool operator!=(const float16& a, const float16& b) {
+  return static_cast<float>(a) != static_cast<float>(b);
+}
+
+inline bool operator<(const float16& a, const float16& b) {
+  return static_cast<float>(a) < static_cast<float>(b);
+}
+
+inline bool operator<=(const float16& a, const float16& b) {
+  return static_cast<float>(a) <= static_cast<float>(b);
+}
+
+inline bool operator>(const float16& a, const float16& b) {
+  return static_cast<float>(a) > static_cast<float>(b);
+}
+
+inline bool operator>=(const float16& a, const float16& b) {
+  return static_cast<float>(a) >= static_cast<float>(b);
+}
+#endif
+
+  template <typename T,
+            typename std::enable_if<std::is_integral<T>::value ||
+                                        std::is_same<T, float>::value,
+                                    bool>::type = true>
+  inline T& operator+=(T& a, const float16& b) {  // NOLINT
+    auto c = static_cast<float>(a) + static_cast<float>(b);
+    a = static_cast<T>(c);
+    return a;
+  }
+
+  inline double& operator+=(double& a, const float16& b) {  // NOLINT
+    a = a + static_cast<double>(b);
+    return a;
+  }
+
+  inline float16 raw_uint16_to_float16(uint16_t a) {
+    float16 res;
+    res.x = a;
+    return res;
+  }
+
+  inline bool(isnan)(const float16& a) { return (a.x & 0x7fff) > 0x7c00; }
+
+  inline bool(isinf)(const float16& a) { return (a.x & 0x7fff) == 0x7c00; }
+
+  inline bool(isfinite)(const float16& a) {
+    return !((isnan)(a)) && !((isinf)(a));
+  }
+
+  inline float16(abs)(const float16& a) {
+    return float16(std::abs(static_cast<float>(a)));
+  }
+
+  inline std::ostream& operator<<(std::ostream& os, const float16& a) {
+    os << static_cast<float>(a);
+    return os;
+  }
+}  // namespace fastdeploy
+
+namespace std {
+
+// Override the std::is_pod::value for float16
+// The reason is that different compilers implemented std::is_pod based on
+// different C++ standards. float16 class is a plain old data in C++11 given
+// that it is both trivial and standard_layout.
+// However, std::is_pod in nvcc 8.0 host c++ compiler follows C++0x and is
+// more restricted in that you cannot provide any customized
+// constructor in float16. Hence, we override is_pod here following C++11
+// so that .cu files can be successfully compiled by nvcc.
+template <>
+struct is_pod<fastdeploy::float16> {
+  static const bool value = is_trivial<fastdeploy::float16>::value &&
+                            is_standard_layout<fastdeploy::float16>::value;
+};
+
+template <>
+struct is_floating_point<fastdeploy::float16>
+    : std::integral_constant<
+          bool, std::is_same<fastdeploy::float16,
+                             typename std::remove_cv<
+                                 fastdeploy::float16>::type>::value> {};
+template <>
+struct is_signed<fastdeploy::float16> {
+  static const bool value = true;
+};
+
+template <>
+struct is_unsigned<fastdeploy::float16> {
+  static const bool value = false;
+};
+
+inline bool isnan(const fastdeploy::float16& a) { return fastdeploy::isnan(a); }
+
+inline bool isinf(const fastdeploy::float16& a) { return fastdeploy::isinf(a); }
+
+template <>
+struct numeric_limits<fastdeploy::float16> {
+  static const bool is_specialized = true;
+  static const bool is_signed = true;
+  static const bool is_integer = false;
+  static const bool is_exact = false;
+  static const bool has_infinity = true;
+  static const bool has_quiet_NaN = true;
+  static const bool has_signaling_NaN = true;
+  static const float_denorm_style has_denorm = denorm_present;
+  static const bool has_denorm_loss = false;
+  static const std::float_round_style round_style = std::round_to_nearest;
+  static const bool is_iec559 = false;
+  static const bool is_bounded = false;
+  static const bool is_modulo = false;
+  static const int digits = 11;
+  static const int digits10 = 3;
+  static const int max_digits10 = 5;
+  static const int radix = 2;
+  static const int min_exponent = -13;
+  static const int min_exponent10 = -4;
+  static const int max_exponent = 16;
+  static const int max_exponent10 = 4;
+  static const bool traps = true;
+  static const bool tinyness_before = false;
+
+  static fastdeploy::float16(min)() {
+    return fastdeploy::raw_uint16_to_float16(0x400);
+  }
+  static fastdeploy::float16 lowest() {
+    return fastdeploy::raw_uint16_to_float16(0xfbff);
+  }
+  static fastdeploy::float16(max)() {
+    return fastdeploy::raw_uint16_to_float16(0x7bff);
+  }
+  static fastdeploy::float16 epsilon() {
+    return fastdeploy::raw_uint16_to_float16(0x0800);
+  }
+  static fastdeploy::float16 round_error() { return fastdeploy::float16(0.5); }
+  static fastdeploy::float16 infinity() {
+    return fastdeploy::raw_uint16_to_float16(0x7c00);
+  }
+  static fastdeploy::float16 quiet_NaN() {
+    return fastdeploy::raw_uint16_to_float16(0x7e00);
+  }
+  static fastdeploy::float16 signaling_NaN() {
+    return fastdeploy::raw_uint16_to_float16(0x7e00);
+  }
+  static fastdeploy::float16 denorm_min() {
+    return fastdeploy::raw_uint16_to_float16(0x1);
+  }
+};
+
+inline fastdeploy::float16 abs(const fastdeploy::float16& a) {
+  return fastdeploy::abs(a);
+}
+
+}  // namespace std

3rdparty/include/fastdeploy/fastdeploy_model.h

@@ -0,0 +1,142 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#pragma once
+#include "fastdeploy/runtime.h"
+
+namespace fastdeploy {
+
+/*! @brief Base model object for all the vision models
+ */
+class FASTDEPLOY_DECL FastDeployModel {
+ public:
+  /// Get model's name
+  virtual std::string ModelName() const { return "NameUndefined"; }
+
+  /** \brief Inference the model by the runtime. This interface is included in the `Predict()` function, so we don't call `Infer()` directly in most common situation
+  */
+  virtual bool Infer(std::vector<FDTensor>& input_tensors,
+                     std::vector<FDTensor>* output_tensors);
+
+  /** \brief Inference the model by the runtime. This interface is using class member reused_input_tensors_ to do inference and writing results to reused_output_tensors_
+  */
+  virtual bool Infer();
+
+  RuntimeOption runtime_option;
+  /** \brief Model's valid cpu backends. This member defined all the cpu backends have successfully tested for the model
+   */
+  std::vector<Backend> valid_cpu_backends = {Backend::ORT};
+  /** Model's valid gpu backends. This member defined all the gpu backends have successfully tested for the model
+   */
+  std::vector<Backend> valid_gpu_backends = {Backend::ORT};
+  /** Model's valid ipu backends. This member defined all the ipu backends have successfully tested for the model
+   */
+  std::vector<Backend> valid_ipu_backends = {};
+  /** Model's valid timvx backends. This member defined all the timvx backends have successfully tested for the model
+   */
+  std::vector<Backend> valid_timvx_backends = {};
+  /** Model's valid hardware backends. This member defined all the gpu backends have successfully tested for the model
+   */
+  std::vector<Backend> valid_rknpu_backends = {};
+
+  /// Get number of inputs for this model
+  virtual int NumInputsOfRuntime() { return runtime_->NumInputs(); }
+  /// Get number of outputs for this model
+  virtual int NumOutputsOfRuntime() { return runtime_->NumOutputs(); }
+  /// Get input information for this model
+  virtual TensorInfo InputInfoOfRuntime(int index) {
+    return runtime_->GetInputInfo(index);
+  }
+  /// Get output information for this model
+  virtual TensorInfo OutputInfoOfRuntime(int index) {
+    return runtime_->GetOutputInfo(index);
+  }
+  /// Check if the model is initialized successfully
+  virtual bool Initialized() const {
+    return runtime_initialized_ && initialized;
+  }
+
+  /** \brief This is a debug interface, used to record the time of backend runtime
+   *
+   * example code @code
+   * auto model = fastdeploy::vision::PPYOLOE("model.pdmodel", "model.pdiparams", "infer_cfg.yml");
+   * if (!model.Initialized()) {
+   *   std::cerr << "Failed to initialize." << std::endl;
+   *   return -1;
+   * }
+   * model.EnableRecordTimeOfRuntime();
+   * cv::Mat im = cv::imread("test.jpg");
+   * for (auto i = 0; i < 1000; ++i) {
+   *   fastdeploy::vision::DetectionResult result;
+   *   model.Predict(&im, &result);
+   * }
+   * model.PrintStatisInfoOfRuntime();
+   * @endcode After called the `PrintStatisInfoOfRuntime()`, the statistical information of runtime will be printed in the console
+   */
+  virtual void EnableRecordTimeOfRuntime() {
+    time_of_runtime_.clear();
+    std::vector<double>().swap(time_of_runtime_);
+    enable_record_time_of_runtime_ = true;
+  }
+
+  /** \brief Disable to record the time of backend runtime, see `EnableRecordTimeOfRuntime()` for more detail
+  */
+  virtual void DisableRecordTimeOfRuntime() {
+    enable_record_time_of_runtime_ = false;
+  }
+
+  /** \brief Print the statistic information of runtime in the console, see function `EnableRecordTimeOfRuntime()` for more detail
+  */
+  virtual std::map<std::string, float> PrintStatisInfoOfRuntime();
+
+  /** \brief Check if the `EnableRecordTimeOfRuntime()` method is enabled.
+  */
+  virtual bool EnabledRecordTimeOfRuntime() {
+    return enable_record_time_of_runtime_;
+  }
+
+  /** \brief Release reused input/output buffers
+  */
+  virtual void ReleaseReusedBuffer() {
+    std::vector<FDTensor>().swap(reused_input_tensors_);
+    std::vector<FDTensor>().swap(reused_output_tensors_);
+  }
+
+ protected:
+  virtual bool InitRuntime();
+
+  bool initialized = false;
+  // Reused input tensors
+  std::vector<FDTensor> reused_input_tensors_;
+  // Reused output tensors
+  std::vector<FDTensor> reused_output_tensors_;
+
+ private:
+  bool InitRuntimeWithSpecifiedBackend();
+  bool InitRuntimeWithSpecifiedDevice();
+  bool CreateCpuBackend();
+  bool CreateGpuBackend();
+  bool CreateIpuBackend();
+  bool CreateRKNPUBackend();
+  bool CreateTimVXBackend();
+
+  std::shared_ptr<Runtime> runtime_;
+  bool runtime_initialized_ = false;
+  // whether to record inference time
+  bool enable_record_time_of_runtime_ = false;
+
+  // record inference time for backend
+  std::vector<double> time_of_runtime_;
+};
+
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/function/concat.h

@@ -0,0 +1,32 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "fastdeploy/core/fd_tensor.h"
+
+namespace fastdeploy {
+namespace function {
+
+/** Excute the concatenate operation for input FDTensor along given axis.
+    @param x The input tensor.
+    @param out The output tensor which stores the result.
+    @param axisi Axis which will be concatenated.
+*/
+
+FASTDEPLOY_DECL void Concat(const std::vector<FDTensor>& x, FDTensor* out,
+                            int axis = 0);
+
+}  // namespace function
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/function/cuda_cast.h

@@ -0,0 +1,29 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "fastdeploy/core/fd_tensor.h"
+
+namespace fastdeploy {
+namespace function {
+/** Cast the type of the data in GPU buffer.
+    @param in The input tensor.
+    @param out The output tensor
+    @param stream CUDA stream
+*/
+FASTDEPLOY_DECL void CudaCast(const FDTensor& in, FDTensor* out,
+                              cudaStream_t stream);
+}  // namespace function
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/function/eigen.h

@@ -0,0 +1,140 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <algorithm>
+#include <memory>
+#include <vector>
+#include "fastdeploy/core/fd_tensor.h"
+#include "fastdeploy/utils/axis_utils.h"
+#include "unsupported/Eigen/CXX11/Tensor"
+
+namespace fastdeploy {
+namespace function {
+// EigenDim converts shape into Eigen::DSizes.
+template <int D>
+struct EigenDim {
+  using Type = Eigen::DSizes<Eigen::DenseIndex, D>;
+
+  static Type From(const std::vector<int64_t>& dims) {
+    Type ret;
+    for (int64_t d = 0; d < dims.size(); d++) {
+      ret[d] = dims[d];
+    }
+    return ret;
+  }
+};
+
+// Interpret FDTensor as EigenTensor and EigenConstTensor.
+template <typename T, size_t D, int MajorType = Eigen::RowMajor,
+          typename IndexType = Eigen::DenseIndex>
+struct EigenTensor {
+  using Type = Eigen::TensorMap<Eigen::Tensor<T, D, MajorType, IndexType>>;
+
+  using ConstType =
+      Eigen::TensorMap<Eigen::Tensor<const T, D, MajorType, IndexType>>;
+
+  static Type From(FDTensor& tensor,
+                   const std::vector<int64_t>& dims) {  // NOLINT
+    return Type(reinterpret_cast<T*>(tensor.Data()), EigenDim<D>::From(dims));
+  }
+
+  static Type From(FDTensor& tensor) {  // NOLINT
+    return From(tensor, tensor.shape);
+  }  // NOLINT
+
+  static ConstType From(const FDTensor& tensor,
+                        const std::vector<int64_t>& dims) {
+    return ConstType(reinterpret_cast<const T*>(tensor.Data()),
+                     EigenDim<D>::From(dims));
+  }
+
+  static ConstType From(const FDTensor& tensor) {
+    return From(tensor, tensor.shape);
+  }
+};
+
+template <typename T, int MajorType = Eigen::RowMajor,
+          typename IndexType = Eigen::DenseIndex>
+struct EigenScalar {
+  // Scalar tensor (implemented as a rank-0 tensor) of scalar type T.
+  using Type = Eigen::TensorMap<
+      Eigen::TensorFixedSize<T, Eigen::Sizes<>, MajorType, IndexType>>;
+  using ConstType = Eigen::TensorMap<
+      Eigen::TensorFixedSize<const T, Eigen::Sizes<>, MajorType, IndexType>>;
+
+  static Type From(FDTensor& tensor) {
+    return Type(reinterpret_cast<T*>(tensor.Data()));
+  }  // NOLINT
+
+  static ConstType From(const FDTensor& tensor) {
+    return ConstType(reinterpret_cast<const T*>(tensor.Data()));
+  }
+};
+
+template <typename T, int MajorType = Eigen::RowMajor,
+          typename IndexType = Eigen::DenseIndex>
+struct EigenVector : public EigenTensor<T, 1, MajorType, IndexType> {
+  // Flatten reshapes a Tensor into an EigenVector.
+  static typename EigenVector::Type Flatten(FDTensor& tensor) {  // NOLINT
+    return EigenVector::From(tensor, {tensor.Numel()});
+  }
+
+  static typename EigenVector::ConstType Flatten(
+      const FDTensor& tensor) {  // NOLINT
+    return EigenVector::From(tensor, {tensor.Numel()});
+  }
+};
+
+template <typename T, int MajorType = Eigen::RowMajor,
+          typename IndexType = Eigen::DenseIndex>
+struct EigenMatrix : public EigenTensor<T, 2, MajorType, IndexType> {
+  static typename EigenMatrix::Type Reshape(FDTensor& tensor,  // NOLINT
+                                            int num_col_dims) {
+    int rank = tensor.shape.size();
+    FDASSERT((num_col_dims > 0 && num_col_dims < rank),
+             "Input dimension number(num_col_dims) must be between 0 and %d, "
+             "but received number is %d.",
+             rank, num_col_dims);
+    const int n = SizeToAxis(num_col_dims, tensor.shape);
+    const int d = SizeFromAxis(num_col_dims, tensor.shape);
+    return EigenMatrix::From(tensor, {n, d});
+  }
+
+  static typename EigenMatrix::ConstType Reshape(const FDTensor& tensor,
+                                                 int num_col_dims) {
+    int rank = tensor.shape.size();
+    FDASSERT((num_col_dims > 0 && num_col_dims < rank),
+             "Input dimension number(num_col_dims) must be between 0 and %d, "
+             "but received number is %d.",
+             rank, num_col_dims);
+    const int n = SizeToAxis(num_col_dims, tensor.shape);
+    const int d = SizeFromAxis(num_col_dims, tensor.shape);
+    return EigenMatrix::From(tensor, {n, d});
+  }
+};
+
+class EigenDeviceWrapper {
+ public:
+  static std::shared_ptr<EigenDeviceWrapper> GetInstance();
+  const Eigen::DefaultDevice* GetDevice() const;
+
+ private:
+  Eigen::DefaultDevice device_;
+  static std::shared_ptr<EigenDeviceWrapper> instance_;
+};
+
+}  // namespace function
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/function/pad.h

@@ -0,0 +1,31 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "fastdeploy/core/fd_tensor.h"
+
+namespace fastdeploy {
+namespace function {
+/** Excute the pad operation for input FDTensor along given dims.
+    @param x The input tensor.
+    @param out The output tensor which stores the result.
+    @param pads The size of padding for each dimension, for 3-D tensor, the pads should be [1d-left, 1d-right, 2d-left, 2d-right, 3d-left, 3d-right]
+    @param pad_value The value which will fill into out tensor
+*/
+FASTDEPLOY_DECL void Pad(const FDTensor& x, FDTensor* out,
+            const std::vector<int>& pads, float pad_value = 0);
+
+}
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/function/reduce.h

@@ -0,0 +1,127 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "fastdeploy/core/fd_tensor.h"
+
+namespace fastdeploy {
+namespace function {
+/** Excute the maximum operation for input FDTensor along given dims.
+    @param x The input tensor.
+    @param out The output tensor which stores the result.
+    @param dims The vector of axis which will be reduced.
+    @param keep_dim Whether to keep the reduced dims, default false.
+    @param reduce_all Whether to reduce all dims, default false.
+*/
+FASTDEPLOY_DECL void Max(const FDTensor& x, FDTensor* out,
+                         const std::vector<int64_t>& dims,
+                         bool keep_dim = false, bool reduce_all = false);
+
+/** Excute the minimum operation for input FDTensor along given dims.
+    @param x The input tensor.
+    @param out The output tensor which stores the result.
+    @param dims The vector of axis which will be reduced.
+    @param keep_dim Whether to keep the reduced dims, default false.
+    @param reduce_all Whether to reduce all dims, default false.
+*/
+FASTDEPLOY_DECL void Min(const FDTensor& x, FDTensor* out,
+                         const std::vector<int64_t>& dims,
+                         bool keep_dim = false, bool reduce_all = false);
+
+/** Excute the sum operation for input FDTensor along given dims.
+    @param x The input tensor.
+    @param out The output tensor which stores the result.
+    @param dims The vector of axis which will be reduced.
+    @param keep_dim Whether to keep the reduced dims, default false.
+    @param reduce_all Whether to reduce all dims, default false.
+*/
+FASTDEPLOY_DECL void Sum(const FDTensor& x, FDTensor* out,
+                         const std::vector<int64_t>& dims,
+                         bool keep_dim = false, bool reduce_all = false);
+
+/** Excute the all operation for input FDTensor along given dims.
+    @param x The input tensor.
+    @param out The output tensor which stores the result.
+    @param dims The vector of axis which will be reduced.
+    @param keep_dim Whether to keep the reduced dims, default false.
+    @param reduce_all Whether to reduce all dims, default false.
+*/
+FASTDEPLOY_DECL void All(const FDTensor& x, FDTensor* out,
+                         const std::vector<int64_t>& dims,
+                         bool keep_dim = false, bool reduce_all = false);
+
+/** Excute the any operation for input FDTensor along given dims.
+    @param x The input tensor.
+    @param out The output tensor which stores the result.
+    @param dims The vector of axis which will be reduced.
+    @param keep_dim Whether to keep the reduced dims, default false.
+    @param reduce_all Whether to reduce all dims, default false.
+*/
+FASTDEPLOY_DECL void Any(const FDTensor& x, FDTensor* out,
+                         const std::vector<int64_t>& dims,
+                         bool keep_dim = false, bool reduce_all = false);
+
+/** Excute the mean operation for input FDTensor along given dims.
+    @param x The input tensor.
+    @param out The output tensor which stores the result.
+    @param dims The vector of axis which will be reduced.
+    @param keep_dim Whether to keep the reduced dims, default false.
+    @param reduce_all Whether to reduce all dims, default false.
+*/
+FASTDEPLOY_DECL void Mean(const FDTensor& x, FDTensor* out,
+                          const std::vector<int64_t>& dims,
+                          bool keep_dim = false, bool reduce_all = false);
+
+/** Excute the product operation for input FDTensor along given dims.
+    @param x The input tensor.
+    @param out The output tensor which stores the result.
+    @param dims The vector of axis which will be reduced.
+    @param keep_dim Whether to keep the reduced dims, default false.
+    @param reduce_all Whether to reduce all dims, default false.
+*/
+FASTDEPLOY_DECL void Prod(const FDTensor& x, FDTensor* out,
+                          const std::vector<int64_t>& dims,
+                          bool keep_dim = false, bool reduce_all = false);
+
+/** Excute the argmax operation for input FDTensor along given dims.
+    @param x The input tensor.
+    @param out The output tensor which stores the result.
+    @param axis The axis which will be reduced.
+    @param output_dtype The data type of output FDTensor, INT64 or INT32,
+   default to INT64.
+    @param keep_dim Whether to keep the reduced dims, default false.
+    @param flatten Whether to flatten FDTensor to get the argmin index, default
+   false.
+*/
+FASTDEPLOY_DECL void ArgMax(const FDTensor& x, FDTensor* out, int64_t axis,
+                            FDDataType output_dtype = FDDataType::INT64,
+                            bool keep_dim = false, bool flatten = false);
+
+/** Excute the argmin operation for input FDTensor along given dims.
+    @param x The input tensor.
+    @param out The output tensor which stores the result.
+    @param axis The axis which will be reduced.
+    @param output_dtype The data type of output FDTensor, INT64 or INT32,
+   default to INT64.
+    @param keep_dim Whether to keep the reduced dims, default false.
+    @param flatten Whether to flatten FDTensor to get the argmin index, default
+   false.
+*/
+FASTDEPLOY_DECL void ArgMin(const FDTensor& x, FDTensor* out, int64_t axis,
+                            FDDataType output_dtype = FDDataType::INT64,
+                            bool keep_dim = false, bool flatten = false);
+
+}  // namespace function
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/function/reduce_functor.h

@@ -0,0 +1,77 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "fastdeploy/function/eigen.h"
+namespace fastdeploy {
+namespace function {
+//////// Max Functor ///////
+struct MaxFunctor {
+  template <typename X, typename Y, typename Dim>
+  void operator()(const Eigen::DefaultDevice& dev, X* x, Y* y, const Dim& dim) {
+    y->device(dev) = x->maximum(dim);
+  }
+};
+
+//////// Min Functor ///////
+struct MinFunctor {
+  template <typename X, typename Y, typename Dim>
+  void operator()(const Eigen::DefaultDevice& dev, X* x, Y* y, const Dim& dim) {
+    y->device(dev) = x->minimum(dim);
+  }
+};
+
+//////// Sum Functor ///////
+struct SumFunctor {
+  template <typename X, typename Y, typename Dim>
+  void operator()(const Eigen::DefaultDevice& dev, X* x, Y* y, const Dim& dim) {
+    y->device(dev) = x->sum(dim);
+  }
+};
+
+//////// All Functor ///////
+struct AllFunctor {
+  template <typename X, typename Y, typename Dim>
+  void operator()(const Eigen::DefaultDevice& dev, X* x, Y* y, const Dim& dim) {
+    y->device(dev) = x->all(dim);
+  }
+};
+
+//////// Any Functor ///////
+struct AnyFunctor {
+  template <typename X, typename Y, typename Dim>
+  void operator()(const Eigen::DefaultDevice& dev, X* x, Y* y, const Dim& dim) {
+    y->device(dev) = x->any(dim);
+  }
+};
+
+//////// Mean Functor ///////
+struct MeanFunctor {
+  template <typename X, typename Y, typename Dim>
+  void operator()(const Eigen::DefaultDevice& dev, X* x, Y* y, const Dim& dim) {
+    y->device(dev) = x->mean(dim);
+  }
+};
+
+//////// Prod Functor ///////
+struct ProdFunctor {
+  template <typename X, typename Y, typename Dim>
+  void operator()(const Eigen::DefaultDevice& dev, X* x, Y* y, const Dim& dim) {
+    y->device(dev) = x->prod(dim);
+  }
+};
+
+}  // namespace function
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/function/softmax.h

@@ -0,0 +1,29 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "fastdeploy/core/fd_tensor.h"
+
+namespace fastdeploy {
+namespace function {
+/** Excute the softmax operation for input FDTensor along given dims.
+    @param x The input tensor.
+    @param out The output tensor which stores the result.
+    @param axis The axis to be computed softmax value.
+*/
+FASTDEPLOY_DECL void Softmax(const FDTensor& x, FDTensor* out, int axis = -1);
+
+}  // namespace function
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/function/transpose.h

@@ -0,0 +1,29 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "fastdeploy/core/fd_tensor.h"
+
+namespace fastdeploy {
+namespace function {
+/** Excute the transpose operation for input FDTensor along given dims.
+    @param x The input tensor.
+    @param out The output tensor which stores the result.
+    @param dims The vector of axis which the input tensor will transpose.
+*/
+FASTDEPLOY_DECL void Transpose(const FDTensor& x, FDTensor* out,
+                               const std::vector<int64_t>& dims);
+}  // namespace function
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/pipeline.h

@@ -0,0 +1,21 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#pragma once
+
+#include "fastdeploy/core/config.h"
+#ifdef ENABLE_VISION
+#include "fastdeploy/pipeline/pptinypose/pipeline.h"
+#endif
+
+#include "fastdeploy/vision/visualize/visualize.h"

3rdparty/include/fastdeploy/pipeline/pptinypose/pipeline.h

@@ -0,0 +1,67 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "fastdeploy/fastdeploy_model.h"
+#include "fastdeploy/vision/common/result.h"
+#include "fastdeploy/vision/detection/ppdet/model.h"
+#include "fastdeploy/vision/keypointdet/pptinypose/pptinypose.h"
+
+namespace fastdeploy {
+/** \brief All pipeline model APIs are defined inside this namespace
+ *
+ */
+namespace pipeline {
+
+/*! @brief PPTinyPose Pipeline object used when to load a detection model + pptinypose model
+ */
+class FASTDEPLOY_DECL PPTinyPose {
+ public:
+  /** \brief Set initialized detection model object and pptinypose model object
+   *
+   * \param[in] det_model Initialized detection model object
+   * \param[in] pptinypose_model Initialized pptinypose model object
+   */
+  PPTinyPose(
+      fastdeploy::vision::detection::PicoDet* det_model,
+      fastdeploy::vision::keypointdetection::PPTinyPose* pptinypose_model);
+
+  /** \brief Predict the keypoint detection result for an input image
+   *
+   * \param[in] img The input image data, comes from cv::imread()
+   * \param[in] result The output keypoint detection result will be writen to this structure
+   * \return true if the prediction successed, otherwise false
+   */
+  virtual bool Predict(cv::Mat* img,
+                       fastdeploy::vision::KeyPointDetectionResult* result);
+
+  /* \brief The score threshold for detectin model to filter bbox before inputting pptinypose model
+   */
+  float detection_model_score_threshold = 0;
+
+ protected:
+  fastdeploy::vision::detection::PicoDet* detector_ = nullptr;
+  fastdeploy::vision::keypointdetection::PPTinyPose* pptinypose_model_ =
+      nullptr;
+
+  virtual bool Detect(cv::Mat* img,
+                      fastdeploy::vision::DetectionResult* result);
+  virtual bool KeypointDetect(
+      cv::Mat* img, fastdeploy::vision::KeyPointDetectionResult* result,
+      fastdeploy::vision::DetectionResult& detection_result);
+};
+
+}  // namespace pipeline
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/runtime.h

@@ -0,0 +1,432 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/*! \file runtime.h
+    \brief A brief file description.
+
+    More details
+ */
+
+#pragma once
+
+#include <algorithm>
+#include <map>
+#include <vector>
+
+#include "fastdeploy/backends/backend.h"
+#include "fastdeploy/utils/perf.h"
+#include "backends/rknpu/rknpu2/rknpu2_config.h"
+
+/** \brief All C++ FastDeploy APIs are defined inside this namespace
+*
+*/
+namespace fastdeploy {
+
+/*! Inference backend supported in FastDeploy */
+enum Backend {
+  UNKNOWN,   ///< Unknown inference backend
+  ORT,     ///< ONNX Runtime, support Paddle/ONNX format model, CPU / Nvidia GPU
+  TRT,      ///< TensorRT, support Paddle/ONNX format model, Nvidia GPU only
+  PDINFER,  ///< Paddle Inference, support Paddle format model, CPU / Nvidia GPU
+  POROS,    ///< Poros, support TorchScript format model, CPU / Nvidia GPU
+  OPENVINO,  ///< Intel OpenVINO, support Paddle/ONNX format, CPU only
+  LITE,     ///< Paddle Lite, support Paddle format model, ARM CPU only
+  RKNPU2,   ///< RKNPU2, support RKNN format model, Rockchip NPU only
+};
+
+FASTDEPLOY_DECL std::ostream& operator<<(std::ostream& out,
+                                         const Backend& backend);
+
+/*! Paddle Lite power mode for mobile device. */
+enum LitePowerMode {
+  LITE_POWER_HIGH = 0,       ///< Use Lite Backend with high power mode
+  LITE_POWER_LOW = 1,        ///< Use Lite Backend with low power mode
+  LITE_POWER_FULL = 2,       ///< Use Lite Backend with full power mode
+  LITE_POWER_NO_BIND = 3,    ///< Use Lite Backend with no bind power mode
+  LITE_POWER_RAND_HIGH = 4,  ///< Use Lite Backend with rand high mode
+  LITE_POWER_RAND_LOW = 5    ///< Use Lite Backend with rand low power mode
+};
+
+FASTDEPLOY_DECL std::string Str(const Backend& b);
+FASTDEPLOY_DECL std::string Str(const ModelFormat& f);
+
+/**
+ * @brief Get all the available inference backend in FastDeploy
+ */
+FASTDEPLOY_DECL std::vector<Backend> GetAvailableBackends();
+
+/**
+ * @brief Check if the inference backend available
+ */
+FASTDEPLOY_DECL bool IsBackendAvailable(const Backend& backend);
+
+bool CheckModelFormat(const std::string& model_file,
+                      const ModelFormat& model_format);
+ModelFormat GuessModelFormat(const std::string& model_file);
+
+/*! @brief Option object used when create a new Runtime object
+ */
+struct FASTDEPLOY_DECL RuntimeOption {
+  /** \brief Set path of model file and parameter file
+   *
+   * \param[in] model_path Path of model file, e.g ResNet50/model.pdmodel for Paddle format model / ResNet50/model.onnx for ONNX format model
+   * \param[in] params_path Path of parameter file, this only used when the model format is Paddle, e.g Resnet50/model.pdiparams
+   * \param[in] format Format of the loaded model
+   */
+  void SetModelPath(const std::string& model_path,
+                    const std::string& params_path = "",
+                    const ModelFormat& format = ModelFormat::PADDLE);
+
+  /// Use cpu to inference, the runtime will inference on CPU by default
+  void UseCpu();
+
+  /// Use Nvidia GPU to inference
+  void UseGpu(int gpu_id = 0);
+
+  void UseRKNPU2(fastdeploy::rknpu2::CpuName rknpu2_name
+                             = fastdeploy::rknpu2::CpuName::RK3588,
+                 fastdeploy::rknpu2::CoreMask rknpu2_core
+                             = fastdeploy::rknpu2::CoreMask::RKNN_NPU_CORE_0);
+
+  /// Use TimVX to inference
+  void UseTimVX();
+
+  void SetExternalStream(void* external_stream);
+
+  /*
+   * @brief Set number of cpu threads while inference on CPU, by default it will decided by the different backends
+   */
+  void SetCpuThreadNum(int thread_num);
+
+  /// Set ORT graph opt level, default is decide by ONNX Runtime itself
+  void SetOrtGraphOptLevel(int level = -1);
+
+  /// Set Paddle Inference as inference backend, support CPU/GPU
+  void UsePaddleBackend();
+
+  /// Wrapper function of UsePaddleBackend()
+  void UsePaddleInferBackend() {
+    return UsePaddleBackend();
+  }
+
+  /// Set ONNX Runtime as inference backend, support CPU/GPU
+  void UseOrtBackend();
+
+  /// Set TensorRT as inference backend, only support GPU
+  void UseTrtBackend();
+
+  /// Set Poros backend as inference backend, support CPU/GPU
+  void UsePorosBackend();
+
+  /// Set OpenVINO as inference backend, only support CPU
+  void UseOpenVINOBackend();
+
+  /// Set Paddle Lite as inference backend, only support arm cpu
+  void UseLiteBackend();
+
+  /// Wrapper function of UseLiteBackend()
+  void UsePaddleLiteBackend() {
+    return UseLiteBackend();
+  }
+
+  /// Set mkldnn switch while using Paddle Inference as inference backend
+  void SetPaddleMKLDNN(bool pd_mkldnn = true);
+
+  /*
+   * @brief If TensorRT backend is used, EnablePaddleToTrt will change to use Paddle Inference backend, and use its integrated TensorRT instead.
+   */
+  void EnablePaddleToTrt();
+
+  /**
+   * @brief Delete pass by name while using Paddle Inference as inference backend, this can be called multiple times to delete a set of passes
+   */
+  void DeletePaddleBackendPass(const std::string& delete_pass_name);
+
+  /**
+   * @brief Enable print debug information while using Paddle Inference as inference backend, the backend disable the debug information by default
+   */
+  void EnablePaddleLogInfo();
+
+  /**
+   * @brief Disable print debug information while using Paddle Inference as inference backend
+   */
+  void DisablePaddleLogInfo();
+
+  /**
+   * @brief Set shape cache size while using Paddle Inference with mkldnn, by default it will cache all the difference shape
+   */
+  void SetPaddleMKLDNNCacheSize(int size);
+
+  /**
+   * @brief Set optimzed model dir for Paddle Lite backend.
+   */
+  void SetLiteOptimizedModelDir(const std::string& optimized_model_dir);
+
+  /**
+   * @brief Set nnadapter subgraph partition path for Paddle Lite backend.
+   */
+  void SetLiteSubgraphPartitionPath(
+    const std::string& nnadapter_subgraph_partition_config_path);
+
+  /**
+   * @brief enable half precision while use paddle lite backend
+   */
+  void EnableLiteFP16();
+
+  /**
+   * @brief disable half precision, change to full precision(float32)
+   */
+  void DisableLiteFP16();
+
+  /**
+    * @brief enable int8 precision while use paddle lite backend
+    */
+  void EnableLiteInt8();
+
+  /**
+    * @brief disable int8 precision, change to full precision(float32)
+    */
+  void DisableLiteInt8();
+
+  /**
+   * @brief Set power mode while using Paddle Lite as inference backend, mode(0: LITE_POWER_HIGH; 1: LITE_POWER_LOW; 2: LITE_POWER_FULL; 3: LITE_POWER_NO_BIND, 4: LITE_POWER_RAND_HIGH; 5: LITE_POWER_RAND_LOW, refer [paddle lite](https://paddle-lite.readthedocs.io/zh/latest/api_reference/cxx_api_doc.html#set-power-mode) for more details)
+   */
+  void SetLitePowerMode(LitePowerMode mode);
+
+  /** \brief Set shape range of input tensor for the model that contain dynamic input shape while using TensorRT backend
+   *
+   * \param[in] input_name The name of input for the model which is dynamic shape
+   * \param[in] min_shape The minimal shape for the input tensor
+   * \param[in] opt_shape The optimized shape for the input tensor, just set the most common shape, if set as default value, it will keep same with min_shape
+   * \param[in] max_shape The maximum shape for the input tensor, if set as default value, it will keep same with min_shape
+   */
+  void SetTrtInputShape(
+      const std::string& input_name, const std::vector<int32_t>& min_shape,
+      const std::vector<int32_t>& opt_shape = std::vector<int32_t>(),
+      const std::vector<int32_t>& max_shape = std::vector<int32_t>());
+
+  /// Set max_workspace_size for TensorRT, default 1<<30
+  void SetTrtMaxWorkspaceSize(size_t trt_max_workspace_size);
+
+  /// Set max_batch_size for TensorRT, default 32
+  void SetTrtMaxBatchSize(size_t max_batch_size);
+
+  /**
+   * @brief Enable FP16 inference while using TensorRT backend. Notice: not all the GPU device support FP16, on those device doesn't support FP16, FastDeploy will fallback to FP32 automaticly
+   */
+  void EnableTrtFP16();
+
+  /// Disable FP16 inference while using TensorRT backend
+  void DisableTrtFP16();
+
+  /**
+   * @brief Set cache file path while use TensorRT backend. Loadding a Paddle/ONNX model and initialize TensorRT will take a long time, by this interface it will save the tensorrt engine to `cache_file_path`, and load it directly while execute the code again
+   */
+  void SetTrtCacheFile(const std::string& cache_file_path);
+
+  /**
+   * @brief Enable pinned memory. Pinned memory can be utilized to speedup the data transfer between CPU and GPU. Currently it's only suppurted in TRT backend and Paddle Inference backend.
+   */
+  void EnablePinnedMemory();
+
+  /**
+   * @brief Disable pinned memory
+   */
+  void DisablePinnedMemory();
+
+  /**
+   * @brief Enable to collect shape in paddle trt backend
+   */
+  void EnablePaddleTrtCollectShape();
+
+  /**
+   * @brief Disable to collect shape in paddle trt backend
+   */
+  void DisablePaddleTrtCollectShape();
+
+  /*
+   * @brief Set number of streams by the OpenVINO backends
+   */
+  void SetOpenVINOStreams(int num_streams);
+
+  /** \Use Graphcore IPU to inference.
+   *
+   * \param[in] device_num the number of IPUs.
+   * \param[in] micro_batch_size the batch size in the graph, only work when graph has no batch shape info.
+   * \param[in] enable_pipelining enable pipelining.
+   * \param[in] batches_per_step the number of batches per run in pipelining.
+   */
+  void UseIpu(int device_num = 1, int micro_batch_size = 1,
+              bool enable_pipelining = false, int batches_per_step = 1);
+
+  /** \brief Set IPU config.
+   *
+   * \param[in] enable_fp16 enable fp16.
+   * \param[in] replica_num the number of graph replication.
+   * \param[in] available_memory_proportion the available memory proportion for matmul/conv.
+   * \param[in] enable_half_partial enable fp16 partial for matmul, only work with fp16.
+   */
+  void SetIpuConfig(bool enable_fp16 = false, int replica_num = 1,
+                    float available_memory_proportion = 1.0,
+                    bool enable_half_partial = false);
+
+  Backend backend = Backend::UNKNOWN;
+  // for cpu inference and preprocess
+  // default will let the backend choose their own default value
+  int cpu_thread_num = -1;
+  int device_id = 0;
+
+  Device device = Device::CPU;
+
+  void* external_stream_ = nullptr;
+
+  bool enable_pinned_memory = false;
+
+  // ======Only for ORT Backend========
+  // -1 means use default value by ort
+  // 0: ORT_DISABLE_ALL 1: ORT_ENABLE_BASIC 2: ORT_ENABLE_EXTENDED 3:
+  // ORT_ENABLE_ALL
+  int ort_graph_opt_level = -1;
+  int ort_inter_op_num_threads = -1;
+  // 0: ORT_SEQUENTIAL 1: ORT_PARALLEL
+  int ort_execution_mode = -1;
+
+  // ======Only for Paddle Backend=====
+  bool pd_enable_mkldnn = true;
+  bool pd_enable_log_info = false;
+  bool pd_enable_trt = false;
+  bool pd_collect_shape = false;
+  int pd_mkldnn_cache_size = 1;
+  std::vector<std::string> pd_delete_pass_names;
+
+  // ======Only for Paddle IPU Backend =======
+  int ipu_device_num = 1;
+  int ipu_micro_batch_size = 1;
+  bool ipu_enable_pipelining = false;
+  int ipu_batches_per_step = 1;
+  bool ipu_enable_fp16 = false;
+  int ipu_replica_num = 1;
+  float ipu_available_memory_proportion = 1.0;
+  bool ipu_enable_half_partial = false;
+
+  // ======Only for Paddle-Lite Backend=====
+  // 0: LITE_POWER_HIGH 1: LITE_POWER_LOW 2: LITE_POWER_FULL
+  // 3: LITE_POWER_NO_BIND 4: LITE_POWER_RAND_HIGH
+  // 5: LITE_POWER_RAND_LOW
+  LitePowerMode lite_power_mode = LitePowerMode::LITE_POWER_NO_BIND;
+  // enable int8 or not
+  bool lite_enable_int8 = false;
+  // enable fp16 or not
+  bool lite_enable_fp16 = false;
+  // optimized model dir for CxxConfig
+  std::string lite_optimized_model_dir = "";
+  std::string lite_nnadapter_subgraph_partition_config_path = "";
+  bool enable_timvx = false;
+
+  // ======Only for Trt Backend=======
+  std::map<std::string, std::vector<int32_t>> trt_max_shape;
+  std::map<std::string, std::vector<int32_t>> trt_min_shape;
+  std::map<std::string, std::vector<int32_t>> trt_opt_shape;
+  std::string trt_serialize_file = "";
+  bool trt_enable_fp16 = false;
+  bool trt_enable_int8 = false;
+  size_t trt_max_batch_size = 32;
+  size_t trt_max_workspace_size = 1 << 30;
+
+  // ======Only for Poros Backend=======
+  bool is_dynamic = false;
+  bool long_to_int = true;
+  bool use_nvidia_tf32 = false;
+  int unconst_ops_thres = -1;
+  std::string poros_file = "";
+
+  // ======Only for OpenVINO Backend=======
+  int ov_num_streams = 1;
+
+  // ======Only for RKNPU2 Backend=======
+  fastdeploy::rknpu2::CpuName rknpu2_cpu_name_
+            = fastdeploy::rknpu2::CpuName::RK3588;
+  fastdeploy::rknpu2::CoreMask rknpu2_core_mask_
+            = fastdeploy::rknpu2::CoreMask::RKNN_NPU_CORE_AUTO;
+
+  std::string model_file = "";  // Path of model file
+  std::string params_file = "";  // Path of parameters file, can be empty
+  // format of input model
+  ModelFormat model_format = ModelFormat::AUTOREC;
+};
+
+/*! @brief Runtime object used to inference the loaded model on different devices
+ */
+struct FASTDEPLOY_DECL Runtime {
+ public:
+  /// Intialize a Runtime object with RuntimeOption
+  bool Init(const RuntimeOption& _option);
+
+  /** \brief Inference the model by the input data, and write to the output
+   *
+   * \param[in] input_tensors Notice the FDTensor::name should keep same with the model's input
+   * \param[in] output_tensors Inference results
+   * \return true if the inference successed, otherwise false
+   */
+  bool Infer(std::vector<FDTensor>& input_tensors,
+             std::vector<FDTensor>* output_tensors);
+
+  /** \brief Compile TorchScript Module, only for Poros backend
+   *
+   * \param[in] prewarm_tensors Prewarm datas for compile
+   * \param[in] _option Runtime option
+   * \return true if compile successed, otherwise false
+   */
+  bool Compile(std::vector<std::vector<FDTensor>>& prewarm_tensors,
+               const RuntimeOption& _option);
+
+  /** \brief Get number of inputs
+   */
+  int NumInputs() { return backend_->NumInputs(); }
+  /** \brief Get number of outputs
+   */
+  int NumOutputs() { return backend_->NumOutputs(); }
+  /** \brief Get input information by index
+   */
+  TensorInfo GetInputInfo(int index);
+  /** \brief Get output information by index
+   */
+  TensorInfo GetOutputInfo(int index);
+  /** \brief Get all the input information
+   */
+  std::vector<TensorInfo> GetInputInfos();
+  /** \brief Get all the output information
+   */
+  std::vector<TensorInfo> GetOutputInfos();
+
+  /** \brief Clone new Runtime when multiple instances of the same model are created
+   *
+   * \param[in] stream CUDA Stream, defualt param is nullptr
+   * \return new Runtime* by this clone
+   */
+  Runtime* Clone(void* stream = nullptr,
+                 int device_id = -1);
+
+  RuntimeOption option;
+
+ private:
+  void CreateOrtBackend();
+  void CreatePaddleBackend();
+  void CreateTrtBackend();
+  void CreateOpenVINOBackend();
+  void CreateLiteBackend();
+  void CreateRKNPU2Backend();
+  std::unique_ptr<BaseBackend> backend_;
+};
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/text.h

@@ -0,0 +1,19 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#pragma once
+
+#include "fastdeploy/core/config.h"
+#ifdef ENABLE_TEXT
+#include "fastdeploy/text/uie/model.h"
+#endif

3rdparty/include/fastdeploy/utils/axis_utils.h

@@ -0,0 +1,52 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+namespace fastdeploy {
+
+static inline int CanonicalAxis(const int axis, const int rank) {
+  if (axis < 0) {
+    return axis + rank;
+  }
+  return axis;
+}
+
+static inline int SizeToAxis(const int axis, const std::vector<int64_t>& dims) {
+  int size = 1;
+  for (int i = 0; i < axis; i++) {
+    size *= dims[i];
+  }
+  return size;
+}
+
+static inline int SizeFromAxis(const int axis,
+                               const std::vector<int64_t>& dims) {
+  int size = 1;
+  for (int i = axis; i < dims.size(); i++) {
+    size *= dims[i];
+  }
+  return size;
+}
+
+static inline int SizeOutAxis(const int axis,
+                              const std::vector<int64_t>& dims) {
+  int size = 1;
+  for (int i = axis + 1; i < dims.size(); i++) {
+    size *= dims[i];
+  }
+  return size;
+}
+
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/utils/path.h

@@ -0,0 +1,74 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <string>
+#include <vector>
+#include <fstream>
+#ifdef _MSC_VER
+#define PATH_SEP "\\"
+#else
+#define PATH_SEP "/"
+#endif
+
+namespace fastdeploy {
+
+inline std::string PathJoin(const std::vector<std::string>& paths,
+                            const std::string& sep = PATH_SEP) {
+  if (paths.size() == 1) {
+    return paths[0];
+  }
+  std::string filepath = "";
+  for (const auto& path : paths) {
+    if (filepath == "") {
+      filepath += path;
+      continue;
+    }
+    if (path[0] == sep[0] || filepath.back() == sep[0]) {
+      filepath += path;
+    } else {
+      filepath += sep + path;
+    }
+  }
+  return filepath;
+}
+
+inline std::string PathJoin(const std::string& folder,
+                            const std::string& filename,
+                            const std::string& sep = PATH_SEP) {
+  return PathJoin(std::vector<std::string>{folder, filename}, sep);
+}
+
+inline std::string GetDirFromPath(const std::string& path) {
+  auto pos = path.find_last_of(PATH_SEP);
+  if (pos == std::string::npos) {
+    return "";
+  }
+  // The root path in UNIX systems
+  if (pos == 0) {
+    return "/";
+  }
+  return path.substr(0, pos);
+}
+
+inline bool CheckFileExists(const std::string& path) {
+  std::fstream fin(path, std::ios::in);
+  if (!fin) {
+    return false;
+  }
+  return true;
+}
+
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/utils/perf.h

@@ -0,0 +1,49 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "fastdeploy/utils/utils.h"
+#include <chrono> // NOLINT
+
+namespace fastdeploy {
+
+class FASTDEPLOY_DECL TimeCounter {
+ public:
+  void Start() { begin_ = std::chrono::system_clock::now(); }
+
+  void End() { end_ = std::chrono::system_clock::now(); }
+
+  double Duration() {
+    auto duration =
+        std::chrono::duration_cast<std::chrono::microseconds>(end_ - begin_);
+    return static_cast<double>(duration.count()) *
+           std::chrono::microseconds::period::num /
+           std::chrono::microseconds::period::den;
+  }
+
+  void PrintInfo(const std::string& prefix = "TimeCounter: ",
+                 bool print_out = true) {
+    if (!print_out) {
+      return;
+    }
+    FDLogger() << prefix << " duration = " << Duration() << "s." << std::endl;
+  }
+
+ private:
+  std::chrono::time_point<std::chrono::system_clock> begin_;
+  std::chrono::time_point<std::chrono::system_clock> end_;
+};
+
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/utils/unique_ptr.h

@@ -0,0 +1,58 @@
+/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+
+#include <memory>
+
+namespace fastdeploy {
+namespace utils {
+// Trait to select overloads and return types for MakeUnique.
+template <typename T>
+struct MakeUniqueResult {
+  using scalar = std::unique_ptr<T>;
+};
+template <typename T>
+struct MakeUniqueResult<T[]> {
+  using array = std::unique_ptr<T[]>;
+};
+template <typename T, size_t N>
+struct MakeUniqueResult<T[N]> {
+  using invalid = void;
+};
+
+// MakeUnique<T>(...) is an early implementation of C++14 std::make_unique.
+// It is designed to be 100% compatible with std::make_unique so that the
+// eventual switchover will be a simple renaming operation.
+template <typename T, typename... Args>
+typename MakeUniqueResult<T>::scalar make_unique(Args &&... args) {  // NOLINT
+  return std::unique_ptr<T>(
+      new T(std::forward<Args>(args)...));  // NOLINT(build/c++11)
+}
+
+// Overload for array of unknown bound.
+// The allocation of arrays needs to use the array form of new,
+// and cannot take element constructor arguments.
+template <typename T>
+typename MakeUniqueResult<T>::array make_unique(size_t n) {
+  return std::unique_ptr<T>(new typename std::remove_extent<T>::type[n]());
+}
+
+// Reject arrays of known bound.
+template <typename T, typename... Args>
+typename MakeUniqueResult<T>::invalid make_unique(Args &&... /* args */) =
+    delete;  // NOLINT
+
+}  // namespace utils
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/utils/utils.h

@@ -0,0 +1,189 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <stdlib.h>
+#include <cstdio>
+
+#include <fstream>
+#include <iostream>
+#include <sstream>
+#include <string>
+#include <vector>
+
+#if defined(_WIN32)
+#ifdef FASTDEPLOY_LIB
+#define FASTDEPLOY_DECL __declspec(dllexport)
+#else
+#define FASTDEPLOY_DECL __declspec(dllimport)
+#endif  // FASTDEPLOY_LIB
+#else
+#define FASTDEPLOY_DECL __attribute__((visibility("default")))
+#endif  // _WIN32
+
+namespace fastdeploy {
+
+class FASTDEPLOY_DECL FDLogger {
+ public:
+  FDLogger() {
+    line_ = "";
+    prefix_ = "[FastDeploy]";
+    verbose_ = true;
+  }
+  explicit FDLogger(bool verbose, const std::string& prefix = "[FastDeploy]");
+
+  template <typename T>
+  FDLogger& operator<<(const T& val) {
+    if (!verbose_) {
+      return *this;
+    }
+    std::stringstream ss;
+    ss << val;
+    line_ += ss.str();
+    return *this;
+  }
+  FDLogger& operator<<(std::ostream& (*os)(std::ostream&));
+  ~FDLogger() {
+    if (!verbose_ && line_ != "") {
+      std::cout << line_ << std::endl;
+    }
+  }
+
+ private:
+  std::string line_;
+  std::string prefix_;
+  bool verbose_ = true;
+};
+
+FASTDEPLOY_DECL bool ReadBinaryFromFile(const std::string& file,
+                                        std::string* contents);
+
+#ifndef __REL_FILE__
+#define __REL_FILE__ __FILE__
+#endif
+
+#define FDERROR                                                \
+  FDLogger(true, "[ERROR]") << __REL_FILE__ << "(" << __LINE__ \
+                            << ")::" << __FUNCTION__ << "\t"
+
+#define FDWARNING                                                \
+  FDLogger(true, "[WARNING]") << __REL_FILE__ << "(" << __LINE__ \
+                              << ")::" << __FUNCTION__ << "\t"
+
+#define FDINFO                                                \
+  FDLogger(true, "[INFO]") << __REL_FILE__ << "(" << __LINE__ \
+                           << ")::" << __FUNCTION__ << "\t"
+
+#define FDASSERT(condition, format, ...)                        \
+  if (!(condition)) {                                           \
+    int n = std::snprintf(nullptr, 0, format, ##__VA_ARGS__);   \
+    std::vector<char> buffer(n + 1);                            \
+    std::snprintf(buffer.data(), n + 1, format, ##__VA_ARGS__); \
+    FDERROR << buffer.data() << std::endl;                      \
+    std::abort();                                               \
+  }
+
+///////// Basic Marco ///////////
+
+#define FD_PRIVATE_CASE_TYPE_USING_HINT(NAME, enum_type, type, HINT, ...) \
+  case enum_type: {                                                       \
+    using HINT = type;                                                    \
+    __VA_ARGS__();                                                        \
+    break;                                                                \
+  }
+
+#define FD_PRIVATE_CASE_TYPE(NAME, enum_type, type, ...) \
+  FD_PRIVATE_CASE_TYPE_USING_HINT(NAME, enum_type, type, data_t, __VA_ARGS__)
+
+// Visit different data type to match the corresponding function of FDTensor
+#define FD_VISIT_ALL_TYPES(TYPE, NAME, ...)                                    \
+  [&] {                                                                        \
+    const auto& __dtype__ = TYPE;                                              \
+    switch (__dtype__) {                                                       \
+      FD_PRIVATE_CASE_TYPE(NAME, ::fastdeploy::FDDataType::BOOL, bool,         \
+                           __VA_ARGS__)                                        \
+      FD_PRIVATE_CASE_TYPE(NAME, ::fastdeploy::FDDataType::INT32, int32_t,     \
+                           __VA_ARGS__)                                        \
+      FD_PRIVATE_CASE_TYPE(NAME, ::fastdeploy::FDDataType::INT64, int64_t,     \
+                           __VA_ARGS__)                                        \
+      FD_PRIVATE_CASE_TYPE(NAME, ::fastdeploy::FDDataType::FP32, float,        \
+                           __VA_ARGS__)                                        \
+      FD_PRIVATE_CASE_TYPE(NAME, ::fastdeploy::FDDataType::FP64, double,       \
+                           __VA_ARGS__)                                        \
+      default:                                                                 \
+        FDASSERT(                                                              \
+            false,                                                             \
+            "Invalid enum data type. Expect to accept data type BOOL, INT32, " \
+            "INT64, FP32, FP64, but receive type %s.",                         \
+            Str(__dtype__).c_str());                                           \
+    }                                                                          \
+  }()
+
+#define FD_VISIT_INT_FLOAT_TYPES(TYPE, NAME, ...)                             \
+  [&] {                                                                       \
+    const auto& __dtype__ = TYPE;                                             \
+    switch (__dtype__) {                                                      \
+      FD_PRIVATE_CASE_TYPE(NAME, ::fastdeploy::FDDataType::INT32, int32_t,    \
+                           __VA_ARGS__)                                       \
+      FD_PRIVATE_CASE_TYPE(NAME, ::fastdeploy::FDDataType::INT64, int64_t,    \
+                           __VA_ARGS__)                                       \
+      FD_PRIVATE_CASE_TYPE(NAME, ::fastdeploy::FDDataType::FP32, float,       \
+                           __VA_ARGS__)                                       \
+      FD_PRIVATE_CASE_TYPE(NAME, ::fastdeploy::FDDataType::FP64, double,      \
+                           __VA_ARGS__)                                       \
+      default:                                                                \
+        FDASSERT(false,                                                       \
+                 "Invalid enum data type. Expect to accept data type INT32, " \
+                 "INT64, FP32, FP64, but receive type %s.",                   \
+                 Str(__dtype__).c_str());                                     \
+    }                                                                         \
+  }()
+
+#define FD_VISIT_FLOAT_TYPES(TYPE, NAME, ...)                                \
+  [&] {                                                                      \
+    const auto& __dtype__ = TYPE;                                            \
+    switch (__dtype__) {                                                     \
+      FD_PRIVATE_CASE_TYPE(NAME, ::fastdeploy::FDDataType::FP32, float,      \
+                           __VA_ARGS__)                                      \
+      FD_PRIVATE_CASE_TYPE(NAME, ::fastdeploy::FDDataType::FP64, double,     \
+                           __VA_ARGS__)                                      \
+      default:                                                               \
+        FDASSERT(false,                                                      \
+                 "Invalid enum data type. Expect to accept data type FP32, " \
+                 "FP64, but receive type %s.",                               \
+                 Str(__dtype__).c_str());                                    \
+    }                                                                        \
+  }()
+
+#define FD_VISIT_INT_TYPES(TYPE, NAME, ...)                                   \
+  [&] {                                                                       \
+    const auto& __dtype__ = TYPE;                                             \
+    switch (__dtype__) {                                                      \
+      FD_PRIVATE_CASE_TYPE(NAME, ::fastdeploy::FDDataType::INT32, int32_t,    \
+                           __VA_ARGS__)                                       \
+      FD_PRIVATE_CASE_TYPE(NAME, ::fastdeploy::FDDataType::INT64, int64_t,    \
+                           __VA_ARGS__)                                       \
+      default:                                                                \
+        FDASSERT(false,                                                       \
+                 "Invalid enum data type. Expect to accept data type INT32, " \
+                 "INT64, but receive type %s.",                               \
+                 Str(__dtype__).c_str());                                     \
+    }                                                                         \
+  }()
+
+FASTDEPLOY_DECL std::vector<int64_t> GetStride(
+    const std::vector<int64_t>& dims);
+
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/vision.h

@@ -0,0 +1,57 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#pragma once
+
+#include "fastdeploy/core/config.h"
+#ifdef ENABLE_VISION
+#include "fastdeploy/vision/classification/contrib/yolov5cls.h"
+#include "fastdeploy/vision/classification/ppcls/model.h"
+#include "fastdeploy/vision/classification/contrib/resnet.h"
+#include "fastdeploy/vision/detection/contrib/nanodet_plus.h"
+#include "fastdeploy/vision/detection/contrib/scaledyolov4.h"
+#include "fastdeploy/vision/detection/contrib/yolor.h"
+#include "fastdeploy/vision/detection/contrib/yolov5/yolov5.h"
+#include "fastdeploy/vision/detection/contrib/yolov5lite.h"
+#include "fastdeploy/vision/detection/contrib/yolov6.h"
+#include "fastdeploy/vision/detection/contrib/yolov7.h"
+#include "fastdeploy/vision/detection/contrib/yolov7end2end_ort.h"
+#include "fastdeploy/vision/detection/contrib/yolov7end2end_trt.h"
+#include "fastdeploy/vision/detection/contrib/yolox.h"
+#include "fastdeploy/vision/detection/ppdet/model.h"
+#include "fastdeploy/vision/facedet/contrib/retinaface.h"
+#include "fastdeploy/vision/facedet/contrib/scrfd.h"
+#include "fastdeploy/vision/facedet/contrib/ultraface.h"
+#include "fastdeploy/vision/facedet/contrib/yolov5face.h"
+#include "fastdeploy/vision/facealign/contrib/pfld.h"
+#include "fastdeploy/vision/faceid/contrib/adaface.h"
+#include "fastdeploy/vision/faceid/contrib/arcface.h"
+#include "fastdeploy/vision/faceid/contrib/cosface.h"
+#include "fastdeploy/vision/faceid/contrib/insightface_rec.h"
+#include "fastdeploy/vision/faceid/contrib/partial_fc.h"
+#include "fastdeploy/vision/faceid/contrib/vpl.h"
+#include "fastdeploy/vision/keypointdet/pptinypose/pptinypose.h"
+#include "fastdeploy/vision/matting/contrib/modnet.h"
+#include "fastdeploy/vision/matting/contrib/rvm.h"
+#include "fastdeploy/vision/matting/ppmatting/ppmatting.h"
+#include "fastdeploy/vision/ocr/ppocr/classifier.h"
+#include "fastdeploy/vision/ocr/ppocr/dbdetector.h"
+#include "fastdeploy/vision/ocr/ppocr/ppocr_v2.h"
+#include "fastdeploy/vision/ocr/ppocr/ppocr_v3.h"
+#include "fastdeploy/vision/ocr/ppocr/recognizer.h"
+#include "fastdeploy/vision/segmentation/ppseg/model.h"
+#include "fastdeploy/vision/tracking/pptracking/model.h"
+#include "fastdeploy/vision/headpose/contrib/fsanet.h"
+#endif
+
+#include "fastdeploy/vision/visualize/visualize.h"

3rdparty/include/fastdeploy/vision/common/processors/base.h

@@ -0,0 +1,76 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "fastdeploy/utils/utils.h"
+#include "fastdeploy/vision/common/processors/mat.h"
+#include "opencv2/highgui/highgui.hpp"
+#include "opencv2/imgproc/imgproc.hpp"
+#include <unordered_map>
+
+namespace fastdeploy {
+namespace vision {
+
+/*! @brief Enable using FlyCV to process image while deploy vision models.
+ * Currently, FlyCV in only available on ARM(Linux aarch64/Android), so will
+ * fallback to using OpenCV in other platform
+ */
+FASTDEPLOY_DECL void EnableFlyCV();
+
+/// Disable using FlyCV to process image while deploy vision models.
+FASTDEPLOY_DECL void DisableFlyCV();
+
+/*! @brief Set the cpu num threads of ProcLib. The cpu num threads
+ *  of FlyCV and OpenCV is 2 by default.
+ */
+FASTDEPLOY_DECL void SetProcLibCpuNumThreads(int threads);
+
+class FASTDEPLOY_DECL Processor {
+ public:
+  // default_lib has the highest priority
+  // all the function in `processor` will force to use
+  // default_lib if this flag is set.
+  // DEFAULT means this flag is not set
+  // static ProcLib default_lib;
+
+  virtual std::string Name() = 0;
+
+  virtual bool ImplByOpenCV(Mat* mat) {
+    FDERROR << Name() << " Not Implement Yet." << std::endl;
+    return false;
+  }
+
+  virtual bool ImplByFlyCV(Mat* mat) {
+    return ImplByOpenCV(mat);
+  }
+
+  virtual bool ImplByCuda(Mat* mat) {
+    return ImplByOpenCV(mat);
+  }
+
+  virtual bool operator()(Mat* mat, ProcLib lib = ProcLib::DEFAULT);
+
+ protected:
+  FDTensor* UpdateAndGetReusedBuffer(
+      const std::vector<int64_t>& new_shape, const int& opencv_dtype,
+      const std::string& buffer_name, const Device& new_device = Device::CPU,
+      const bool& use_pinned_memory = false);
+
+ private:
+  std::unordered_map<std::string, FDTensor> reused_buffers_;
+};
+
+}  // namespace vision
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/vision/common/processors/cast.h

@@ -0,0 +1,39 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "fastdeploy/vision/common/processors/base.h"
+
+namespace fastdeploy {
+namespace vision {
+
+class FASTDEPLOY_DECL Cast : public Processor {
+ public:
+  explicit Cast(const std::string& dtype = "float") : dtype_(dtype) {}
+  bool ImplByOpenCV(Mat* mat);
+#ifdef ENABLE_FLYCV
+  bool ImplByFlyCV(Mat* mat);
+#endif
+  std::string Name() { return "Cast"; }
+  static bool Run(Mat* mat, const std::string& dtype,
+                  ProcLib lib = ProcLib::DEFAULT);
+
+  std::string GetDtype() const { return dtype_; }
+
+ private:
+  std::string dtype_;
+};
+}  // namespace vision
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/vision/common/processors/center_crop.h

@@ -0,0 +1,40 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "fastdeploy/vision/common/processors/base.h"
+
+namespace fastdeploy {
+namespace vision {
+
+class FASTDEPLOY_DECL CenterCrop : public Processor {
+ public:
+  CenterCrop(int width, int height) : height_(height), width_(width) {}
+  bool ImplByOpenCV(Mat* mat);
+#ifdef ENABLE_FLYCV
+  bool ImplByFlyCV(Mat* mat);
+#endif
+  std::string Name() { return "CenterCrop"; }
+
+  static bool Run(Mat* mat, const int& width, const int& height,
+                  ProcLib lib = ProcLib::DEFAULT);
+
+ private:
+  int height_;
+  int width_;
+};
+
+}  // namespace vision
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/vision/common/processors/color_space_convert.h

@@ -0,0 +1,44 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "fastdeploy/vision/common/processors/base.h"
+
+namespace fastdeploy {
+namespace vision {
+
+class FASTDEPLOY_DECL BGR2RGB : public Processor {
+ public:
+  bool ImplByOpenCV(Mat* mat);
+#ifdef ENABLE_FLYCV
+  bool ImplByFlyCV(Mat* mat);
+#endif
+  virtual std::string Name() { return "BGR2RGB"; }
+
+  static bool Run(Mat* mat, ProcLib lib = ProcLib::DEFAULT);
+};
+
+class FASTDEPLOY_DECL RGB2BGR : public Processor {
+ public:
+  bool ImplByOpenCV(Mat* mat);
+#ifdef ENABLE_FLYCV
+  bool ImplByFlyCV(Mat* mat);
+#endif
+  std::string Name() { return "RGB2BGR"; }
+
+  static bool Run(Mat* mat, ProcLib lib = ProcLib::DEFAULT);
+};
+}  // namespace vision
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/vision/common/processors/convert.h

@@ -0,0 +1,42 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "fastdeploy/vision/common/processors/base.h"
+
+namespace fastdeploy {
+namespace vision {
+class FASTDEPLOY_DECL Convert : public Processor {
+ public:
+  Convert(const std::vector<float>& alpha, const std::vector<float>& beta);
+
+  bool ImplByOpenCV(Mat* mat);
+#ifdef ENABLE_FLYCV
+  bool ImplByFlyCV(Mat* mat);
+#endif
+  std::string Name() { return "Convert"; }
+
+  // Compute `result = mat * alpha + beta` directly by channel.
+  // The default behavior is the same as OpenCV's convertTo method.
+  static bool Run(Mat* mat, const std::vector<float>& alpha,
+                  const std::vector<float>& beta,
+                  ProcLib lib = ProcLib::DEFAULT);
+
+ private:
+  std::vector<float> alpha_;
+  std::vector<float> beta_;
+};
+}  // namespace vision
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/vision/common/processors/convert_and_permute.h

@@ -0,0 +1,66 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "fastdeploy/vision/common/processors/base.h"
+
+namespace fastdeploy {
+namespace vision {
+class FASTDEPLOY_DECL ConvertAndPermute : public Processor {
+ public:
+  ConvertAndPermute(const std::vector<float>& alpha = std::vector<float>(),
+                    const std::vector<float>& beta = std::vector<float>(),
+                    bool swap_rb = false);
+  bool ImplByOpenCV(FDMat* mat);
+#ifdef ENABLE_FLYCV
+  bool ImplByFlyCV(FDMat* mat);
+#endif
+  std::string Name() { return "ConvertAndPermute"; }
+
+  static bool Run(FDMat* mat, const std::vector<float>& alpha,
+                  const std::vector<float>& beta, bool swap_rb = false,
+                  ProcLib lib = ProcLib::DEFAULT);
+
+  std::vector<float> GetAlpha() const { return alpha_; }
+
+  void SetAlpha(const std::vector<float>& alpha) {
+    alpha_.clear();
+    std::vector<float>().swap(alpha_);
+    alpha_.assign(alpha.begin(), alpha.end());
+  }
+
+  std::vector<float> GetBeta() const { return beta_; }
+
+  void SetBeta(const std::vector<float>& beta) {
+    beta_.clear();
+    std::vector<float>().swap(beta_);
+    beta_.assign(beta.begin(), beta.end());
+  }
+
+  bool GetSwapRB() {
+    return swap_rb_;
+  }
+
+  void SetSwapRB(bool swap_rb) {
+    swap_rb_ = swap_rb;
+  }
+
+ private:
+  std::vector<float> alpha_;
+  std::vector<float> beta_;
+  bool swap_rb_;
+};
+}  // namespace vision
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/vision/common/processors/crop.h

@@ -0,0 +1,49 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "fastdeploy/vision/common/processors/base.h"
+
+namespace fastdeploy {
+namespace vision {
+
+class FASTDEPLOY_DECL Crop : public Processor {
+ public:
+  Crop(int offset_w, int offset_h, int width, int height) {
+    offset_w_ = offset_w;
+    offset_h_ = offset_h;
+    width_ = width;
+    height_ = height;
+  }
+
+  bool ImplByOpenCV(Mat* mat);
+
+#ifdef ENABLE_FLYCV
+  bool ImplByFlyCV(Mat* mat);
+#endif
+  std::string Name() { return "Crop"; }
+
+  static bool Run(Mat* mat, int offset_w, int offset_h, int width, int height,
+                  ProcLib lib = ProcLib::DEFAULT);
+
+ private:
+  int offset_w_;
+  int offset_h_;
+  int height_;
+  int width_;
+};
+
+}  // namespace vision
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/vision/common/processors/hwc2chw.h

@@ -0,0 +1,33 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "fastdeploy/vision/common/processors/base.h"
+
+namespace fastdeploy {
+namespace vision {
+
+class FASTDEPLOY_DECL HWC2CHW : public Processor {
+ public:
+  bool ImplByOpenCV(Mat* mat);
+#ifdef ENABLE_FLYCV
+  bool ImplByFlyCV(Mat* mat);
+#endif
+  std::string Name() { return "HWC2CHW"; }
+
+  static bool Run(Mat* mat, ProcLib lib = ProcLib::DEFAULT);
+};
+}  // namespace vision
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/vision/common/processors/limit_by_stride.h

@@ -0,0 +1,44 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "fastdeploy/vision/common/processors/base.h"
+
+namespace fastdeploy {
+namespace vision {
+
+class FASTDEPLOY_DECL LimitByStride : public Processor {
+ public:
+  explicit LimitByStride(int stride = 32, int interp = 1) {
+    stride_ = stride;
+    interp_ = interp;
+  }
+
+  // Resize Mat* mat to make the size divisible by stride_.
+  bool ImplByOpenCV(Mat* mat);
+#ifdef ENABLE_FLYCV
+  bool ImplByFlyCV(Mat* mat);
+#endif
+  std::string Name() { return "LimitByStride"; }
+
+  static bool Run(Mat* mat, int stride = 32, int interp = 1,
+                  ProcLib lib = ProcLib::DEFAULT);
+
+ private:
+  int interp_;
+  int stride_;
+};
+}  // namespace vision
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/vision/common/processors/limit_short.h

@@ -0,0 +1,51 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "fastdeploy/vision/common/processors/base.h"
+
+namespace fastdeploy {
+namespace vision {
+
+class LimitShort : public Processor {
+ public:
+  explicit LimitShort(int max_short = -1, int min_short = -1, int interp = 1) {
+    max_short_ = max_short;
+    min_short_ = min_short;
+    interp_ = interp;
+  }
+
+  // Limit the short edge of image.
+  // If the short edge is larger than max_short_, resize the short edge
+  // to max_short_, while scale the long edge proportionally.
+  // If the short edge is smaller than min_short_, resize the short edge
+  // to min_short_, while scale the long edge proportionally.
+  bool ImplByOpenCV(Mat* mat);
+#ifdef ENABLE_FLYCV
+  bool ImplByFlyCV(Mat* mat);
+#endif
+  std::string Name() { return "LimitShort"; }
+
+  static bool Run(Mat* mat, int max_short = -1, int min_short = -1,
+                  int interp = 1, ProcLib lib = ProcLib::DEFAULT);
+  int GetMaxShort() const { return max_short_; }
+
+ private:
+  int max_short_;
+  int min_short_;
+  int interp_;
+};
+}  // namespace vision
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/vision/common/processors/mat.h

@@ -0,0 +1,162 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#pragma once
+#include "fastdeploy/core/fd_tensor.h"
+#include "fastdeploy/vision/common/processors/utils.h"
+#include "fastdeploy/vision/common/processors/proc_lib.h"
+#include "opencv2/core/core.hpp"
+
+namespace fastdeploy {
+namespace vision {
+
+enum Layout { HWC, CHW };
+
+
+struct FASTDEPLOY_DECL Mat {
+  explicit Mat(const cv::Mat& mat) {
+    cpu_mat = mat;
+    layout = Layout::HWC;
+    height = cpu_mat.rows;
+    width = cpu_mat.cols;
+    channels = cpu_mat.channels();
+    mat_type = ProcLib::OPENCV;
+  }
+
+#ifdef ENABLE_FLYCV
+  explicit Mat(const fcv::Mat& mat) {
+    fcv_mat = mat;
+    layout = Layout::HWC;
+    height = fcv_mat.height();
+    width = fcv_mat.width();
+    channels = fcv_mat.channels();
+    mat_type = ProcLib::FLYCV;
+  }
+#endif
+
+  Mat(const Mat& mat) = default;
+  Mat& operator=(const Mat& mat) = default;
+
+  // Careful if you use this interface
+  // this only used if you don't want to write
+  // the original data, and write to a new cv::Mat
+  // then replace the old cv::Mat of this structure
+  void SetMat(const cv::Mat& mat) {
+    cpu_mat = mat;
+    mat_type = ProcLib::OPENCV;
+  }
+
+  cv::Mat* GetOpenCVMat() {
+    if (mat_type == ProcLib::OPENCV) {
+      return &cpu_mat;
+    } else if (mat_type == ProcLib::FLYCV) {
+#ifdef ENABLE_FLYCV
+      // Just a reference to fcv_mat, zero copy. After you
+      // call this method, cpu_mat and fcv_mat will point
+      // to the same memory buffer.
+      cpu_mat = ConvertFlyCVMatToOpenCV(fcv_mat);
+      mat_type = ProcLib::OPENCV;
+      return &cpu_mat;
+#else
+      FDASSERT(false, "FastDeploy didn't compiled with FlyCV!");
+#endif
+    } else {
+      FDASSERT(false, "The mat_type of custom Mat can not be ProcLib::DEFAULT");
+    }
+  }
+
+#ifdef ENABLE_FLYCV
+  void SetMat(const fcv::Mat& mat) {
+    fcv_mat = mat;
+    mat_type = ProcLib::FLYCV;
+  }
+
+  fcv::Mat* GetFlyCVMat() {
+    if (mat_type == ProcLib::FLYCV) {
+      return &fcv_mat;
+    } else if (mat_type == ProcLib::OPENCV) {
+      // Just a reference to cpu_mat, zero copy. After you
+      // call this method, fcv_mat and cpu_mat will point
+      // to the same memory buffer.
+      fcv_mat = ConvertOpenCVMatToFlyCV(cpu_mat);
+      mat_type = ProcLib::FLYCV;
+      return &fcv_mat;
+    } else {
+      FDASSERT(false, "The mat_type of custom Mat can not be ProcLib::DEFAULT");
+    }
+  }
+#endif
+
+  void* Data();
+
+ private:
+  int channels;
+  int height;
+  int width;
+  cv::Mat cpu_mat;
+#ifdef ENABLE_FLYCV
+  fcv::Mat fcv_mat;
+#endif
+
+ public:
+  FDDataType Type();
+  int Channels() const { return channels; }
+  int Width() const { return width; }
+  int Height() const { return height; }
+  void SetChannels(int s) { channels = s; }
+  void SetWidth(int w) { width = w; }
+  void SetHeight(int h) { height = h; }
+
+  // Transfer the vision::Mat to FDTensor
+  void ShareWithTensor(FDTensor* tensor);
+  // Only support copy to cpu tensor now
+  bool CopyToTensor(FDTensor* tensor);
+
+  // Debug functions
+  // TODO(jiangjiajun) Develop a right process pipeline with c++
+  // is not a easy things, Will add more debug function here to
+  // help debug processed image. This function will print shape
+  // and mean of each channels of the Mat
+  void PrintInfo(const std::string& flag);
+
+  ProcLib mat_type = ProcLib::OPENCV;
+  Layout layout = Layout::HWC;
+  Device device = Device::CPU;
+
+  // Create FD Mat from FD Tensor. This method only create a
+  // new FD Mat with zero copy and it's data pointer is reference
+  // to the original memory buffer of input FD Tensor. Carefully,
+  // any operation on this Mat may change memory that points to
+  // FDTensor. We assume that the memory Mat points to is mutable.
+  // This method will create a FD Mat according to current global
+  // default ProcLib (OPENCV,FLYCV,...).
+  static Mat Create(const FDTensor& tensor);
+  static Mat Create(const FDTensor& tensor, ProcLib lib);
+  static Mat Create(int height, int width, int channels,
+                    FDDataType type, void* data);
+  static Mat Create(int height, int width, int channels,
+                    FDDataType type, void* data, ProcLib lib);
+};
+
+typedef Mat FDMat;
+/*
+ * @brief Wrap a cv::Mat to FDMat, there's no memory copy, memory buffer is managed by user
+ */
+FASTDEPLOY_DECL FDMat WrapMat(const cv::Mat& image);
+/*
+ * Warp a vector<cv::Mat> to vector<FDMat>, there's no memory copy, memory buffer is managed by user
+ */
+FASTDEPLOY_DECL std::vector<FDMat> WrapMat(const std::vector<cv::Mat>& images);
+
+}  // namespace vision
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/vision/common/processors/normalize.h

@@ -0,0 +1,67 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "fastdeploy/vision/common/processors/base.h"
+
+namespace fastdeploy {
+namespace vision {
+class FASTDEPLOY_DECL Normalize : public Processor {
+ public:
+  Normalize(const std::vector<float>& mean, const std::vector<float>& std,
+            bool is_scale = true,
+            const std::vector<float>& min = std::vector<float>(),
+            const std::vector<float>& max = std::vector<float>(),
+            bool swap_rb = false);
+  bool ImplByOpenCV(Mat* mat);
+#ifdef ENABLE_FLYCV
+  bool ImplByFlyCV(Mat* mat);
+#endif
+  std::string Name() { return "Normalize"; }
+
+  // While use normalize, it is more recommend not use this function
+  // this function will need to compute result = ((mat / 255) - mean) / std
+  // if we use the following method
+  // ```
+  // auto norm = Normalize(...)
+  // norm(mat)
+  // ```
+  // There will be some precomputation in contruct function
+  // and the `norm(mat)` only need to compute result = mat * alpha + beta
+  // which will reduce lots of time
+  static bool Run(Mat* mat, const std::vector<float>& mean,
+                  const std::vector<float>& std, bool is_scale = true,
+                  const std::vector<float>& min = std::vector<float>(),
+                  const std::vector<float>& max = std::vector<float>(),
+                  ProcLib lib = ProcLib::DEFAULT, bool swap_rb = false);
+
+  std::vector<float> GetAlpha() const { return alpha_; }
+  std::vector<float> GetBeta() const { return beta_; }
+
+  bool GetSwapRB() {
+    return swap_rb_;
+  }
+
+  void SetSwapRB(bool swap_rb) {
+    swap_rb_ = swap_rb;
+  }
+
+ private:
+  std::vector<float> alpha_;
+  std::vector<float> beta_;
+  bool swap_rb_;
+};
+}  // namespace vision
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/vision/common/processors/normalize_and_permute.h

@@ -0,0 +1,79 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "fastdeploy/vision/common/processors/base.h"
+
+namespace fastdeploy {
+namespace vision {
+class FASTDEPLOY_DECL NormalizeAndPermute : public Processor {
+ public:
+  NormalizeAndPermute(const std::vector<float>& mean,
+                      const std::vector<float>& std, bool is_scale = true,
+                      const std::vector<float>& min = std::vector<float>(),
+                      const std::vector<float>& max = std::vector<float>(),
+                      bool swap_rb = false);
+  bool ImplByOpenCV(Mat* mat);
+#ifdef ENABLE_FLYCV
+  bool ImplByFlyCV(Mat* mat);
+#endif
+#ifdef WITH_GPU
+  bool ImplByCuda(Mat* mat);
+#endif
+  std::string Name() { return "NormalizeAndPermute"; }
+
+  // While use normalize, it is more recommend not use this function
+  // this function will need to compute result = ((mat / 255) - mean) / std
+  // if we use the following method
+  // ```
+  // auto norm = Normalize(...)
+  // norm(mat)
+  // ```
+  // There will be some precomputation in contruct function
+  // and the `norm(mat)` only need to compute result = mat * alpha + beta
+  // which will reduce lots of time
+  static bool Run(Mat* mat, const std::vector<float>& mean,
+                  const std::vector<float>& std, bool is_scale = true,
+                  const std::vector<float>& min = std::vector<float>(),
+                  const std::vector<float>& max = std::vector<float>(),
+                  ProcLib lib = ProcLib::DEFAULT, bool swap_rb = false);
+
+  void SetAlpha(const std::vector<float>& alpha) {
+    alpha_.clear();
+    std::vector<float>().swap(alpha_);
+    alpha_.assign(alpha.begin(), alpha.end());
+  }
+
+  void SetBeta(const std::vector<float>& beta) {
+    beta_.clear();
+    std::vector<float>().swap(beta_);
+    beta_.assign(beta.begin(), beta.end());
+  }
+
+  bool GetSwapRB() {
+    return swap_rb_;
+  }
+
+  void SetSwapRB(bool swap_rb) {
+    swap_rb_ = swap_rb;
+  }
+
+ private:
+  std::vector<float> alpha_;
+  std::vector<float> beta_;
+  bool swap_rb_;
+};
+}  // namespace vision
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/vision/common/processors/pad.h

@@ -0,0 +1,50 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "fastdeploy/vision/common/processors/base.h"
+
+namespace fastdeploy {
+namespace vision {
+
+class FASTDEPLOY_DECL Pad : public Processor {
+ public:
+  Pad(int top, int bottom, int left, int right,
+      const std::vector<float>& value) {
+    top_ = top;
+    bottom_ = bottom;
+    left_ = left;
+    right_ = right;
+    value_ = value;
+  }
+  bool ImplByOpenCV(Mat* mat);
+#ifdef ENABLE_FLYCV
+  bool ImplByFlyCV(Mat* mat);
+#endif
+  std::string Name() { return "Pad"; }
+
+  static bool Run(Mat* mat, const int& top, const int& bottom, const int& left,
+                  const int& right, const std::vector<float>& value,
+                  ProcLib lib = ProcLib::DEFAULT);
+
+ private:
+  int top_;
+  int bottom_;
+  int left_;
+  int right_;
+  std::vector<float> value_;
+};
+}  // namespace vision
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/vision/common/processors/pad_to_size.h

@@ -0,0 +1,46 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "fastdeploy/vision/common/processors/base.h"
+
+namespace fastdeploy {
+namespace vision {
+
+class FASTDEPLOY_DECL PadToSize : public Processor {
+ public:
+  // only support pad with right-bottom padding mode
+  PadToSize(int width, int height, const std::vector<float>& value) {
+    width_ = width;
+    height_ = height;
+    value_ = value;
+  }
+  bool ImplByOpenCV(Mat* mat);
+#ifdef ENABLE_FLYCV
+  bool ImplByFlyCV(Mat* mat);
+#endif
+  std::string Name() { return "PadToSize"; }
+
+  static bool Run(Mat* mat, int width, int height,
+                  const std::vector<float>& value,
+                  ProcLib lib = ProcLib::DEFAULT);
+
+ private:
+  int width_;
+  int height_;
+  std::vector<float> value_;
+};
+}  // namespace vision
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/vision/common/processors/proc_lib.h

@@ -0,0 +1,34 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+#include "fastdeploy/utils/utils.h"
+
+namespace fastdeploy {
+namespace vision {
+
+enum class FASTDEPLOY_DECL ProcLib { DEFAULT, OPENCV, FLYCV, CUDA };
+
+FASTDEPLOY_DECL std::ostream& operator<<(std::ostream& out, const ProcLib& p);
+
+struct FASTDEPLOY_DECL DefaultProcLib {
+  // default_lib has the highest priority
+  // all the function in `processor` will force to use
+  // default_lib if this flag is set.
+  // DEFAULT means this flag is not set
+  static ProcLib default_lib;
+};
+
+}  // namespace vision
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/vision/common/processors/resize.h

@@ -0,0 +1,63 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "fastdeploy/vision/common/processors/base.h"
+
+namespace fastdeploy {
+namespace vision {
+
+class FASTDEPLOY_DECL Resize : public Processor {
+ public:
+  Resize(int width, int height, float scale_w = -1.0, float scale_h = -1.0,
+         int interp = 1, bool use_scale = false) {
+    width_ = width;
+    height_ = height;
+    scale_w_ = scale_w;
+    scale_h_ = scale_h;
+    interp_ = interp;
+    use_scale_ = use_scale;
+  }
+
+  bool ImplByOpenCV(Mat* mat);
+#ifdef ENABLE_FLYCV
+  bool ImplByFlyCV(Mat* mat);
+#endif
+  std::string Name() { return "Resize"; }
+
+  static bool Run(Mat* mat, int width, int height, float scale_w = -1.0,
+                  float scale_h = -1.0, int interp = 1, bool use_scale = false,
+                  ProcLib lib = ProcLib::DEFAULT);
+
+  bool SetWidthAndHeight(int width, int height) {
+    width_ = width;
+    height_ = height;
+    return true;
+  }
+
+  std::tuple<int, int> GetWidthAndHeight() {
+    return std::make_tuple(width_, height_);
+  }
+
+ private:
+  int width_;
+  int height_;
+  float scale_w_ = -1.0;
+  float scale_h_ = -1.0;
+  int interp_ = 1;
+  bool use_scale_ = false;
+};
+}  // namespace vision
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/vision/common/processors/resize_by_short.h

@@ -0,0 +1,50 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "fastdeploy/vision/common/processors/base.h"
+
+namespace fastdeploy {
+namespace vision {
+
+class FASTDEPLOY_DECL ResizeByShort : public Processor {
+ public:
+  ResizeByShort(int target_size, int interp = 1, bool use_scale = true,
+                const std::vector<int>& max_hw = std::vector<int>()) {
+    target_size_ = target_size;
+    max_hw_ = max_hw;
+    interp_ = interp;
+    use_scale_ = use_scale;
+  }
+  bool ImplByOpenCV(Mat* mat);
+#ifdef ENABLE_FLYCV
+  bool ImplByFlyCV(Mat* mat);
+#endif
+  std::string Name() { return "ResizeByShort"; }
+
+  static bool Run(Mat* mat, int target_size, int interp = 1,
+                  bool use_scale = true,
+                  const std::vector<int>& max_hw = std::vector<int>(),
+                  ProcLib lib = ProcLib::DEFAULT);
+
+ private:
+  double GenerateScale(const int origin_w, const int origin_h);
+  int target_size_;
+  std::vector<int> max_hw_;
+  int interp_;
+  bool use_scale_;
+};
+}  // namespace vision
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/vision/common/processors/stride_pad.h

@@ -0,0 +1,44 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "fastdeploy/vision/common/processors/base.h"
+
+namespace fastdeploy {
+namespace vision {
+
+class FASTDEPLOY_DECL StridePad : public Processor {
+ public:
+  // only support pad with left-top padding mode
+  StridePad(int stride, const std::vector<float>& value) {
+    stride_ = stride;
+    value_ = value;
+  }
+  bool ImplByOpenCV(Mat* mat);
+#ifdef ENABLE_FLYCV
+  bool ImplByFlyCV(Mat* mat);
+#endif
+  std::string Name() { return "StridePad"; }
+
+  static bool Run(Mat* mat, int stride,
+                  const std::vector<float>& value = std::vector<float>(),
+                  ProcLib lib = ProcLib::DEFAULT);
+
+ private:
+  int stride_ = 32;
+  std::vector<float> value_;
+};
+}  // namespace vision
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/vision/common/processors/transform.h

@@ -0,0 +1,50 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "fastdeploy/vision/common/processors/cast.h"
+#include "fastdeploy/vision/common/processors/center_crop.h"
+#include "fastdeploy/vision/common/processors/color_space_convert.h"
+#include "fastdeploy/vision/common/processors/convert.h"
+#include "fastdeploy/vision/common/processors/convert_and_permute.h"
+#include "fastdeploy/vision/common/processors/crop.h"
+#include "fastdeploy/vision/common/processors/hwc2chw.h"
+#include "fastdeploy/vision/common/processors/limit_by_stride.h"
+#include "fastdeploy/vision/common/processors/limit_short.h"
+#include "fastdeploy/vision/common/processors/normalize.h"
+#include "fastdeploy/vision/common/processors/normalize_and_permute.h"
+#include "fastdeploy/vision/common/processors/pad.h"
+#include "fastdeploy/vision/common/processors/pad_to_size.h"
+#include "fastdeploy/vision/common/processors/resize.h"
+#include "fastdeploy/vision/common/processors/resize_by_short.h"
+#include "fastdeploy/vision/common/processors/stride_pad.h"
+#include "fastdeploy/vision/common/processors/warp_affine.h"
+#include <unordered_set>
+
+namespace fastdeploy {
+namespace vision {
+
+void FuseTransforms(std::vector<std::shared_ptr<Processor>>* processors);
+
+// Fuse Normalize + Cast(Float) to Normalize
+void FuseNormalizeCast(std::vector<std::shared_ptr<Processor>>* processors);
+// Fuse Normalize + HWC2CHW to NormalizeAndPermute
+void FuseNormalizeHWC2CHW(std::vector<std::shared_ptr<Processor>>* processors);
+// Fuse Normalize + Color Convert
+void FuseNormalizeColorConvert(
+    std::vector<std::shared_ptr<Processor>>* processors);
+
+}  // namespace vision
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/vision/common/processors/utils.h

@@ -0,0 +1,53 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "fastdeploy/core/fd_tensor.h"
+#include "fastdeploy/utils/utils.h"
+#include "opencv2/core/core.hpp"
+
+#ifdef ENABLE_FLYCV
+#include "flycv.h"  // NOLINT
+#endif
+
+namespace fastdeploy {
+namespace vision {
+
+// Convert data type of opencv to FDDataType
+FDDataType OpenCVDataTypeToFD(int type);
+// Create data type of opencv by FDDataType
+int CreateOpenCVDataType(FDDataType type, int channel = 1);
+#ifdef ENABLE_FLYCV
+// Convert data type of flycv to FDDataType
+FDDataType FlyCVDataTypeToFD(fcv::FCVImageType type);
+// Create data type of flycv by FDDataType
+fcv::FCVImageType CreateFlyCVDataType(FDDataType type, int channel = 1);
+// Convert cv::Mat to fcv::Mat
+fcv::Mat ConvertOpenCVMatToFlyCV(cv::Mat& im);
+// Convert fcv::Mat to fcv::mat
+cv::Mat ConvertFlyCVMatToOpenCV(fcv::Mat& fim);
+#endif
+
+// Create zero copy OpenCV/FlyCV Mat from FD Tensor / Buffer
+cv::Mat CreateZeroCopyOpenCVMatFromBuffer(int height, int width,
+  int channels, FDDataType type, void* data);
+cv::Mat CreateZeroCopyOpenCVMatFromTensor(const FDTensor& tensor);
+#ifdef ENABLE_FLYCV
+fcv::Mat CreateZeroCopyFlyCVMatFromBuffer(int height, int width,
+  int channels, FDDataType type, void* data);
+fcv::Mat CreateZeroCopyFlyCVMatFromTensor(const FDTensor& tensor);
+#endif
+}  // namespace vision
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/vision/common/processors/warp_affine.h

@@ -0,0 +1,61 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "fastdeploy/vision/common/processors/base.h"
+
+namespace fastdeploy {
+namespace vision {
+
+class WarpAffine : public Processor {
+ public:
+  WarpAffine(const cv::Mat& trans_matrix, int width, int height, int interp = 1,
+             int border_mode = 0,
+             const cv::Scalar& borderValue = cv::Scalar()) {
+    trans_matrix_ = trans_matrix;
+    width_ = width;
+    height_ = height;
+    interp_ = interp;
+    border_mode_ = border_mode;
+    borderValue_ = borderValue;
+  }
+
+  bool ImplByOpenCV(Mat* mat);
+  std::string Name() { return "WarpAffine"; }
+
+  bool SetTransformMatrix(const cv::Mat& trans_matrix) {
+    trans_matrix_ = trans_matrix;
+    return true;
+  }
+
+  std::tuple<int, int> GetWidthAndHeight() {
+    return std::make_tuple(width_, height_);
+  }
+
+  static bool Run(Mat* mat, const cv::Mat& trans_matrix, int width, int height,
+                  int interp = 1, int border_mode = 0,
+                  const cv::Scalar& borderValue = cv::Scalar(),
+                  ProcLib lib = ProcLib::DEFAULT);
+
+ private:
+  cv::Mat trans_matrix_;
+  int width_;
+  int height_;
+  int interp_;
+  int border_mode_;
+  cv::Scalar borderValue_;
+};
+}  // namespace vision
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/vision/common/result.h

@@ -0,0 +1,347 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#pragma once
+#include "fastdeploy/fastdeploy_model.h"
+#include "opencv2/core/core.hpp"
+#include <set>
+
+namespace fastdeploy {
+/** \brief All C++ FastDeploy Vision Models APIs are defined inside this namespace
+*
+*/
+namespace vision {
+enum FASTDEPLOY_DECL ResultType {
+  UNKNOWN_RESULT,
+  CLASSIFY,
+  DETECTION,
+  SEGMENTATION,
+  OCR,
+  MOT,
+  FACE_DETECTION,
+  FACE_ALIGNMENT,
+  FACE_RECOGNITION,
+  MATTING,
+  MASK,
+  KEYPOINT_DETECTION,
+  HEADPOSE,
+};
+
+struct FASTDEPLOY_DECL BaseResult {
+  ResultType type = ResultType::UNKNOWN_RESULT;
+};
+
+/*! @brief Classify result structure for all the image classify models
+ */
+struct FASTDEPLOY_DECL ClassifyResult : public BaseResult {
+  ClassifyResult() = default;
+  /// Classify result for an image
+  std::vector<int32_t> label_ids;
+  /// The confidence for each classify result
+  std::vector<float> scores;
+  ResultType type = ResultType::CLASSIFY;
+
+  /// Clear result
+  void Clear();
+
+  /// Copy constructor
+  ClassifyResult(const ClassifyResult& other) = default;
+  /// Move assignment
+  ClassifyResult& operator=(ClassifyResult&& other);
+
+  /// Debug function, convert the result to string to print
+  std::string Str();
+};
+
+/*! Mask structure, used in DetectionResult for instance segmentation models
+ */
+struct FASTDEPLOY_DECL Mask : public BaseResult {
+  /// Mask data buffer
+  std::vector<int32_t> data;
+  /// Shape of mask
+  std::vector<int64_t> shape;  // (H,W) ...
+  ResultType type = ResultType::MASK;
+
+  /// clear mask
+  void Clear();
+
+  /// Return a mutable pointer of the mask data buffer
+  void* Data() { return data.data(); }
+
+  /// Return a pointer of the mask data buffer for read only
+  const void* Data() const { return data.data(); }
+
+  /// Reserve size for mask data buffer
+  void Reserve(int size);
+
+  /// Resize the mask data buffer
+  void Resize(int size);
+
+  /// Debug function, convert the result to string to print
+  std::string Str();
+};
+
+
+/*! @brief Detection result structure for all the object detection models and instance segmentation models
+ */
+struct FASTDEPLOY_DECL DetectionResult : public BaseResult {
+  /** \brief All the detected object boxes for an input image, the size of `boxes` is the number of detected objects, and the element of `boxes` is a array of 4 float values, means [xmin, ymin, xmax, ymax]
+   */
+  std::vector<std::array<float, 4>> boxes;
+  /** \brief The confidence for all the detected objects
+   */
+  std::vector<float> scores;
+  /// The classify label for all the detected objects
+  std::vector<int32_t> label_ids;
+  /** \brief For instance segmentation model, `masks` is the predict mask for all the deteced objects
+   */
+  std::vector<Mask> masks;
+  //// Shows if the DetectionResult has mask
+  bool contain_masks = false;
+
+  ResultType type = ResultType::DETECTION;
+
+  DetectionResult() {}
+  DetectionResult(const DetectionResult& res);
+
+  /// Clear detection result
+  void Clear();
+
+  void Reserve(int size);
+
+  void Resize(int size);
+
+  /// Debug function, convert the result to string to print
+  std::string Str();
+};
+
+/*! @brief KeyPoint Detection result structure for all the keypoint detection models
+ */
+struct FASTDEPLOY_DECL KeyPointDetectionResult : public BaseResult {
+  /** \brief All the coordinates of detected keypoints for an input image, the size of `keypoints` is num_detected_objects * num_joints, and the element of `keypoint` is a array of 2 float values, means [x, y]
+   */
+  std::vector<std::array<float, 2>> keypoints;
+  //// The confidence for all the detected points
+  std::vector<float> scores;
+  //// Number of joints for a detected object
+  int num_joints = -1;
+
+  ResultType type = ResultType::KEYPOINT_DETECTION;
+  /// Clear detection result
+  void Clear();
+
+  void Reserve(int size);
+
+  void Resize(int size);
+
+  /// Debug function, convert the result to string to print
+  std::string Str();
+};
+
+struct FASTDEPLOY_DECL OCRResult : public BaseResult {
+  std::vector<std::array<int, 8>> boxes;
+
+  std::vector<std::string> text;
+  std::vector<float> rec_scores;
+
+  std::vector<float> cls_scores;
+  std::vector<int32_t> cls_labels;
+
+  ResultType type = ResultType::OCR;
+
+  void Clear();
+
+  std::string Str();
+};
+
+/*! @brief MOT(Multi-Object Tracking) result structure for all the MOT models
+ */
+struct FASTDEPLOY_DECL MOTResult : public BaseResult {
+  /** \brief All the tracking object boxes for an input image, the size of `boxes` is the number of tracking objects, and the element of `boxes` is a array of 4 float values, means [xmin, ymin, xmax, ymax]
+   */
+  std::vector<std::array<int, 4>> boxes;
+  /** \brief All the tracking object ids
+   */
+  std::vector<int> ids;
+  /** \brief The confidence for all the tracking objects
+   */
+  std::vector<float> scores;
+  /** \brief The classify label id for all the tracking object
+   */
+  std::vector<int> class_ids;
+
+  ResultType type = ResultType::MOT;
+  /// Clear MOT result
+  void Clear();
+  /// Debug function, convert the result to string to print
+  std::string Str();
+};
+
+
+
+/*! @brief Face detection result structure for all the face detection models
+ */
+struct FASTDEPLOY_DECL FaceDetectionResult : public BaseResult {
+  /** \brief All the detected object boxes for an input image, the size of `boxes` is the number of detected objects, and the element of `boxes` is a array of 4 float values, means [xmin, ymin, xmax, ymax]
+   */
+  std::vector<std::array<float, 4>> boxes;
+  /** \brief
+   * If the model detect face with landmarks, every detected object box correspoing to a landmark, which is a array of 2 float values, means location [x,y]
+  */
+  std::vector<std::array<float, 2>> landmarks;
+  /** \brief
+   * Indicates the confidence of all targets detected from a single image, and the number of elements is consistent with boxes.size()
+   */
+  std::vector<float> scores;
+  ResultType type = ResultType::FACE_DETECTION;
+  /** \brief
+   * `landmarks_per_face` indicates the number of face landmarks for each detected face
+   * if the model's output contains face landmarks (such as YOLOv5Face, SCRFD, ...)
+  */
+  int landmarks_per_face;
+
+  FaceDetectionResult() { landmarks_per_face = 0; }
+  FaceDetectionResult(const FaceDetectionResult& res);
+  /// Clear detection result
+  void Clear();
+
+  void Reserve(int size);
+
+  void Resize(int size);
+  /// Debug function, convert the result to string to print
+  std::string Str();
+};
+
+/*! @brief Face Alignment result structure for all the face alignment models
+ */
+struct FASTDEPLOY_DECL FaceAlignmentResult : public BaseResult {
+  /** \brief All the coordinates of detected landmarks for an input image, and the element of `landmarks` is a array of 2 float values, means [x, y]
+   */
+  std::vector<std::array<float, 2>> landmarks;
+
+  ResultType type = ResultType::FACE_ALIGNMENT;
+  /// Clear facealignment result
+  void Clear();
+
+  void Reserve(int size);
+
+  void Resize(int size);
+
+  /// Debug function, convert the result to string to print
+  std::string Str();
+};
+
+/*! @brief Segmentation result structure for all the segmentation models
+ */
+struct FASTDEPLOY_DECL SegmentationResult : public BaseResult {
+  /** \brief
+   * `label_map` stores the pixel-level category labels for input image. the number of pixels is equal to label_map.size()
+  */
+  std::vector<uint8_t> label_map;
+  /** \brief
+   * `score_map` stores the probability of the predicted label for each pixel of input image.
+  */
+  std::vector<float> score_map;
+  /// The output shape, means [H, W]
+  std::vector<int64_t> shape;
+  bool contain_score_map = false;
+
+  ResultType type = ResultType::SEGMENTATION;
+  /// Clear detection result
+  void Clear();
+
+  void Reserve(int size);
+
+  void Resize(int size);
+
+  /// Debug function, convert the result to string to print
+  std::string Str();
+};
+
+/*! @brief Face recognition result structure for all the Face recognition models
+ */
+struct FASTDEPLOY_DECL FaceRecognitionResult : public BaseResult {
+  /** \brief The feature embedding that represents the final extraction of the face recognition model can be used to calculate the feature similarity between faces.
+   */
+  std::vector<float> embedding;
+
+  ResultType type = ResultType::FACE_RECOGNITION;
+
+  FaceRecognitionResult() {}
+  FaceRecognitionResult(const FaceRecognitionResult& res);
+  /// Clear detection result
+  void Clear();
+
+  void Reserve(int size);
+
+  void Resize(int size);
+  /// Debug function, convert the result to string to print
+  std::string Str();
+};
+
+/*! @brief Matting result structure for all the Matting models
+ */
+struct FASTDEPLOY_DECL MattingResult : public BaseResult {
+  /** \brief
+  `alpha` is a one-dimensional vector, which is the predicted alpha transparency value. The range of values is [0., 1.], and the length is hxw. h, w are the height and width of the input image
+  */
+  std::vector<float> alpha;       // h x w
+  /** \brief
+  If the model can predict foreground, `foreground` save the predicted foreground image, the shape is [hight,width,channel] generally.
+  */
+  std::vector<float> foreground;  // h x w x c (c=3 default)
+  /** \brief
+   * The shape of output result, when contain_foreground == false, shape only contains (h, w), when contain_foreground == true, shape contains (h, w, c), and c is generally 3
+   */
+  std::vector<int64_t> shape;
+  /** \brief
+  If the model can predict alpha matte and foreground, contain_foreground = true, default false
+  */
+  bool contain_foreground = false;
+
+  ResultType type = ResultType::MATTING;
+
+  MattingResult() {}
+  MattingResult(const MattingResult& res);
+  /// Clear detection result
+  void Clear();
+
+  void Reserve(int size);
+
+  void Resize(int size);
+  /// Debug function, convert the result to string to print
+  std::string Str();
+};
+
+/*! @brief HeadPose result structure for all the headpose models
+ */
+struct FASTDEPLOY_DECL HeadPoseResult : public BaseResult {
+  /** \brief EulerAngles for an input image, and the element of `euler_angles` is a vector, contains {yaw, pitch, roll}
+   */
+  std::vector<float> euler_angles;
+
+  ResultType type = ResultType::HEADPOSE;
+  /// Clear headpose result
+  void Clear();
+
+  void Reserve(int size);
+
+  void Resize(int size);
+
+  /// Debug function, convert the result to string to print
+  std::string Str();
+};
+
+
+}  // namespace vision
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/vision/ocr/ppocr/classifier.h

@@ -0,0 +1,71 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+#include "fastdeploy/fastdeploy_model.h"
+#include "fastdeploy/vision/common/processors/transform.h"
+#include "fastdeploy/vision/common/result.h"
+#include "fastdeploy/vision/ocr/ppocr/utils/ocr_postprocess_op.h"
+
+namespace fastdeploy {
+namespace vision {
+/** \brief All OCR series model APIs are defined inside this namespace
+ *
+ */
+namespace ocr {
+/*! @brief Classifier object is used to load the classification model provided by PaddleOCR.
+ */
+class FASTDEPLOY_DECL Classifier : public FastDeployModel {
+ public:
+  Classifier();
+  /** \brief Set path of model file, and the configuration of runtime
+   *
+   * \param[in] model_file Path of model file, e.g ./ch_ppocr_mobile_v2.0_cls_infer/model.pdmodel.
+   * \param[in] params_file Path of parameter file, e.g ./ch_ppocr_mobile_v2.0_cls_infer/model.pdiparams, if the model format is ONNX, this parameter will be ignored.
+   * \param[in] custom_option RuntimeOption for inference, the default will use cpu, and choose the backend defined in `valid_cpu_backends`.
+   * \param[in] model_format Model format of the loaded model, default is Paddle format.
+   */
+  Classifier(const std::string& model_file, const std::string& params_file = "",
+             const RuntimeOption& custom_option = RuntimeOption(),
+             const ModelFormat& model_format = ModelFormat::PADDLE);
+  /// Get model's name
+  std::string ModelName() const { return "ppocr/ocr_cls"; }
+  /** \brief Predict the input image and get OCR classification model result.
+   *
+   * \param[in] im The input image data, comes from cv::imread(), is a 3-D array with layout HWC, BGR format.
+   * \param[in] result The output of OCR classification model result will be writen to this structure.
+   * \return true if the prediction is successed, otherwise false.
+   */
+  virtual bool Predict(cv::Mat* img, std::tuple<int, float>* result);
+
+  // Pre & Post parameters
+  float cls_thresh;
+  std::vector<int> cls_image_shape;
+  int cls_batch_num;
+
+  std::vector<float> mean;
+  std::vector<float> scale;
+  bool is_scale;
+
+ private:
+  bool Initialize();
+  /// Preprocess the input data, and set the preprocessed results to `outputs`
+  bool Preprocess(Mat* img, FDTensor* output);
+  /// Postprocess the inferenced results, and set the final result to `result`
+  bool Postprocess(FDTensor& infer_result, std::tuple<int, float>* result);
+};
+
+}  // namespace ocr
+}  // namespace vision
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/vision/ocr/ppocr/dbdetector.h

@@ -0,0 +1,85 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+#include "fastdeploy/fastdeploy_model.h"
+#include "fastdeploy/vision/common/processors/transform.h"
+#include "fastdeploy/vision/common/result.h"
+#include "fastdeploy/vision/ocr/ppocr/utils/ocr_postprocess_op.h"
+
+namespace fastdeploy {
+namespace vision {
+/** \brief All OCR series model APIs are defined inside this namespace
+ *
+ */
+namespace ocr {
+
+/*! @brief DBDetector object is used to load the detection model provided by PaddleOCR.
+ */
+class FASTDEPLOY_DECL DBDetector : public FastDeployModel {
+ public:
+  DBDetector();
+  /** \brief Set path of model file, and the configuration of runtime
+   *
+   * \param[in] model_file Path of model file, e.g ./ch_PP-OCRv3_det_infer/model.pdmodel.
+   * \param[in] params_file Path of parameter file, e.g ./ch_PP-OCRv3_det_infer/model.pdiparams, if the model format is ONNX, this parameter will be ignored.
+   * \param[in] custom_option RuntimeOption for inference, the default will use cpu, and choose the backend defined in `valid_cpu_backends`.
+   * \param[in] model_format Model format of the loaded model, default is Paddle format.
+   */
+  DBDetector(const std::string& model_file, const std::string& params_file = "",
+             const RuntimeOption& custom_option = RuntimeOption(),
+             const ModelFormat& model_format = ModelFormat::PADDLE);
+  /// Get model's name
+  std::string ModelName() const { return "ppocr/ocr_det"; }
+  /** \brief Predict the input image and get OCR detection model result.
+   *
+   * \param[in] im The input image data, comes from cv::imread(), is a 3-D array with layout HWC, BGR format.
+   * \param[in] boxes_result The output of OCR detection model result will be writen to this structure.
+   * \return true if the prediction is successed, otherwise false.
+   */
+  virtual bool Predict(cv::Mat* im,
+                       std::vector<std::array<int, 8>>* boxes_result);
+
+  // Pre & Post process parameters
+  int max_side_len;
+
+  float ratio_h{};
+  float ratio_w{};
+
+  double det_db_thresh;
+  double det_db_box_thresh;
+  double det_db_unclip_ratio;
+  std::string det_db_score_mode;
+  bool use_dilation;
+
+  std::vector<float> mean;
+  std::vector<float> scale;
+  bool is_scale;
+
+ private:
+  bool Initialize();
+  /// Preprocess the input data, and set the preprocessed results to `outputs`
+  bool Preprocess(Mat* mat, FDTensor* outputs,
+                  std::map<std::string, std::array<float, 2>>* im_info);
+  /*! @brief Postprocess the inferenced results, and set the final result to `boxes_result`
+   */
+  bool Postprocess(FDTensor& infer_result,
+                   std::vector<std::array<int, 8>>* boxes_result,
+                   const std::map<std::string, std::array<float, 2>>& im_info);
+  PostProcessor post_processor_;
+};
+
+}  // namespace ocr
+}  // namespace vision
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/vision/ocr/ppocr/ppocr_v2.h

@@ -0,0 +1,83 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <vector>
+
+#include "fastdeploy/fastdeploy_model.h"
+#include "fastdeploy/vision/common/processors/transform.h"
+#include "fastdeploy/vision/common/result.h"
+
+#include "fastdeploy/vision/ocr/ppocr/classifier.h"
+#include "fastdeploy/vision/ocr/ppocr/dbdetector.h"
+#include "fastdeploy/vision/ocr/ppocr/recognizer.h"
+#include "fastdeploy/vision/ocr/ppocr/utils/ocr_postprocess_op.h"
+
+namespace fastdeploy {
+/** \brief This pipeline can launch detection model, classification model and recognition model sequentially. All OCR pipeline APIs are defined inside this namespace.
+ *
+ */
+namespace pipeline {
+/*! @brief PPOCRv2 is used to load PP-OCRv2 series models provided by PaddleOCR.
+ */
+class FASTDEPLOY_DECL PPOCRv2 : public FastDeployModel {
+ public:
+  /** \brief Set up the detection model path, classification model path and recognition model path respectively.
+   *
+   * \param[in] det_model Path of detection model, e.g ./ch_PP-OCRv2_det_infer
+   * \param[in] cls_model Path of classification model, e.g ./ch_ppocr_mobile_v2.0_cls_infer
+   * \param[in] rec_model Path of recognition model, e.g ./ch_PP-OCRv2_rec_infer
+   */
+  PPOCRv2(fastdeploy::vision::ocr::DBDetector* det_model,
+                fastdeploy::vision::ocr::Classifier* cls_model,
+                fastdeploy::vision::ocr::Recognizer* rec_model);
+
+  /** \brief Classification model is optional, so this function is set up the detection model path and recognition model path respectively.
+   *
+   * \param[in] det_model Path of detection model, e.g ./ch_PP-OCRv2_det_infer
+   * \param[in] rec_model Path of recognition model, e.g ./ch_PP-OCRv2_rec_infer
+   */
+  PPOCRv2(fastdeploy::vision::ocr::DBDetector* det_model,
+                fastdeploy::vision::ocr::Recognizer* rec_model);
+
+  /** \brief Predict the input image and get OCR result.
+   *
+   * \param[in] im The input image data, comes from cv::imread(), is a 3-D array with layout HWC, BGR format.
+   * \param[in] result The output OCR result will be writen to this structure.
+   * \return true if the prediction successed, otherwise false.
+   */
+  virtual bool Predict(cv::Mat* img, fastdeploy::vision::OCRResult* result);
+  bool Initialized() const override;
+
+ protected:
+  fastdeploy::vision::ocr::DBDetector* detector_ = nullptr;
+  fastdeploy::vision::ocr::Classifier* classifier_ = nullptr;
+  fastdeploy::vision::ocr::Recognizer* recognizer_ = nullptr;
+  /// Launch the detection process in OCR.
+  virtual bool Detect(cv::Mat* img, fastdeploy::vision::OCRResult* result);
+  /// Launch the recognition process in OCR.
+  virtual bool Recognize(cv::Mat* img, fastdeploy::vision::OCRResult* result);
+  /// Launch the classification process in OCR.
+  virtual bool Classify(cv::Mat* img, fastdeploy::vision::OCRResult* result);
+};
+
+namespace application {
+namespace ocrsystem {
+  typedef pipeline::PPOCRv2 PPOCRSystemv2;
+}  // namespace ocrsystem
+}  // namespace application
+
+}  // namespace pipeline
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/vision/ocr/ppocr/ppocr_v3.h

@@ -0,0 +1,62 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "fastdeploy/vision/ocr/ppocr/ppocr_v2.h"
+
+namespace fastdeploy {
+/** \brief This pipeline can launch detection model, classification model and recognition model sequentially. All OCR pipeline APIs are defined inside this namespace.
+ *
+ */
+namespace pipeline {
+/*! @brief PPOCRv3 is used to load PP-OCRv3 series models provided by PaddleOCR.
+ */
+class FASTDEPLOY_DECL PPOCRv3 : public PPOCRv2 {
+ public:
+   /** \brief Set up the detection model path, classification model path and recognition model path respectively.
+   *
+   * \param[in] det_model Path of detection model, e.g ./ch_PP-OCRv3_det_infer
+   * \param[in] cls_model Path of classification model, e.g ./ch_ppocr_mobile_v2.0_cls_infer
+   * \param[in] rec_model Path of recognition model, e.g ./ch_PP-OCRv3_rec_infer
+   */
+  PPOCRv3(fastdeploy::vision::ocr::DBDetector* det_model,
+                fastdeploy::vision::ocr::Classifier* cls_model,
+                fastdeploy::vision::ocr::Recognizer* rec_model)
+                : PPOCRv2(det_model, cls_model, rec_model) {
+    // The only difference between v2 and v3
+    recognizer_->rec_image_shape[1] = 48;
+  }
+  /** \brief Classification model is optional, so this function is set up the detection model path and recognition model path respectively.
+   *
+   * \param[in] det_model Path of detection model, e.g ./ch_PP-OCRv3_det_infer
+   * \param[in] rec_model Path of recognition model, e.g ./ch_PP-OCRv3_rec_infer
+   */
+  PPOCRv3(fastdeploy::vision::ocr::DBDetector* det_model,
+                fastdeploy::vision::ocr::Recognizer* rec_model)
+                : PPOCRv2(det_model, rec_model) {
+    // The only difference between v2 and v3
+    recognizer_->rec_image_shape[1] = 48;
+  }
+};
+
+}  // namespace pipeline
+
+namespace application {
+namespace ocrsystem {
+  typedef pipeline::PPOCRv3 PPOCRSystemv3;
+}  // namespace ocrsystem
+}  // namespace application
+
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/vision/ocr/ppocr/recognizer.h

@@ -0,0 +1,79 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+#include "fastdeploy/fastdeploy_model.h"
+#include "fastdeploy/vision/common/processors/transform.h"
+#include "fastdeploy/vision/common/result.h"
+#include "fastdeploy/vision/ocr/ppocr/utils/ocr_postprocess_op.h"
+
+namespace fastdeploy {
+namespace vision {
+/** \brief All OCR series model APIs are defined inside this namespace
+ *
+ */
+namespace ocr {
+/*! @brief Recognizer object is used to load the recognition model provided by PaddleOCR.
+ */
+class FASTDEPLOY_DECL Recognizer : public FastDeployModel {
+ public:
+  Recognizer();
+  /** \brief Set path of model file, and the configuration of runtime
+   *
+   * \param[in] model_file Path of model file, e.g ./ch_PP-OCRv3_rec_infer/model.pdmodel.
+   * \param[in] params_file Path of parameter file, e.g ./ch_PP-OCRv3_rec_infer/model.pdiparams, if the model format is ONNX, this parameter will be ignored.
+   * \param[in] label_path Path of label file used by OCR recognition model. e.g ./ppocr_keys_v1.txt
+   * \param[in] custom_option RuntimeOption for inference, the default will use cpu, and choose the backend defined in `valid_cpu_backends`.
+   * \param[in] model_format Model format of the loaded model, default is Paddle format.
+   */
+  Recognizer(const std::string& model_file, const std::string& params_file = "",
+             const std::string& label_path = "",
+             const RuntimeOption& custom_option = RuntimeOption(),
+             const ModelFormat& model_format = ModelFormat::PADDLE);
+  /// Get model's name
+  std::string ModelName() const { return "ppocr/ocr_rec"; }
+  /** \brief Predict the input image and get OCR recognition model result.
+   *
+   * \param[in] im The input image data, comes from cv::imread(), is a 3-D array with layout HWC, BGR format.
+   * \param[in] rec_result The output of OCR recognition model result will be writen to this structure.
+   * \return true if the prediction is successed, otherwise false.
+   */
+  virtual bool Predict(cv::Mat* img,
+                       std::tuple<std::string, float>* rec_result);
+
+  // Pre & Post parameters
+  std::vector<std::string> label_list;
+  int rec_batch_num;
+  int rec_img_h;
+  int rec_img_w;
+  std::vector<int> rec_image_shape;
+
+  std::vector<float> mean;
+  std::vector<float> scale;
+  bool is_scale;
+
+ private:
+  bool Initialize();
+  /// Preprocess the input data, and set the preprocessed results to `outputs`
+  bool Preprocess(Mat* img, FDTensor* outputs,
+                  const std::vector<int>& rec_image_shape);
+  /*! @brief Postprocess the inferenced results, and set the final result to `rec_result`
+   */
+  bool Postprocess(FDTensor& infer_result,
+                   std::tuple<std::string, float>* rec_result);
+};
+
+}  // namespace ocr
+}  // namespace vision
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/vision/ocr/ppocr/utils/clipper.h

@@ -0,0 +1,425 @@
+/*******************************************************************************
+*                                                                              *
+* Author    :  Angus Johnson                                                   *
+* Version   :  6.4.2                                                           *
+* Date      :  27 February 2017                                                *
+* Website   :  http://www.angusj.com                                           *
+* Copyright :  Angus Johnson 2010-2017                                         *
+*                                                                              *
+* License:                                                                     *
+* Use, modification & distribution is subject to Boost Software License Ver 1. *
+* http://www.boost.org/LICENSE_1_0.txt                                         *
+*                                                                              *
+* Attributions:                                                                *
+* The code in this library is an extension of Bala Vatti's clipping algorithm: *
+* "A generic solution to polygon clipping"                                     *
+* Communications of the ACM, Vol 35, Issue 7 (July 1992) pp 56-63.             *
+* http://portal.acm.org/citation.cfm?id=129906                                 *
+*                                                                              *
+* Computer graphics and geometric modeling: implementation and algorithms      *
+* By Max K. Agoston                                                            *
+* Springer; 1 edition (January 4, 2005)                                        *
+* http://books.google.com/books?q=vatti+clipping+agoston                       *
+*                                                                              *
+* See also:                                                                    *
+* "Polygon Offsetting by Computing Winding Numbers"                            *
+* Paper no. DETC2005-85513 pp. 565-575                                         *
+* ASME 2005 International Design Engineering Technical Conferences             *
+* and Computers and Information in Engineering Conference (IDETC/CIE2005)      *
+* September 24-28, 2005 , Long Beach, California, USA                          *
+* http://www.me.berkeley.edu/~mcmains/pubs/DAC05OffsetPolygon.pdf              *
+*                                                                              *
+*******************************************************************************/
+
+#pragma once
+
+#ifndef clipper_hpp
+#define clipper_hpp
+
+#define CLIPPER_VERSION "6.4.2"
+
+// use_int32: When enabled 32bit ints are used instead of 64bit ints. This
+// improve performance but coordinate values are limited to the range +/- 46340
+//#define use_int32
+
+// use_xyz: adds a Z member to IntPoint. Adds a minor cost to perfomance.
+//#define use_xyz
+
+// use_lines: Enables line clipping. Adds a very minor cost to performance.
+#define use_lines
+
+// use_deprecated: Enables temporary support for the obsolete functions
+//#define use_deprecated
+
+#include <cstdlib>
+#include <cstring>
+#include <functional>
+#include <list>
+#include <ostream>
+#include <queue>
+#include <set>
+#include <stdexcept>
+#include <vector>
+
+namespace ClipperLib {
+
+enum ClipType { ctIntersection, ctUnion, ctDifference, ctXor };
+enum PolyType { ptSubject, ptClip };
+// By far the most widely used winding rules for polygon filling are
+// EvenOdd & NonZero (GDI, GDI+, XLib, OpenGL, Cairo, AGG, Quartz, SVG, Gr32)
+// Others rules include Positive, Negative and ABS_GTR_EQ_TWO (only in OpenGL)
+// see http://glprogramming.com/red/chapter11.html
+enum PolyFillType { pftEvenOdd, pftNonZero, pftPositive, pftNegative };
+
+#ifdef use_int32
+typedef int cInt;
+static cInt const loRange = 0x7FFF;
+static cInt const hiRange = 0x7FFF;
+#else
+typedef signed long long cInt;
+static cInt const loRange = 0x3FFFFFFF;
+static cInt const hiRange = 0x3FFFFFFFFFFFFFFFLL;
+typedef signed long long long64;  // used by Int128 class
+typedef unsigned long long ulong64;
+
+#endif
+
+struct IntPoint {
+  cInt X;
+  cInt Y;
+#ifdef use_xyz
+  cInt Z;
+  IntPoint(cInt x = 0, cInt y = 0, cInt z = 0) : X(x), Y(y), Z(z){};
+#else
+  IntPoint(cInt x = 0, cInt y = 0) : X(x), Y(y){};
+#endif
+
+  friend inline bool operator==(const IntPoint &a, const IntPoint &b) {
+    return a.X == b.X && a.Y == b.Y;
+  }
+  friend inline bool operator!=(const IntPoint &a, const IntPoint &b) {
+    return a.X != b.X || a.Y != b.Y;
+  }
+};
+//------------------------------------------------------------------------------
+
+typedef std::vector<IntPoint> Path;
+typedef std::vector<Path> Paths;
+
+inline Path &operator<<(Path &poly, const IntPoint &p) {
+  poly.push_back(p);
+  return poly;
+}
+inline Paths &operator<<(Paths &polys, const Path &p) {
+  polys.push_back(p);
+  return polys;
+}
+
+std::ostream &operator<<(std::ostream &s, const IntPoint &p);
+std::ostream &operator<<(std::ostream &s, const Path &p);
+std::ostream &operator<<(std::ostream &s, const Paths &p);
+
+struct DoublePoint {
+  double X;
+  double Y;
+  DoublePoint(double x = 0, double y = 0) : X(x), Y(y) {}
+  DoublePoint(IntPoint ip) : X((double)ip.X), Y((double)ip.Y) {}
+};
+//------------------------------------------------------------------------------
+
+#ifdef use_xyz
+typedef void (*ZFillCallback)(IntPoint &e1bot, IntPoint &e1top, IntPoint &e2bot,
+                              IntPoint &e2top, IntPoint &pt);
+#endif
+
+enum InitOptions {
+  ioReverseSolution = 1,
+  ioStrictlySimple = 2,
+  ioPreserveCollinear = 4
+};
+enum JoinType { jtSquare, jtRound, jtMiter };
+enum EndType {
+  etClosedPolygon,
+  etClosedLine,
+  etOpenButt,
+  etOpenSquare,
+  etOpenRound
+};
+
+class PolyNode;
+typedef std::vector<PolyNode *> PolyNodes;
+
+class PolyNode {
+ public:
+  PolyNode();
+  virtual ~PolyNode(){};
+  Path Contour;
+  PolyNodes Childs;
+  PolyNode *Parent;
+  PolyNode *GetNext() const;
+  bool IsHole() const;
+  bool IsOpen() const;
+  int ChildCount() const;
+
+ private:
+  // PolyNode& operator =(PolyNode& other);
+  unsigned Index;  // node index in Parent.Childs
+  bool m_IsOpen;
+  JoinType m_jointype;
+  EndType m_endtype;
+  PolyNode *GetNextSiblingUp() const;
+  void AddChild(PolyNode &child);
+  friend class Clipper;  // to access Index
+  friend class ClipperOffset;
+};
+
+class PolyTree : public PolyNode {
+ public:
+  ~PolyTree() { Clear(); };
+  PolyNode *GetFirst() const;
+  void Clear();
+  int Total() const;
+
+ private:
+  // PolyTree& operator =(PolyTree& other);
+  PolyNodes AllNodes;
+  friend class Clipper;  // to access AllNodes
+};
+
+bool Orientation(const Path &poly);
+double Area(const Path &poly);
+int PointInPolygon(const IntPoint &pt, const Path &path);
+
+void SimplifyPolygon(const Path &in_poly, Paths &out_polys,
+                     PolyFillType fillType = pftEvenOdd);
+void SimplifyPolygons(const Paths &in_polys, Paths &out_polys,
+                      PolyFillType fillType = pftEvenOdd);
+void SimplifyPolygons(Paths &polys, PolyFillType fillType = pftEvenOdd);
+
+void CleanPolygon(const Path &in_poly, Path &out_poly, double distance = 1.415);
+void CleanPolygon(Path &poly, double distance = 1.415);
+void CleanPolygons(const Paths &in_polys, Paths &out_polys,
+                   double distance = 1.415);
+void CleanPolygons(Paths &polys, double distance = 1.415);
+
+void MinkowskiSum(const Path &pattern, const Path &path, Paths &solution,
+                  bool pathIsClosed);
+void MinkowskiSum(const Path &pattern, const Paths &paths, Paths &solution,
+                  bool pathIsClosed);
+void MinkowskiDiff(const Path &poly1, const Path &poly2, Paths &solution);
+
+void PolyTreeToPaths(const PolyTree &polytree, Paths &paths);
+void ClosedPathsFromPolyTree(const PolyTree &polytree, Paths &paths);
+void OpenPathsFromPolyTree(PolyTree &polytree, Paths &paths);
+
+void ReversePath(Path &p);
+void ReversePaths(Paths &p);
+
+struct IntRect {
+  cInt left;
+  cInt top;
+  cInt right;
+  cInt bottom;
+};
+
+// enums that are used internally ...
+enum EdgeSide { esLeft = 1, esRight = 2 };
+
+// forward declarations (for stuff used internally) ...
+struct TEdge;
+struct IntersectNode;
+struct LocalMinimum;
+struct OutPt;
+struct OutRec;
+struct Join;
+
+typedef std::vector<OutRec *> PolyOutList;
+typedef std::vector<TEdge *> EdgeList;
+typedef std::vector<Join *> JoinList;
+typedef std::vector<IntersectNode *> IntersectList;
+
+//------------------------------------------------------------------------------
+
+// ClipperBase is the ancestor to the Clipper class. It should not be
+// instantiated directly. This class simply abstracts the conversion of sets of
+// polygon coordinates into edge objects that are stored in a LocalMinima list.
+class ClipperBase {
+ public:
+  ClipperBase();
+  virtual ~ClipperBase();
+  virtual bool AddPath(const Path &pg, PolyType PolyTyp, bool Closed);
+  bool AddPaths(const Paths &ppg, PolyType PolyTyp, bool Closed);
+  virtual void Clear();
+  IntRect GetBounds();
+  bool PreserveCollinear() { return m_PreserveCollinear; };
+  void PreserveCollinear(bool value) { m_PreserveCollinear = value; };
+
+ protected:
+  void DisposeLocalMinimaList();
+  TEdge *AddBoundsToLML(TEdge *e, bool IsClosed);
+  virtual void Reset();
+  TEdge *ProcessBound(TEdge *E, bool IsClockwise);
+  void InsertScanbeam(const cInt Y);
+  bool PopScanbeam(cInt &Y);
+  bool LocalMinimaPending();
+  bool PopLocalMinima(cInt Y, const LocalMinimum *&locMin);
+  OutRec *CreateOutRec();
+  void DisposeAllOutRecs();
+  void DisposeOutRec(PolyOutList::size_type index);
+  void SwapPositionsInAEL(TEdge *edge1, TEdge *edge2);
+  void DeleteFromAEL(TEdge *e);
+  void UpdateEdgeIntoAEL(TEdge *&e);
+
+  typedef std::vector<LocalMinimum> MinimaList;
+  MinimaList::iterator m_CurrentLM;
+  MinimaList m_MinimaList;
+
+  bool m_UseFullRange;
+  EdgeList m_edges;
+  bool m_PreserveCollinear;
+  bool m_HasOpenPaths;
+  PolyOutList m_PolyOuts;
+  TEdge *m_ActiveEdges;
+
+  typedef std::priority_queue<cInt> ScanbeamList;
+  ScanbeamList m_Scanbeam;
+};
+//------------------------------------------------------------------------------
+
+class Clipper : public virtual ClipperBase {
+ public:
+  Clipper(int initOptions = 0);
+  bool Execute(ClipType clipType, Paths &solution,
+               PolyFillType fillType = pftEvenOdd);
+  bool Execute(ClipType clipType, Paths &solution, PolyFillType subjFillType,
+               PolyFillType clipFillType);
+  bool Execute(ClipType clipType, PolyTree &polytree,
+               PolyFillType fillType = pftEvenOdd);
+  bool Execute(ClipType clipType, PolyTree &polytree, PolyFillType subjFillType,
+               PolyFillType clipFillType);
+  bool ReverseSolution() { return m_ReverseOutput; };
+  void ReverseSolution(bool value) { m_ReverseOutput = value; };
+  bool StrictlySimple() { return m_StrictSimple; };
+  void StrictlySimple(bool value) { m_StrictSimple = value; };
+// set the callback function for z value filling on intersections (otherwise Z
+// is 0)
+#ifdef use_xyz
+  void ZFillFunction(ZFillCallback zFillFunc);
+#endif
+ protected:
+  virtual bool ExecuteInternal();
+
+ private:
+  JoinList m_Joins;
+  JoinList m_GhostJoins;
+  IntersectList m_IntersectList;
+  ClipType m_ClipType;
+  typedef std::list<cInt> MaximaList;
+  MaximaList m_Maxima;
+  TEdge *m_SortedEdges;
+  bool m_ExecuteLocked;
+  PolyFillType m_ClipFillType;
+  PolyFillType m_SubjFillType;
+  bool m_ReverseOutput;
+  bool m_UsingPolyTree;
+  bool m_StrictSimple;
+#ifdef use_xyz
+  ZFillCallback m_ZFill;  // custom callback
+#endif
+  void SetWindingCount(TEdge &edge);
+  bool IsEvenOddFillType(const TEdge &edge) const;
+  bool IsEvenOddAltFillType(const TEdge &edge) const;
+  void InsertLocalMinimaIntoAEL(const cInt botY);
+  void InsertEdgeIntoAEL(TEdge *edge, TEdge *startEdge);
+  void AddEdgeToSEL(TEdge *edge);
+  bool PopEdgeFromSEL(TEdge *&edge);
+  void CopyAELToSEL();
+  void DeleteFromSEL(TEdge *e);
+  void SwapPositionsInSEL(TEdge *edge1, TEdge *edge2);
+  bool IsContributing(const TEdge &edge) const;
+  bool IsTopHorz(const cInt XPos);
+  void DoMaxima(TEdge *e);
+  void ProcessHorizontals();
+  void ProcessHorizontal(TEdge *horzEdge);
+  void AddLocalMaxPoly(TEdge *e1, TEdge *e2, const IntPoint &pt);
+  OutPt *AddLocalMinPoly(TEdge *e1, TEdge *e2, const IntPoint &pt);
+  OutRec *GetOutRec(int idx);
+  void AppendPolygon(TEdge *e1, TEdge *e2);
+  void IntersectEdges(TEdge *e1, TEdge *e2, IntPoint &pt);
+  OutPt *AddOutPt(TEdge *e, const IntPoint &pt);
+  OutPt *GetLastOutPt(TEdge *e);
+  bool ProcessIntersections(const cInt topY);
+  void BuildIntersectList(const cInt topY);
+  void ProcessIntersectList();
+  void ProcessEdgesAtTopOfScanbeam(const cInt topY);
+  void BuildResult(Paths &polys);
+  void BuildResult2(PolyTree &polytree);
+  void SetHoleState(TEdge *e, OutRec *outrec);
+  void DisposeIntersectNodes();
+  bool FixupIntersectionOrder();
+  void FixupOutPolygon(OutRec &outrec);
+  void FixupOutPolyline(OutRec &outrec);
+  bool IsHole(TEdge *e);
+  bool FindOwnerFromSplitRecs(OutRec &outRec, OutRec *&currOrfl);
+  void FixHoleLinkage(OutRec &outrec);
+  void AddJoin(OutPt *op1, OutPt *op2, const IntPoint offPt);
+  void ClearJoins();
+  void ClearGhostJoins();
+  void AddGhostJoin(OutPt *op, const IntPoint offPt);
+  bool JoinPoints(Join *j, OutRec *outRec1, OutRec *outRec2);
+  void JoinCommonEdges();
+  void DoSimplePolygons();
+  void FixupFirstLefts1(OutRec *OldOutRec, OutRec *NewOutRec);
+  void FixupFirstLefts2(OutRec *InnerOutRec, OutRec *OuterOutRec);
+  void FixupFirstLefts3(OutRec *OldOutRec, OutRec *NewOutRec);
+#ifdef use_xyz
+  void SetZ(IntPoint &pt, TEdge &e1, TEdge &e2);
+#endif
+};
+//------------------------------------------------------------------------------
+
+class ClipperOffset {
+ public:
+  ClipperOffset(double miterLimit = 2.0, double roundPrecision = 0.25);
+  ~ClipperOffset();
+  void AddPath(const Path &path, JoinType joinType, EndType endType);
+  void AddPaths(const Paths &paths, JoinType joinType, EndType endType);
+  void Execute(Paths &solution, double delta);
+  void Execute(PolyTree &solution, double delta);
+  void Clear();
+  double MiterLimit;
+  double ArcTolerance;
+
+ private:
+  Paths m_destPolys;
+  Path m_srcPoly;
+  Path m_destPoly;
+  std::vector<DoublePoint> m_normals;
+  double m_delta, m_sinA, m_sin, m_cos;
+  double m_miterLim, m_StepsPerRad;
+  IntPoint m_lowest;
+  PolyNode m_polyNodes;
+
+  void FixOrientations();
+  void DoOffset(double delta);
+  void OffsetPoint(int j, int &k, JoinType jointype);
+  void DoSquare(int j, int k);
+  void DoMiter(int j, int k, double r);
+  void DoRound(int j, int k);
+};
+//------------------------------------------------------------------------------
+
+class clipperException : public std::exception {
+ public:
+  clipperException(const char *description) : m_descr(description) {}
+  virtual ~clipperException() throw() {}
+  virtual const char *what() const throw() { return m_descr.c_str(); }
+
+ private:
+  std::string m_descr;
+};
+//------------------------------------------------------------------------------
+
+}  // ClipperLib namespace
+
+#endif  // clipper_hpp

3rdparty/include/fastdeploy/vision/ocr/ppocr/utils/ocr_postprocess_op.h

@@ -0,0 +1,91 @@
+// Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <iomanip>
+#include <iostream>
+#include <map>
+#include <ostream>
+#include <vector>
+#include "opencv2/core.hpp"
+#include "opencv2/imgcodecs.hpp"
+#include "opencv2/imgproc.hpp"
+
+#include <cstring>
+#include <fstream>
+#include <numeric>
+
+#include "fastdeploy/vision/ocr/ppocr/utils/clipper.h"
+
+namespace fastdeploy {
+namespace vision {
+namespace ocr {
+
+class PostProcessor {
+ public:
+  void GetContourArea(const std::vector<std::vector<float>> &box,
+                      float unclip_ratio, float &distance);
+
+  cv::RotatedRect UnClip(std::vector<std::vector<float>> box,
+                         const float &unclip_ratio);
+
+  float **Mat2Vec(cv::Mat mat);
+
+  std::vector<std::vector<int>> OrderPointsClockwise(
+      std::vector<std::vector<int>> pts);
+
+  std::vector<std::vector<float>> GetMiniBoxes(cv::RotatedRect box,
+                                               float &ssid);
+
+  float BoxScoreFast(std::vector<std::vector<float>> box_array, cv::Mat pred);
+  float PolygonScoreAcc(std::vector<cv::Point> contour, cv::Mat pred);
+
+  std::vector<std::vector<std::vector<int>>> BoxesFromBitmap(
+      const cv::Mat pred, const cv::Mat bitmap, const float &box_thresh,
+      const float &det_db_unclip_ratio, const std::string &det_db_score_mode);
+
+  std::vector<std::vector<std::vector<int>>> FilterTagDetRes(
+      std::vector<std::vector<std::vector<int>>> boxes, float ratio_h,
+      float ratio_w,
+      const std::map<std::string, std::array<float, 2>> &im_info);
+
+ private:
+  static bool XsortInt(std::vector<int> a, std::vector<int> b);
+
+  static bool XsortFp32(std::vector<float> a, std::vector<float> b);
+
+  std::vector<std::vector<float>> Mat2Vector(cv::Mat mat);
+
+  inline int _max(int a, int b) { return a >= b ? a : b; }
+
+  inline int _min(int a, int b) { return a >= b ? b : a; }
+
+  template <class T>
+  inline T clamp(T x, T min, T max) {
+    if (x > max) return max;
+    if (x < min) return min;
+    return x;
+  }
+
+  inline float clampf(float x, float min, float max) {
+    if (x > max) return max;
+    if (x < min) return min;
+    return x;
+  }
+};
+
+}  // namespace ocr
+}  // namespace vision
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/vision/ocr/ppocr/utils/ocr_utils.h

@@ -0,0 +1,38 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <set>
+#include <vector>
+#include "fastdeploy/core/fd_tensor.h"
+#include "fastdeploy/utils/utils.h"
+#include "fastdeploy/vision/common/result.h"
+
+#include "opencv2/core.hpp"
+#include "opencv2/imgcodecs.hpp"
+#include "opencv2/imgproc.hpp"
+
+namespace fastdeploy {
+namespace vision {
+namespace ocr {
+
+cv::Mat GetRotateCropImage(const cv::Mat& srcimage,
+                           const std::array<int, 8>& box);
+
+void SortBoxes(OCRResult* result);
+
+}  // namespace ocr
+}  // namespace vision
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/vision/utils/cuda_utils.h

@@ -0,0 +1,42 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <cuda_runtime.h>
+#include <cstdint>
+#include <vector>
+
+#ifndef CUDA_CHECK
+#define CUDA_CHECK(callstr)\
+  {\
+    cudaError_t error_code = callstr;\
+    if (error_code != cudaSuccess) {\
+      std::cerr << "CUDA error " << error_code << " at " << __FILE__ << ":";\
+      std::cerr << __LINE__;\
+      assert(0);\
+    }\
+  }
+#endif  // CUDA_CHECK
+
+namespace fastdeploy {
+namespace vision {
+namespace utils {
+void CudaYoloPreprocess(uint8_t* src, int src_width, int src_height,
+                        float* dst, int dst_width, int dst_height,
+                        const std::vector<float> padding_value,
+                        cudaStream_t stream);
+}  // namespace utils
+}  // namespace vision
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/vision/utils/utils.h

@@ -0,0 +1,104 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <opencv2/opencv.hpp>
+#include <set>
+#include <vector>
+
+#include "fastdeploy/core/fd_tensor.h"
+#include "fastdeploy/utils/utils.h"
+#include "fastdeploy/vision/common/result.h"
+
+// #include "unsupported/Eigen/CXX11/Tensor"
+#include "fastdeploy/function/reduce.h"
+#include "fastdeploy/function/softmax.h"
+#include "fastdeploy/function/transpose.h"
+#include "fastdeploy/vision/common/processors/mat.h"
+
+namespace fastdeploy {
+namespace vision {
+namespace utils {
+// topk sometimes is a very small value
+// so this implementation is simple but I don't think it will
+// cost too much time
+// Also there may be cause problem since we suppose the minimum value is
+// -99999999
+// Do not use this function on array which topk contains value less than
+// -99999999
+template <typename T>
+std::vector<int32_t> TopKIndices(const T* array, int array_size, int topk) {
+  topk = std::min(array_size, topk);
+  std::vector<int32_t> res(topk);
+  std::set<int32_t> searched;
+  for (int32_t i = 0; i < topk; ++i) {
+    T min = static_cast<T>(-99999999);
+    for (int32_t j = 0; j < array_size; ++j) {
+      if (searched.find(j) != searched.end()) {
+        continue;
+      }
+      if (*(array + j) > min) {
+        res[i] = j;
+        min = *(array + j);
+      }
+    }
+    searched.insert(res[i]);
+  }
+  return res;
+}
+
+void NMS(DetectionResult* output, float iou_threshold = 0.5);
+
+void NMS(FaceDetectionResult* result, float iou_threshold = 0.5);
+
+// MergeSort
+void SortDetectionResult(DetectionResult* output);
+
+void SortDetectionResult(FaceDetectionResult* result);
+
+// L2 Norm / cosine similarity  (for face recognition, ...)
+FASTDEPLOY_DECL std::vector<float> L2Normalize(
+    const std::vector<float>& values);
+
+FASTDEPLOY_DECL float CosineSimilarity(const std::vector<float>& a,
+                                       const std::vector<float>& b,
+                                       bool normalized = true);
+
+bool CropImageByBox(Mat& src_im, Mat* dst_im,
+                    const std::vector<float>& box, std::vector<float>* center,
+                    std::vector<float>* scale, const float expandratio = 0.3);
+
+/**
+ * Function: for keypoint detection model, fine positioning of keypoints in
+ * postprocess
+ * Parameters:
+ * heatmap: model inference results for keypoint detection models
+ * dim: shape information of the inference result
+ * coords: coordinates after refined positioning
+ * px: px = int(coords[ch * 2] + 0.5) , refer to API detection::GetFinalPredictions
+ * py: px = int(coords[ch * 2 + 1] + 0.5), refer to API detection::GetFinalPredictions
+ * index: index information of heatmap pixels
+ * ch: channel
+ * Paper reference: DARK postpocessing, Zhang et al.
+ * Distribution-Aware Coordinate Representation for Human Pose Estimation (CVPR
+ * 2020).
+ */
+void DarkParse(const std::vector<float>& heatmap, const std::vector<int>& dim,
+               std::vector<float>* coords, const int px, const int py,
+               const int index, const int ch);
+
+}  // namespace utils
+}  // namespace vision
+}  // namespace fastdeploy

3rdparty/resource/PaddleCharOCR/det/version.txt

@@ -1 +1 @@
-en_PP-OCRv3_det_slim_infer
+en_PP-OCRv3_det_infer

3rdparty/resource/PaddleCharOCR/rec/version.txt

@@ -1 +1 @@
-en_PP-OCRv3_rec_slim_infer
+en_PP-OCRv3_rec_infer

resource/config.json

@@ -30,9 +30,7 @@
             "Doc": "仓库识别导出结果模板",
             "arkPlanner": "{\"@type\": \"@penguin-statistics/depot\",\"items\": []}",
             "arkPlanner_Doc": "https://penguin-stats.cn/planner"
-        },
-        "ocrWithRawData": true,
-        "ocrWithRawData_Doc": "是否使用图片原始数据进行OCR，可以提高性能，但在部分设备上可能有兼容性问题"
+        }
     },
     "intent": {
         "Official": "com.hypergryph.arknights/com.u8.sdk.U8UnityContext",

src/MeoAssistant/Common/AsstConf.h

@@ -19,7 +19,7 @@
 #ifdef _MSC_VER
 #define ASST_SUPPRESS_CV_WARNINGS_START \
     ASST_DO_PRAGMA(warning(push))       \
-    ASST_DO_PRAGMA(warning(disable : 5054))
+    ASST_DO_PRAGMA(warning(disable : 5054 4251 4305 4275 4100))
 #define ASST_SUPPRESS_CV_WARNINGS_END ASST_DO_PRAGMA(warning(pop))
 #elif defined(__clang__)
 #define ASST_SUPPRESS_CV_WARNINGS_START                                               \

src/MeoAssistant/MeoAssistant.vcxproj

@@ -294,7 +294,7 @@
       <EnableCOMDATFolding>true</EnableCOMDATFolding>
       <OptimizeReferences>true</OptimizeReferences>
       <GenerateDebugInformation>true</GenerateDebugInformation>
-      <AdditionalDependencies>ppocr.lib;opencv_world453.lib;zlibstatic.lib;ws2_32.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>fastdeploy.lib;opencv_world453.lib;zlibstatic.lib;ws2_32.lib;%(AdditionalDependencies)</AdditionalDependencies>
       <UACExecutionLevel>RequireAdministrator</UACExecutionLevel>
       <AdditionalLibraryDirectories>
       </AdditionalLibraryDirectories>
@@ -347,7 +347,7 @@
       <EnableCOMDATFolding>true</EnableCOMDATFolding>
       <OptimizeReferences>true</OptimizeReferences>
       <GenerateDebugInformation>true</GenerateDebugInformation>
-      <AdditionalDependencies>ppocr.lib;opencv_world453.lib;zlibstatic.lib;ws2_32.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>fastdeploy.lib;opencv_world453.lib;zlibstatic.lib;ws2_32.lib;%(AdditionalDependencies)</AdditionalDependencies>
       <UACExecutionLevel>RequireAdministrator</UACExecutionLevel>
       <AdditionalLibraryDirectories>
       </AdditionalLibraryDirectories>

src/MeoAssistant/Resource/GeneralConfig.cpp

@@ -22,7 +22,6 @@ bool asst::GeneralConfig::parse(const json::value& json)
         m_options.yituliu_report.cmd_format = options_json.get("yituliuReport", "cmdFormat", std::string());
         m_options.depot_export_template.ark_planner =
             options_json.get("depotExportTemplate", "arkPlanner", std::string());
-        m_options.ocr_with_rawdata = options_json.get("ocrWithRawData", true);
     }
 
     for (const auto& [client_type, intent_name] : json.at("intent").as_object()) {

src/MeoAssistant/Resource/GeneralConfig.h

@@ -46,7 +46,6 @@ namespace asst
         // 每次到结算界面，汇报掉落数据至企鹅物流 https://penguin-stats.cn/
         DepotExportTemplate depot_export_template; // 仓库识别结果导出模板
         yituliuReportCfg yituliu_report; // 一图流大数据汇报：目前只有公招功能，https://yituliu.site/maarecruitdata
-        bool ocr_with_rawdata = true; // 使用原始数据进行OCR识别，可以提高性能，但会带来少量兼容性问题
     };
 
     struct AdbCfg

src/MeoAssistant/Resource/Miscellaneous/OcrPack.cpp

@@ -3,7 +3,11 @@
 #include <filesystem>
 
 #include "Utils/NoWarningCV.h"
-#include <PaddleOCR/paddle_ocr.h>
+ASST_SUPPRESS_CV_WARNINGS_START
+#include "fastdeploy/vision/ocr/ppocr/dbdetector.h"
+#include "fastdeploy/vision/ocr/ppocr/ppocr_v3.h"
+#include "fastdeploy/vision/ocr/ppocr/recognizer.h"
+ASST_SUPPRESS_CV_WARNINGS_END
 
 #include "Resource/GeneralConfig.h"
 #include "Utils/Demangle.hpp"
@@ -24,22 +28,11 @@ static std::filesystem::path prepare_paddle_dir(const std::filesystem::path& dir
 
 asst::OcrPack::OcrPack()
 {
-    Log.info("hardware_concurrency:", std::thread::hardware_concurrency());
-    for (size_t i = 0; i != MaxBoxSize; ++i) {
-        static constexpr size_t MaxTextSize = 4096;
-        *(m_strs_buffer + i) = new char[MaxTextSize];
-        // memset(*(m_strs_buffer + i), 0, MaxTextSize);
-    }
+    m_ocr_option = std::make_unique<fastdeploy::RuntimeOption>();
+    m_ocr_option->UseOrtBackend();
 }
 
-asst::OcrPack::~OcrPack()
-{
-    for (size_t i = 0; i != MaxBoxSize; ++i) {
-        delete[] *(m_strs_buffer + i);
-    }
-
-    PaddleOcrDestroy(m_ocr);
-}
+asst::OcrPack::~OcrPack() {}
 
 bool asst::OcrPack::load(const std::filesystem::path& path)
 {
@@ -50,29 +43,26 @@ bool asst::OcrPack::load(const std::filesystem::path& path)
         return false;
     }
 
-    constexpr static auto DetName = "det";
-    // constexpr static const char* ClsName = "cls";
-    constexpr static auto RecName = "rec";
-    constexpr static auto KeysName = "ppocr_keys_v1.txt";
+    using namespace asst::utils::path_literals;
+    const auto dst_model_file = paddle_dir / "det"_p / "inference.pdmodel"_p;
+    const auto dst_params_file = paddle_dir / "det"_p / "inference.pdiparams"_p;
+    const auto rec_model_file = paddle_dir / "rec"_p / "inference.pdmodel"_p;
+    const auto rec_params_file = paddle_dir / "rec"_p / "inference.pdiparams"_p;
+    const auto rec_label_file = paddle_dir / "ppocr_keys_v1.txt"_p;
 
-    const auto dst_filename = paddle_dir / asst::utils::path(DetName);
-    // const std::string cls_filename = dir + ClsName;
-    const auto rec_filename = paddle_dir / asst::utils::path(RecName);
-    const auto keys_filename = paddle_dir / asst::utils::path(KeysName);
-
-    if (m_ocr != nullptr) {
-        PaddleOcrDestroy(m_ocr);
-    }
-
-    const auto det4paddle = asst::utils::path_to_ansi_string(dst_filename);
-    const auto rec4paddle = asst::utils::path_to_ansi_string(rec_filename);
-    const auto keys4paddle = asst::utils::path_to_ansi_string(keys_filename);
-
-    if (det4paddle.empty() || rec4paddle.empty() || keys4paddle.empty()) {
+    if (!std::filesystem::exists(dst_model_file) || !std::filesystem::exists(dst_params_file) ||
+        !std::filesystem::exists(rec_model_file) || !std::filesystem::exists(rec_params_file) ||
+        !std::filesystem::exists(rec_label_file)) {
         return false;
     }
 
-    m_ocr = PaddleOcrCreate(det4paddle.c_str(), rec4paddle.c_str(), keys4paddle.c_str(), nullptr);
+    m_det = std::make_unique<fastdeploy::vision::ocr::DBDetector>(asst::utils::path_to_ansi_string(dst_model_file),
+                                                                  asst::utils::path_to_ansi_string(dst_params_file),
+                                                                  *m_ocr_option);
+    m_rec = std::make_unique<fastdeploy::vision::ocr::Recognizer>(
+        asst::utils::path_to_ansi_string(rec_model_file), asst::utils::path_to_ansi_string(rec_params_file),
+        asst::utils::path_to_ansi_string(rec_label_file), *m_ocr_option);
+    m_ocr = std::make_unique<fastdeploy::pipeline::PPOCRv3>(m_det.get(), m_rec.get());
 
     if (use_temp_dir) {
         // files can be removed after load
@@ -84,87 +74,8 @@ bool asst::OcrPack::load(const std::filesystem::path& path)
         }).detach();
     }
 
-    return m_ocr != nullptr;
-}
-
-std::vector<asst::TextRect> asst::OcrPack::recognize(const cv::Mat& image, const asst::TextRectProc& pred,
-                                                     bool without_det, bool trim)
-{
-    size_t size = 0;
-    std::string class_type = utils::demangle(typeid(*this).name());
-
-    if (Config.get_options().ocr_with_rawdata) {
-        // 如果是带 ROI 的 cv::Mat, data 仍是指向完整的图片数据，仅通过内部的一些其他参数标识 ROI
-        // 直接取 data 拿到的不是正确的图，所以拷贝一份出来
-        cv::Mat copied = image.clone();
-        if (!without_det) {
-            Log.trace("Ocr System with RawData and", class_type);
-            PaddleOcrSystemWithData(m_ocr, copied.rows, copied.cols, copied.type(), copied.data, false, m_boxes_buffer,
-                                    m_strs_buffer, m_scores_buffer, &size, nullptr, nullptr);
-        }
-        else {
-            Log.trace("Ocr Rec with RawData and", class_type);
-            PaddleOcrRecWithData(m_ocr, copied.rows, copied.cols, copied.type(), copied.data, m_strs_buffer,
-                                 m_scores_buffer, &size, nullptr, nullptr);
-        }
-    }
-    else {
-        std::vector<uchar> buf;
-        cv::imencode(".png", image, buf);
-
-        if (!without_det) {
-            Log.trace("Ocr System with Encode and", class_type);
-            PaddleOcrSystem(m_ocr, buf.data(), buf.size(), false, m_boxes_buffer, m_strs_buffer, m_scores_buffer, &size,
-                            nullptr, nullptr);
-        }
-        else {
-            Log.trace("Ocr Rec with Encode and", class_type);
-            PaddleOcrRec(m_ocr, buf.data(), buf.size(), m_strs_buffer, m_scores_buffer, &size, nullptr, nullptr);
-        }
-    }
-
-    std::vector<TextRect> result;
-    std::vector<TextRect> raw_result;
-
-#ifdef ASST_DEBUG
-    cv::Mat draw = image.clone();
-#endif
-
-    for (size_t i = 0; i != size; ++i) {
-        // the box rect like ↓
-        // 0 - 1
-        // 3 - 2
-        Rect rect;
-        if (!without_det) {
-            int* box = m_boxes_buffer + i * 8;
-            int x_collect[4] = { *(box + 0), *(box + 2), *(box + 4), *(box + 6) };
-            int y_collect[4] = { *(box + 1), *(box + 3), *(box + 5), *(box + 7) };
-            auto [left, right] = ranges::minmax(x_collect);
-            auto [top, bottom] = ranges::minmax(y_collect);
-            rect = Rect(left, top, right - left, bottom - top);
-        }
-        std::string text(*(m_strs_buffer + i));
-        float score = *(m_scores_buffer + i);
-        if (score > 2.0) {
-            score = 0;
-        }
-
-        TextRect tr { score, rect, text };
-#ifdef ASST_DEBUG
-        cv::rectangle(draw, make_rect<cv::Rect>(rect), cv::Scalar(0, 0, 255), 2);
-#endif
-        raw_result.emplace_back(tr);
-        if (trim) {
-            utils::string_trim(tr.text);
-        }
-        if (!pred || pred(tr)) {
-            result.emplace_back(std::move(tr));
-        }
-    }
-
-    Log.trace("OcrPack::recognize | raw:", raw_result);
-    Log.trace("OcrPack::recognize | proc:", result);
-    return result;
+    return m_det != nullptr && m_rec != nullptr && m_ocr != nullptr && m_det->Initialized() && m_rec->Initialized() &&
+           m_ocr->Initialized();
 }
 
 std::vector<asst::TextRect> asst::OcrPack::recognize(const cv::Mat& image, const Rect& roi,
@@ -185,6 +96,77 @@ std::vector<asst::TextRect> asst::OcrPack::recognize(const cv::Mat& image, const
     return recognize(roi_img, rect_cor, without_det, trim);
 }
 
+std::vector<asst::TextRect> asst::OcrPack::recognize(const cv::Mat& image, const asst::TextRectProc& pred,
+                                                     bool without_det, bool trim)
+{
+    std::string class_type = utils::demangle(typeid(*this).name());
+    // 如果是带 ROI 的 cv::Mat, data 仍是指向完整的图片数据，仅通过内部的一些其他参数标识 ROI
+    // 直接取 data 拿到的不是正确的图，所以拷贝一份出来
+    cv::Mat copied = image.clone();
+    fastdeploy::vision::OCRResult ocr_result;
+    if (!without_det) {
+        LogTraceScope("Ocr System with " + class_type);
+
+        m_ocr->Predict(&copied, &ocr_result);
+    }
+    else {
+        LogTraceScope("Ocr Rec with " + class_type);
+
+        std::tuple<std::string, float> rec_result;
+        m_rec->Predict(&copied, &rec_result);
+
+        auto& [text, score] = rec_result;
+        ocr_result.text.emplace_back(std::move(text));
+        ocr_result.rec_scores.emplace_back(score);
+    }
+
+#ifdef ASST_DEBUG
+    cv::Mat draw = image.clone();
+#endif
+
+    std::vector<TextRect> raw_result;
+    std::vector<TextRect> proced_result;
+    for (size_t i = 0; i != ocr_result.text.size(); ++i) {
+        // the box rect like ↓
+        // 0 - 1
+        // 3 - 2
+        Rect det_rect;
+        if (!without_det && i < ocr_result.boxes.size()) {
+            const auto& box = ocr_result.boxes.at(i);
+            int x_collect[] = { box[0], box[2], box[4], box[6] };
+            int y_collect[] = { box[1], box[3], box[5], box[7] };
+            auto [left, right] = ranges::minmax(x_collect);
+            auto [top, bottom] = ranges::minmax(y_collect);
+            det_rect = Rect(left, top, right - left, bottom - top);
+        }
+        else {
+            det_rect = Rect(0, 0, image.cols, image.rows);
+        }
+        std::string text = ocr_result.text.at(i);
+        double score = ocr_result.rec_scores.at(i);
+        if (score > 2.0) {
+            score = 0;
+        }
+
+#ifdef ASST_DEBUG
+        cv::rectangle(draw, make_rect<cv::Rect>(det_rect), cv::Scalar(0, 0, 255), 2);
+#endif
+
+        TextRect tr(score, det_rect, std::move(text));
+        raw_result.emplace_back(tr);
+        if (trim) {
+            utils::string_trim(tr.text);
+        }
+        if (!pred || pred(tr)) {
+            proced_result.emplace_back(std::move(tr));
+        }
+    }
+
+    Log.trace("OcrPack::recognize | raw:", raw_result);
+    Log.trace("OcrPack::recognize | proc:", proced_result);
+    return proced_result;
+}
+
 #ifdef _WIN32
 
 static std::filesystem::path prepare_paddle_dir(const std::filesystem::path& dir, bool* is_temp)
@@ -235,4 +217,4 @@ static std::filesystem::path prepare_paddle_dir(const std::filesystem::path& dir
     }
 }
 
-#endif
+#endif

src/MeoAssistant/Resource/Miscellaneous/OcrPack.h

@@ -9,7 +9,22 @@ namespace cv
 {
     class Mat;
 }
-struct paddle_ocr_t;
+
+namespace fastdeploy
+{
+    namespace vision::ocr
+    {
+        class DBDetector;
+        class Recognizer;
+    }
+
+    namespace pipeline
+    {
+        class PPOCRv3;
+    }
+
+    struct RuntimeOption;
+}
 
 namespace asst
 {
@@ -31,12 +46,10 @@ namespace asst
     protected:
         OcrPack();
 
-        paddle_ocr_t* m_ocr = nullptr;
-
-        // each box has 8 value ( 4 points, x and y )
-        int m_boxes_buffer[MaxBoxSize * 8] = { 0 };
-        char* m_strs_buffer[MaxBoxSize] = { nullptr };
-        float m_scores_buffer[MaxBoxSize] = { 0 };
+        std::unique_ptr<fastdeploy::RuntimeOption> m_ocr_option = nullptr;
+        std::unique_ptr<fastdeploy::vision::ocr::DBDetector> m_det = nullptr;
+        std::unique_ptr<fastdeploy::vision::ocr::Recognizer> m_rec = nullptr;
+        std::unique_ptr<fastdeploy::pipeline::PPOCRv3> m_ocr = nullptr;
     };
 
     class WordOcr final : public SingletonHolder<WordOcr>, public OcrPack