diff --git a/.gitmodules b/.gitmodules index 70db001202..198c4a26ee 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,6 @@ [submodule "test"] path = test url = https://github.com/MaaAssistantArknights/MaaTestSet.git -[submodule "3rdparty/VisionOCR"] - path = 3rdparty/VisionOCR - url = https://github.com/MaaAssistantArknights/VisionOCR.git [submodule "src/MeoAsstMac"] path = src/MeoAsstMac url = https://github.com/MaaAssistantArknights/MeoAsstMac.git diff --git a/3rdparty/VisionOCR b/3rdparty/VisionOCR deleted file mode 160000 index 4f63d97232..0000000000 --- a/3rdparty/VisionOCR +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 4f63d97232ae10764f456bd7a3ea0d47b6f28576 diff --git a/3rdparty/include/PaddleOCR/exports.h b/3rdparty/include/PaddleOCR/exports.h deleted file mode 100644 index 3b0915b078..0000000000 --- a/3rdparty/include/PaddleOCR/exports.h +++ /dev/null @@ -1,37 +0,0 @@ -#pragma once - -// The way how the function is called -#if !defined(OCR_CALL) -#if defined(_WIN32) -#define OCR_CALL __stdcall -#else -#define OCR_CALL -#endif /* _WIN32 */ -#endif /* OCR_CALL */ - -// The function exported symbols -#if defined _WIN32 || defined __CYGWIN__ -#define OCR_IMPORT __declspec(dllimport) -#define OCR_EXPORT __declspec(dllexport) -#define OCR_LOCAL -#else -#if __GNUC__ >= 4 -#define OCR_IMPORT __attribute__ ((visibility ("default"))) -#define OCR_EXPORT __attribute__ ((visibility ("default"))) -#define OCR_LOCAL __attribute__ ((visibility ("hidden"))) -#else -#define OCR_IMPORT -#define OCR_EXPORT -#define OCR_LOCAL -#endif -#endif - -#ifdef OCR_EXPORTS // defined if we are building the DLL (instead of using it) -#define OCRAPI_PORT OCR_EXPORT -#else -#define OCRAPI_PORT OCR_IMPORT -#endif // OCR_EXPORTS - -#define OCRAPI OCRAPI_PORT OCR_CALL - -#define OCRLOCAL OCR_LOCAL OCR_CALL \ No newline at end of file diff --git a/3rdparty/include/PaddleOCR/paddle_ocr.h b/3rdparty/include/PaddleOCR/paddle_ocr.h deleted file mode 100644 index dbbacaabb0..0000000000 --- a/3rdparty/include/PaddleOCR/paddle_ocr.h +++ /dev/null @@ -1,56 +0,0 @@ -#pragma once - -#include "exports.h" - -struct paddle_ocr_t; -typedef int OCR_ERROR; -typedef unsigned char uint8_t; - -#define OCR_SUCCESS 0 -#define OCR_FAILURE 1 - -#ifdef __cplusplus -extern "C" { -#endif - - OCRAPI_PORT paddle_ocr_t* OCR_CALL PaddleOcrCreate( - const char* det_model_dir, const char* rec_model_dir, - const char* char_list_file, const char* cls_model_dir); - - void OCRAPI PaddleOcrDestroy(paddle_ocr_t* ocr_ptr); - - OCR_ERROR OCRAPI PaddleOcrDet( - paddle_ocr_t* ocr_ptr, const uint8_t* encode_buf, size_t encode_buf_size, - int* out_boxes, size_t* out_boxes_size, - double* out_times, size_t* out_times_size); - - OCR_ERROR OCRAPI PaddleOcrDetWithData( - paddle_ocr_t* ocr_ptr, int rows, int cols, int type, void* data, - int* out_boxes, size_t* out_boxes_size, - double* out_times, size_t* out_times_size); - - OCR_ERROR OCRAPI PaddleOcrRec( - paddle_ocr_t* ocr_ptr, const uint8_t* encode_buf, size_t encode_buf_size, - char** out_strs, float* out_scores, size_t* out_size, - double* out_times, size_t* out_times_size); - - OCR_ERROR OCRAPI PaddleOcrRecWithData( - paddle_ocr_t* ocr_ptr, int rows, int cols, int type, void* data, - char** out_strs, float* out_scores, size_t* out_size, - double* out_times, size_t* out_times_size); - - OCR_ERROR OCRAPI PaddleOcrSystem( - paddle_ocr_t* ocr_ptr, const uint8_t* encode_buf, size_t encode_buf_size, - bool with_cls, - int* out_boxes, char** out_strs, float* out_scores, size_t* out_size, - double* out_times, size_t* out_times_size); - - OCR_ERROR OCRAPI PaddleOcrSystemWithData( - paddle_ocr_t* ocr_ptr, int rows, int cols, int type, void* data, - bool with_cls, - int* out_boxes, char** out_strs, float* out_scores, size_t* out_size, - double* out_times, size_t* out_times_size); - -#ifdef __cplusplus -} -#endif \ No newline at end of file diff --git a/3rdparty/include/fastdeploy/backends/backend.h b/3rdparty/include/fastdeploy/backends/backend.h new file mode 100644 index 0000000000..652d94cb88 --- /dev/null +++ b/3rdparty/include/fastdeploy/backends/backend.h @@ -0,0 +1,74 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include + +#include "fastdeploy/backends/common/multiclass_nms.h" +#include "fastdeploy/core/fd_tensor.h" +#include "fastdeploy/core/fd_type.h" + +namespace fastdeploy { + +/*! @brief Information of Tensor + */ +struct TensorInfo { + std::string name; ///< Name of tensor + std::vector shape; ///< Shape of tensor + FDDataType dtype; ///< Data type of tensor + + friend std::ostream& operator<<(std::ostream& output, + const TensorInfo& info) { + output << "TensorInfo(name: " << info.name << ", shape: ["; + for (size_t i = 0; i < info.shape.size(); ++i) { + if (i == info.shape.size() - 1) { + output << info.shape[i]; + } else { + output << info.shape[i] << ", "; + } + } + output << "], dtype: " << Str(info.dtype) << ")"; + return output; + } +}; + +class BaseBackend { + public: + bool initialized_ = false; + + BaseBackend() {} + virtual ~BaseBackend() = default; + + virtual bool Initialized() const { return initialized_; } + + virtual int NumInputs() const = 0; + virtual int NumOutputs() const = 0; + virtual TensorInfo GetInputInfo(int index) = 0; + virtual TensorInfo GetOutputInfo(int index) = 0; + virtual std::vector GetInputInfos() = 0; + virtual std::vector GetOutputInfos() = 0; + virtual bool Infer(std::vector& inputs, + std::vector* outputs) = 0; + virtual std::unique_ptr Clone(void *stream = nullptr, + int device_id = -1) { + FDERROR << "Clone no support" << std::endl; + return nullptr; + } +}; + +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/backends/common/multiclass_nms.h b/3rdparty/include/fastdeploy/backends/common/multiclass_nms.h new file mode 100644 index 0000000000..48a3d9336d --- /dev/null +++ b/3rdparty/include/fastdeploy/backends/common/multiclass_nms.h @@ -0,0 +1,45 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#include +#include +#include + +namespace fastdeploy { +namespace backend { +struct MultiClassNMS { + int64_t background_label = -1; + int64_t keep_top_k = -1; + float nms_eta; + float nms_threshold = 0.7; + int64_t nms_top_k; + bool normalized; + float score_threshold; + + std::vector out_num_rois_data; + std::vector out_index_data; + std::vector out_box_data; + void FastNMS(const float* boxes, const float* scores, const int& num_boxes, + std::vector* keep_indices); + int NMSForEachSample(const float* boxes, const float* scores, int num_boxes, + int num_classes, + std::map>* keep_indices); + void Compute(const float* boxes, const float* scores, + const std::vector& boxes_dim, + const std::vector& scores_dim); +}; +} // namespace backend + +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/backends/ort/ops/multiclass_nms.h b/3rdparty/include/fastdeploy/backends/ort/ops/multiclass_nms.h new file mode 100644 index 0000000000..4e167d669b --- /dev/null +++ b/3rdparty/include/fastdeploy/backends/ort/ops/multiclass_nms.h @@ -0,0 +1,81 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +#ifndef NON_64_PLATFORM +#include "onnxruntime_cxx_api.h" // NOLINT + +namespace fastdeploy { + +struct MultiClassNmsKernel { + protected: + int64_t background_label = -1; + int64_t keep_top_k = -1; + float nms_eta; + float nms_threshold = 0.7; + int64_t nms_top_k; + bool normalized; + float score_threshold; + Ort::CustomOpApi ort_; + + public: + MultiClassNmsKernel(Ort::CustomOpApi ort, const OrtKernelInfo* info) + : ort_(ort) { + GetAttribute(info); + } + + void GetAttribute(const OrtKernelInfo* info); + + void Compute(OrtKernelContext* context); + void FastNMS(const float* boxes, const float* scores, const int& num_boxes, + std::vector* keep_indices); + int NMSForEachSample(const float* boxes, const float* scores, int num_boxes, + int num_classes, + std::map>* keep_indices); +}; + +struct MultiClassNmsOp + : Ort::CustomOpBase { + void* CreateKernel(Ort::CustomOpApi api, const OrtKernelInfo* info) const { + return new MultiClassNmsKernel(api, info); + } + + const char* GetName() const { return "MultiClassNMS"; } + + size_t GetInputTypeCount() const { return 2; } + + ONNXTensorElementDataType GetInputType(size_t index) const { + return ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT; + } + + size_t GetOutputTypeCount() const { return 3; } + + ONNXTensorElementDataType GetOutputType(size_t index) const { + if (index == 0) { + return ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT; + } + return ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32; + } + + const char* GetExecutionProviderType() const { + return "CPUExecutionProvider"; + } +}; + +} // namespace fastdeploy + +#endif \ No newline at end of file diff --git a/3rdparty/include/fastdeploy/backends/ort/ort_backend.h b/3rdparty/include/fastdeploy/backends/ort/ort_backend.h new file mode 100644 index 0000000000..31c7698240 --- /dev/null +++ b/3rdparty/include/fastdeploy/backends/ort/ort_backend.h @@ -0,0 +1,98 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include + +#include "fastdeploy/backends/backend.h" +#include "onnxruntime_cxx_api.h" // NOLINT + +namespace fastdeploy { + +struct OrtValueInfo { + std::string name; + std::vector shape; + ONNXTensorElementDataType dtype; +}; + +struct OrtBackendOption { + // -1 means default + // 0: ORT_DISABLE_ALL + // 1: ORT_ENABLE_BASIC + // 2: ORT_ENABLE_EXTENDED + // 99: ORT_ENABLE_ALL (enable some custom optimizations e.g bert) + int graph_optimization_level = -1; + int intra_op_num_threads = -1; + int inter_op_num_threads = -1; + // 0: ORT_SEQUENTIAL + // 1: ORT_PARALLEL + int execution_mode = -1; + bool use_gpu = false; + int gpu_id = 0; + void* external_stream_ = nullptr; + + // inside parameter, maybe remove next version + bool remove_multiclass_nms_ = false; + std::map custom_op_info_; +}; + +class OrtBackend : public BaseBackend { + public: + OrtBackend() {} + virtual ~OrtBackend() = default; + + void BuildOption(const OrtBackendOption& option); + + bool InitFromPaddle(const std::string& model_file, + const std::string& params_file, + const OrtBackendOption& option = OrtBackendOption(), + bool verbose = false); + + bool InitFromOnnx(const std::string& model_file, + const OrtBackendOption& option = OrtBackendOption(), + bool from_memory_buffer = false); + + bool Infer(std::vector& inputs, + std::vector* outputs) override; + + int NumInputs() const override { return inputs_desc_.size(); } + + int NumOutputs() const override { return outputs_desc_.size(); } + + TensorInfo GetInputInfo(int index) override; + TensorInfo GetOutputInfo(int index) override; + std::vector GetInputInfos() override; + std::vector GetOutputInfos() override; + static std::vector custom_operators_; + void InitCustomOperators(); + + private: + Ort::Env env_; + Ort::Session session_{nullptr}; + Ort::SessionOptions session_options_; + std::shared_ptr binding_; + std::vector inputs_desc_; + std::vector outputs_desc_; +#ifndef NON_64_PLATFORM + Ort::CustomOpDomain custom_op_domain_ = Ort::CustomOpDomain("Paddle"); +#endif + OrtBackendOption option_; + void CopyToCpu(const Ort::Value& value, FDTensor* tensor, + const std::string& name); +}; +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/backends/ort/utils.h b/3rdparty/include/fastdeploy/backends/ort/utils.h new file mode 100644 index 0000000000..e2912ad38f --- /dev/null +++ b/3rdparty/include/fastdeploy/backends/ort/utils.h @@ -0,0 +1,39 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include + +#include "fastdeploy/backends/backend.h" +#include "onnxruntime_cxx_api.h" // NOLINT + +namespace fastdeploy { + +// Convert FDDataType to OrtDataType +ONNXTensorElementDataType GetOrtDtype(const FDDataType& fd_dtype); + +// Convert OrtDataType to FDDataType +FDDataType GetFdDtype(const ONNXTensorElementDataType& ort_dtype); + +// Create Ort::Value +// is_backend_cuda specify if the onnxruntime use CUDAExectionProvider +// While is_backend_cuda = true, and tensor.device = Device::GPU +// Will directly share the cuda data in tensor to OrtValue +Ort::Value CreateOrtValue(FDTensor& tensor, bool is_backend_cuda = false); + +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/backends/rknpu/rknpu2/rknpu2_backend.h b/3rdparty/include/fastdeploy/backends/rknpu/rknpu2/rknpu2_backend.h new file mode 100644 index 0000000000..1aba24ec3b --- /dev/null +++ b/3rdparty/include/fastdeploy/backends/rknpu/rknpu2/rknpu2_backend.h @@ -0,0 +1,103 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once + +#include "fastdeploy/backends/backend.h" +#include "fastdeploy/core/fd_tensor.h" +#include "rknn_api.h" // NOLINT +#include "fastdeploy/backends/rknpu/rknpu2/rknpu2_config.h" +#include +#include +#include +#include +#include + +namespace fastdeploy { +struct RKNPU2BackendOption { + rknpu2::CpuName cpu_name = rknpu2::CpuName::RK3588; + + // The specification of NPU core setting.It has the following choices : + // RKNN_NPU_CORE_AUTO : Referring to automatic mode, meaning that it will + // select the idle core inside the NPU. + // RKNN_NPU_CORE_0 : Running on the NPU0 core + // RKNN_NPU_CORE_1: Runing on the NPU1 core + // RKNN_NPU_CORE_2: Runing on the NPU2 core + // RKNN_NPU_CORE_0_1: Running on both NPU0 and NPU1 core simultaneously. + // RKNN_NPU_CORE_0_1_2: Running on both NPU0, NPU1 and NPU2 simultaneously. + rknpu2::CoreMask core_mask = rknpu2::CoreMask::RKNN_NPU_CORE_AUTO; +}; + +class RKNPU2Backend : public BaseBackend { + public: + RKNPU2Backend() = default; + + virtual ~RKNPU2Backend(); + + // RKNN API + bool LoadModel(void* model); + + bool GetSDKAndDeviceVersion(); + + bool SetCoreMask(rknpu2::CoreMask& core_mask) const; + + bool GetModelInputOutputInfos(); + + // BaseBackend API + void BuildOption(const RKNPU2BackendOption& option); + + bool InitFromRKNN(const std::string& model_file, + const RKNPU2BackendOption& option = RKNPU2BackendOption()); + + int NumInputs() const override { + return static_cast(inputs_desc_.size()); + } + + int NumOutputs() const override { + return static_cast(outputs_desc_.size()); + } + + TensorInfo GetInputInfo(int index) override; + TensorInfo GetOutputInfo(int index) override; + std::vector GetInputInfos() override; + std::vector GetOutputInfos() override; + bool Infer(std::vector& inputs, + std::vector* outputs) override; + + private: + // The object of rknn context. + rknn_context ctx{}; + // The structure rknn_sdk_version is used to indicate the version + // information of the RKNN SDK. + rknn_sdk_version sdk_ver{}; + // The structure rknn_input_output_num represents the number of + // input and output Tensor + rknn_input_output_num io_num{}; + std::vector inputs_desc_; + std::vector outputs_desc_; + + rknn_tensor_attr* input_attrs_ = nullptr; + rknn_tensor_attr* output_attrs_ = nullptr; + + rknn_tensor_mem** input_mems_; + rknn_tensor_mem** output_mems_; + + bool infer_init = false; + + RKNPU2BackendOption option_; + + static void DumpTensorAttr(rknn_tensor_attr& attr); + static FDDataType RknnTensorTypeToFDDataType(rknn_tensor_type type); + static rknn_tensor_type FDDataTypeToRknnTensorType(FDDataType type); +}; +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/backends/rknpu/rknpu2/rknpu2_config.h b/3rdparty/include/fastdeploy/backends/rknpu/rknpu2/rknpu2_config.h new file mode 100644 index 0000000000..60e1a76aab --- /dev/null +++ b/3rdparty/include/fastdeploy/backends/rknpu/rknpu2/rknpu2_config.h @@ -0,0 +1,37 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once + +namespace fastdeploy { +namespace rknpu2 { +typedef enum _rknpu2_cpu_name { + RK356X = 0, /* run on RK356X. */ + RK3588 = 1, /* default,run on RK3588. */ + UNDEFINED, +} CpuName; + +/*! RKNPU2 core mask for mobile device. */ +typedef enum _rknpu2_core_mask { + RKNN_NPU_CORE_AUTO = 0, //< default, run on NPU core randomly. + RKNN_NPU_CORE_0 = 1, //< run on NPU core 0. + RKNN_NPU_CORE_1 = 2, //< run on NPU core 1. + RKNN_NPU_CORE_2 = 4, //< run on NPU core 2. + RKNN_NPU_CORE_0_1 = + RKNN_NPU_CORE_0 | RKNN_NPU_CORE_1, //< run on NPU core 1 and core 2. + RKNN_NPU_CORE_0_1_2 = + RKNN_NPU_CORE_0_1 | RKNN_NPU_CORE_2, //< run on NPU core 1 and core 2. + RKNN_NPU_CORE_UNDEFINED, +} CoreMask; +} // namespace rknpu2 +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/core/allocate.h b/3rdparty/include/fastdeploy/core/allocate.h new file mode 100644 index 0000000000..1e88787f4d --- /dev/null +++ b/3rdparty/include/fastdeploy/core/allocate.h @@ -0,0 +1,60 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once + +#include +#include +#include +#include +#include + +#include "fastdeploy/utils/utils.h" + +namespace fastdeploy { + +class FASTDEPLOY_DECL FDHostAllocator { + public: + bool operator()(void** ptr, size_t size) const; +}; + +class FASTDEPLOY_DECL FDHostFree { + public: + void operator()(void* ptr) const; +}; + +#ifdef WITH_GPU + +class FASTDEPLOY_DECL FDDeviceAllocator { + public: + bool operator()(void** ptr, size_t size) const; +}; + +class FASTDEPLOY_DECL FDDeviceFree { + public: + void operator()(void* ptr) const; +}; + +class FASTDEPLOY_DECL FDDeviceHostAllocator { + public: + bool operator()(void** ptr, size_t size) const; +}; + +class FASTDEPLOY_DECL FDDeviceHostFree { + public: + void operator()(void* ptr) const; +}; + +#endif + +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/core/config.h b/3rdparty/include/fastdeploy/core/config.h new file mode 100644 index 0000000000..e587d97df8 --- /dev/null +++ b/3rdparty/include/fastdeploy/core/config.h @@ -0,0 +1,72 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once + +#ifndef FASTDEPLOY_LIB +/* #undef FASTDEPLOY_LIB */ +#endif + +#ifndef ENABLE_PADDLE_FRONTEND +/* #undef ENABLE_PADDLE_FRONTEND */ +#endif + +#ifndef ENABLE_ORT_BACKEND +#define ENABLE_ORT_BACKEND +#endif + +#ifndef ENABLE_PADDLE_BACKEND +#define ENABLE_PADDLE_BACKEND +#endif + +#ifndef ENABLE_POROS_BACKEND +/* #undef ENABLE_POROS_BACKEND */ +#endif + +#ifndef ENABLE_OPENVINO_BACKEND +#define ENABLE_OPENVINO_BACKEND +#endif + +#ifndef WITH_GPU +/* #undef WITH_GPU */ +#endif + +#ifndef ENABLE_TRT_BACKEND +/* #undef ENABLE_TRT_BACKEND */ +#endif + +#ifndef ENABLE_VISION +#define ENABLE_VISION +#endif + +#ifndef ENABLE_FLYCV +/* #undef ENABLE_FLYCV */ +#endif + +#ifndef ENABLE_TEXT +#define ENABLE_TEXT +#endif + +#ifndef ENABLE_OPENCV_CUDA +/* #undef ENABLE_OPENCV_CUDA */ +#endif + +#ifdef ENABLE_VISION +#ifndef ENABLE_VISION_VISUALIZE +#define ENABLE_VISION_VISUALIZE +#endif +#endif + +#ifndef ENABLE_FDTENSOR_FUNC +#define ENABLE_FDTENSOR_FUNC +#endif diff --git a/3rdparty/include/fastdeploy/core/fd_tensor.h b/3rdparty/include/fastdeploy/core/fd_tensor.h new file mode 100644 index 0000000000..7deb482298 --- /dev/null +++ b/3rdparty/include/fastdeploy/core/fd_tensor.h @@ -0,0 +1,147 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once + +#include +#include +#include +#include + +#include "fastdeploy/core/allocate.h" +#include "fastdeploy/core/fd_type.h" + +namespace fastdeploy { + +struct FASTDEPLOY_DECL FDTensor { + // std::vector data; + void* buffer_ = nullptr; + std::vector shape = {0}; + std::string name = ""; + FDDataType dtype = FDDataType::INT8; + + // This use to skip memory copy step + // the external_data_ptr will point to the user allocated memory + // user has to maintain the memory, allocate and release + void* external_data_ptr = nullptr; + // The internal data will be on CPU + // Some times, the external data is on the GPU, and we are going to use + // GPU to inference the model + // so we can skip data transfer, which may improve the efficience + Device device = Device::CPU; + // By default the device id of FDTensor is -1, which means this value is + // invalid, and FDTensor is using the same device id as Runtime. + int device_id = -1; + + // Whether the data buffer is in pinned memory, which is allocated + // with cudaMallocHost() + bool is_pinned_memory = false; + + // if the external data is not on CPU, we use this temporary buffer + // to transfer data to CPU at some cases we need to visit the + // other devices' data + std::vector temporary_cpu_buffer; + + // Get data buffer pointer + void* MutableData(); + + void* Data(); + + bool IsShared() { + return external_data_ptr != nullptr; + } + + void StopSharing(); + + const void* Data() const; + + // Use this data to get the tensor data to process + // Since the most senario is process data in CPU + // this function will return a pointer to cpu memory + // buffer. + // If the original data is on other device, the data + // will copy to cpu store in `temporary_cpu_buffer` + const void* CpuData() const; + + // Set user memory buffer for Tensor, the memory is managed by + // the user it self, but the Tensor will share the memory with user + // So take care with the user buffer + void SetExternalData(const std::vector& new_shape, + const FDDataType& data_type, void* data_buffer, + const Device& new_device = Device::CPU); + + // Expand the shape of a Tensor. Insert a new axis that will appear + // at the `axis` position in the expanded Tensor shape. + void ExpandDim(int64_t axis = 0); + + // Squeeze the shape of a Tensor. Erase the axis that will appear + // at the `axis` position in the squeezed Tensor shape. + void Squeeze(int64_t axis = 0); + + // Initialize Tensor + // Include setting attribute for tensor + // and allocate cpu memory buffer + void Allocate(const std::vector& new_shape, + const FDDataType& data_type, + const std::string& tensor_name = "", + const Device& new_device = Device::CPU); + + // Total size of tensor memory buffer in bytes + int Nbytes() const; + + // Total number of elements in this tensor + int Numel() const; + + // Get shape of FDTensor + std::vector Shape() const { return shape; } + + // Get dtype of FDTensor + FDDataType Dtype() const { return dtype; } + + void Resize(size_t nbytes); + + void Resize(const std::vector& new_shape); + + void Resize(const std::vector& new_shape, + const FDDataType& data_type, const std::string& tensor_name = "", + const Device& new_device = Device::CPU); + + // Debug function + // Use this function to print shape, dtype, mean, max, min + // prefix will also be printed as tag + void PrintInfo(const std::string& prefix = "TensorInfo: ") const; + + bool ReallocFn(size_t nbytes); + + void FreeFn(); + + FDTensor() {} + explicit FDTensor(const std::string& tensor_name); + // Deep copy + FDTensor(const FDTensor& other); + // Move constructor + FDTensor(FDTensor&& other); + + // Deep copy assignment + FDTensor& operator=(const FDTensor& other); + // Move assignment + FDTensor& operator=(FDTensor&& other); + + ~FDTensor() { FreeFn(); } + + static void CopyBuffer(void* dst, const void* src, size_t nbytes, + const Device& device = Device::CPU, + bool is_pinned_memory = false); +}; + +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/core/fd_type.h b/3rdparty/include/fastdeploy/core/fd_type.h new file mode 100644 index 0000000000..ba18d8e36a --- /dev/null +++ b/3rdparty/include/fastdeploy/core/fd_type.h @@ -0,0 +1,80 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once + +#include +#include +#include + +#include "fastdeploy/core/config.h" +#include "fastdeploy/utils/utils.h" + +namespace fastdeploy { + +enum FASTDEPLOY_DECL Device { CPU, GPU, RKNPU, IPU, TIMVX}; + +FASTDEPLOY_DECL std::string Str(const Device& d); + +enum FASTDEPLOY_DECL FDDataType { + BOOL, + INT16, + INT32, + INT64, + FP16, + FP32, + FP64, + UNKNOWN1, + UNKNOWN2, + UNKNOWN3, + UNKNOWN4, + UNKNOWN5, + UNKNOWN6, + UNKNOWN7, + UNKNOWN8, + UNKNOWN9, + UNKNOWN10, + UNKNOWN11, + UNKNOWN12, + UNKNOWN13, + UINT8, + INT8 +}; + +FASTDEPLOY_DECL std::ostream& operator<<(std::ostream& out, const Device& d); + +FASTDEPLOY_DECL std::ostream& operator<<(std::ostream& out, + const FDDataType& fdt); + +FASTDEPLOY_DECL std::string Str(const FDDataType& fdt); + +FASTDEPLOY_DECL int32_t FDDataTypeSize(const FDDataType& data_dtype); + +template +struct FASTDEPLOY_DECL TypeToDataType { + static const FDDataType dtype; +}; + +/*! Deep learning model format */ +enum ModelFormat { + AUTOREC, ///< Auto recognize the model format by model file name + PADDLE, ///< Model with paddlepaddle format + ONNX, ///< Model with ONNX format + RKNN, ///< Model with RKNN format + TORCHSCRIPT, ///< Model with TorchScript format +}; + +FASTDEPLOY_DECL std::ostream& operator<<(std::ostream& out, + const ModelFormat& format); + +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/core/float16.h b/3rdparty/include/fastdeploy/core/float16.h new file mode 100644 index 0000000000..148adde480 --- /dev/null +++ b/3rdparty/include/fastdeploy/core/float16.h @@ -0,0 +1,665 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +#include +#include +#include + +#if !defined(_WIN32) +#define FD_ALIGN(x) __attribute__((aligned(x))) +#else +#define FD_ALIGN(x) __declspec(align(x)) +#endif + +namespace fastdeploy { + +struct FD_ALIGN(2) float16 { + public: + uint16_t x; + + // The following defaulted special class member functions + // are added to make float16 pass the std::is_trivial test + float16() = default; + float16(const float16& o) = default; + float16& operator=(const float16& o) = default; + float16(float16&& o) = default; + float16& operator=(float16&& o) = default; + ~float16() = default; + +// Constructors + +#ifdef FD_WITH_NATIVE_FP16 + // __fp16 is a native half precision data type for arm cpu, + // float16_t is an alias for __fp16 + inline explicit float16(const float16_t& h) { + x = *reinterpret_cast(&h); + } +#endif + + inline explicit float16(float val) { +#if defined(FD_WITH_NATIVE_FP16) + float32x4_t tmp = vld1q_dup_f32(&val); + float16_t res = vget_lane_f16(vcvt_f16_f32(tmp), 0); + x = *reinterpret_cast(&res); + +#elif defined(__F16C__) + x = _cvtss_sh(val, 0); + +#else + // Conversion routine adapted from + // http://stackoverflow.com/questions/1659440/32-bit-to-16-bit-floating-point-conversion + Bits v, s; + v.f = val; + uint32_t sign = v.si & sigN; + v.si ^= sign; + sign >>= shiftSign; // logical shift + s.si = mulN; + s.si = s.f * v.f; // correct subnormals + v.si ^= (s.si ^ v.si) & -(minN > v.si); + v.si ^= (infN ^ v.si) & -((infN > v.si) & (v.si > maxN)); + v.si ^= (nanN ^ v.si) & -((nanN > v.si) & (v.si > infN)); + v.ui >>= shift; // logical shift + v.si ^= ((v.si - maxD) ^ v.si) & -(v.si > maxC); + v.si ^= ((v.si - minD) ^ v.si) & -(v.si > subC); + x = v.ui | sign; + +#endif + } + + inline explicit float16(bool b) : x(b ? 0x3c00 : 0) {} + + template + inline explicit float16(const T& val) + : x(float16(static_cast(val)).x) {} + +// Assignment operators + +#ifdef FD_WITH_NATIVE_FP16 + inline float16& operator=(const float16_t& rhs) { + x = *reinterpret_cast(&rhs); + return *this; + } +#endif + + inline float16& operator=(bool b) { + x = b ? 0x3c00 : 0; + return *this; + } + + inline float16& operator=(int8_t val) { + x = float16(val).x; + return *this; + } + + inline float16& operator=(uint8_t val) { + x = float16(val).x; + return *this; + } + + inline float16& operator=(int16_t val) { + x = float16(val).x; + return *this; + } + + inline float16& operator=(uint16_t val) { + x = float16(val).x; + return *this; + } + + inline float16& operator=(int32_t val) { + x = float16(val).x; + return *this; + } + + inline float16& operator=(uint32_t val) { + x = float16(val).x; + return *this; + } + + inline float16& operator=(int64_t val) { + x = float16(val).x; + return *this; + } + + inline float16& operator=(uint64_t val) { + x = float16(val).x; + return *this; + } + + inline float16& operator=(float val) { + x = float16(val).x; + return *this; + } + + inline float16& operator=(double val) { + x = float16(val).x; + return *this; + } + +// Conversion opertors +#ifdef FD_WITH_NATIVE_FP16 + HOSTDEVICE inline explicit operator float16_t() const { + return *reinterpret_cast(this); + } +#endif + + inline operator float() const { +#if defined(FD_WITH_NATIVE_FP16) + float16x4_t res = vld1_dup_f16(reinterpret_cast(this)); + return vgetq_lane_f32(vcvt_f32_f16(res), 0); + +#elif defined(__F16C__) + return _cvtsh_ss(this->x); + +#else + // Conversion routine adapted from + // http://stackoverflow.com/questions/1659440/32-bit-to-16-bit-floating-point-conversion + Bits v; + v.ui = this->x; + int32_t sign = v.si & sigC; + v.si ^= sign; + sign <<= shiftSign; + v.si ^= ((v.si + minD) ^ v.si) & -(v.si > subC); + v.si ^= ((v.si + maxD) ^ v.si) & -(v.si > maxC); + Bits s; + s.si = mulC; + s.f *= v.si; + int32_t mask = -(norC > v.si); + v.si <<= shift; + v.si ^= (s.si ^ v.si) & mask; + v.si |= sign; + return v.f; + +#endif + } + + inline explicit operator bool() const { return (x & 0x7fff) != 0; } + + inline explicit operator int8_t() const { + return static_cast(static_cast(*this)); + } + + inline explicit operator uint8_t() const { + return static_cast(static_cast(*this)); + } + + inline explicit operator int16_t() const { + return static_cast(static_cast(*this)); + } + + inline explicit operator uint16_t() const { + return static_cast(static_cast(*this)); + } + + inline explicit operator int32_t() const { + return static_cast(static_cast(*this)); + } + + inline explicit operator uint32_t() const { + return static_cast(static_cast(*this)); + } + + inline explicit operator int64_t() const { + return static_cast(static_cast(*this)); + } + + inline explicit operator uint64_t() const { + return static_cast(static_cast(*this)); + } + + inline operator double() const { + return static_cast(static_cast(*this)); + } + + inline bool operator>(const float& other) const { + return this->operator float() > other; + } + + inline bool operator>(const double& other) const { + return this->operator double() > other; + } + + inline bool operator<(const float& other) const { + return this->operator float() > other; + } + + inline bool operator<(const double& other) const { + return this->operator double() > other; + } + + template ::value, + bool>::type = true> + inline float16& operator+=(const T& other) { + *this = float16(static_cast(*this) + other); + return *this; + } + + private: + union Bits { + float f; + int32_t si; + uint32_t ui; + }; + + static const int shift = 13; + static const int shiftSign = 16; + + static const int32_t infN = 0x7F800000; + static const int32_t maxN = 0x477FE000; // max flt16 as flt32 + static const int32_t minN = 0x38800000; // min flt16 normal as flt32 + static const int32_t sigN = 0x80000000; // sign bit + + static constexpr int32_t infC = infN >> shift; + static constexpr int32_t nanN = (infC + 1) + << shift; // minimum flt16 nan as float32 + static constexpr int32_t maxC = maxN >> shift; + static constexpr int32_t minC = minN >> shift; + static constexpr int32_t sigC = sigN >> shiftSign; + + static const int32_t mulN = 0x52000000; // (1 << 23) / minN + static const int32_t mulC = 0x33800000; // minN / (1 << (23 - shift)) + static const int32_t subC = 0x003FF; // max flt32 subnormal downshifted + static const int32_t norC = 0x00400; // min flt32 normal downshifted + + static constexpr int32_t maxD = infC - maxC - 1; + static constexpr int32_t minD = minC - subC - 1; +}; + +// Arithmetic operators for float16 on ARMv8.2-A CPU +#if defined(FD_WITH_NATIVE_FP16) +inline float16 operator+(const float16& a, const float16& b) { + float16 res; + asm volatile( + "ld1 {v0.h}[0], [%[a_ptr]]\n" + "ld1 {v1.h}[0], [%[b_ptr]]\n" + "fadd h0, h0, h1\n" + "st1 {v0.h}[0], [%[res_ptr]]\n" + : // outputs + : // inputs + [a_ptr] "r"(&(a.x)), [b_ptr] "r"(&(b.x)), + [res_ptr] "r"(&(res.x)) + : // clobbers + "memory", "v0", "v1"); + return res; +} + +inline float16 operator-(const float16& a, const float16& b) { + float16 res; + asm volatile( + "ld1 {v0.h}[0], [%[a_ptr]]\n" + "ld1 {v1.h}[0], [%[b_ptr]]\n" + "fsub h0, h0, h1\n" + "st1 {v0.h}[0], [%[res_ptr]]\n" + : // outputs + : // inputs + [a_ptr] "r"(&(a.x)), [b_ptr] "r"(&(b.x)), + [res_ptr] "r"(&(res.x)) + : // clobbers + "memory", "v0", "v1"); + return res; +} + +inline float16 operator*(const float16& a, const float16& b) { + float16 res; + asm volatile( + "ld1 {v0.h}[0], [%[a_ptr]]\n" + "ld1 {v1.h}[0], [%[b_ptr]]\n" + "fmul h0, h0, h1\n" + "st1 {v0.h}[0], [%[res_ptr]]\n" + : // outputs + : // inputs + [a_ptr] "r"(&(a.x)), [b_ptr] "r"(&(b.x)), + [res_ptr] "r"(&(res.x)) + : // clobbers + "memory", "v0", "v1"); + return res; +} + +inline float16 operator/(const float16& a, const float16& b) { + float16 res; + asm volatile( + "ld1 {v0.h}[0], [%[a_ptr]]\n" + "ld1 {v1.h}[0], [%[b_ptr]]\n" + "fdiv h0, h0, h1\n" + "st1 {v0.h}[0], [%[res_ptr]]\n" + : // outputs + : // inputs + [a_ptr] "r"(&(a.x)), [b_ptr] "r"(&(b.x)), + [res_ptr] "r"(&(res.x)) + : // clobbers + "memory", "v0", "v1"); + return res; +} + +inline float16 operator-(const float16& a) { + float16 res; + asm volatile( + "ld1 {v0.h}[0], [%[a_ptr]]\n" + "fneg h0, h0\n" + "st1 {v0.h}[0], [%[res_ptr]]\n" + : // outputs + : // inputs + [a_ptr] "r"(&(a.x)), + [res_ptr] "r"(&(res.x)) + : // clobbers + "memory", "v0"); + return res; +} + +inline float16& operator+=(float16& a, const float16& b) { // NOLINT + a = a + b; + return a; +} + +inline float16& operator-=(float16& a, const float16& b) { // NOLINT + a = a - b; + return a; +} + +inline float16& operator*=(float16& a, const float16& b) { // NOLINT + a = a * b; + return a; +} + +inline float16& operator/=(float16& a, const float16& b) { // NOLINT + a = a / b; + return a; +} + +inline bool operator==(const float16& a, const float16& b) { + uint16_t res; + asm volatile( + "ld1 {v0.h}[0], [%[a_ptr]]\n" + "ld1 {v1.h}[0], [%[b_ptr]]\n" + "fcmeq h0, h0, h1\n" + "st1 {v0.h}[0], [%[res_ptr]]\n" + : // outputs + : // inputs + [a_ptr] "r"(&(a.x)), [b_ptr] "r"(&(b.x)), + [res_ptr] "r"(&res) + : // clobbers + "memory", "v0", "v1"); + return (res & 0xffff) != 0; +} + +inline bool operator!=(const float16& a, const float16& b) { return !(a == b); } + +inline bool operator<(const float16& a, const float16& b) { + uint16_t res; + asm volatile( + "ld1 {v1.h}[0], [%[a_ptr]]\n" + "ld1 {v0.h}[0], [%[b_ptr]]\n" + "fcmgt h0, h0, h1\n" + "st1 {v0.h}[0], [%[res_ptr]]\n" + : // outputs + : // inputs + [a_ptr] "r"(&(a.x)), [b_ptr] "r"(&(b.x)), + [res_ptr] "r"(&res) + : // clobbers + "memory", "v0", "v1"); + return (res & 0xffff) != 0; +} + +inline bool operator<=(const float16& a, const float16& b) { + uint16_t res; + asm volatile( + "ld1 {v1.h}[0], [%[a_ptr]]\n" + "ld1 {v0.h}[0], [%[b_ptr]]\n" + "fcmge h0, h0, h1\n" + "st1 {v0.h}[0], [%[res_ptr]]\n" + : // outputs + : // inputs + [a_ptr] "r"(&(a.x)), [b_ptr] "r"(&(b.x)), + [res_ptr] "r"(&res) + : // clobbers + "memory", "v0", "v1"); + return (res & 0xffff) != 0; +} + +inline bool operator>(const float16& a, const float16& b) { + uint16_t res; + asm volatile( + "ld1 {v0.h}[0], [%[a_ptr]]\n" + "ld1 {v1.h}[0], [%[b_ptr]]\n" + "fcmgt h0, h0, h1\n" + "st1 {v0.h}[0], [%[res_ptr]]\n" + : // outputs + : // inputs + [a_ptr] "r"(&(a.x)), [b_ptr] "r"(&(b.x)), + [res_ptr] "r"(&res) + : // clobbers + "memory", "v0", "v1"); + return (res & 0xffff) != 0; +} + +inline bool operator>=(const float16& a, const float16& b) { + uint16_t res; + asm volatile( + "ld1 {v0.h}[0], [%[a_ptr]]\n" + "ld1 {v1.h}[0], [%[b_ptr]]\n" + "fcmge h0, h0, h1\n" + "st1 {v0.h}[0], [%[res_ptr]]\n" + : // outputs + : // inputs + [a_ptr] "r"(&(a.x)), [b_ptr] "r"(&(b.x)), + [res_ptr] "r"(&res) + : // clobbers + "memory", "v0", "v1"); + return (res & 0xffff) != 0; +#else +inline float16 operator+(const float16& a, const float16& b) { + return float16(static_cast(a) + static_cast(b)); +} + +inline float16 operator-(const float16& a, const float16& b) { + return float16(static_cast(a) - static_cast(b)); +} + +inline float16 operator*(const float16& a, const float16& b) { + return float16(static_cast(a) * static_cast(b)); +} + +inline float16 operator/(const float16& a, const float16& b) { + return float16(static_cast(a) / static_cast(b)); +} + +inline float16 operator-(const float16& a) { + float16 res; + res.x = a.x ^ 0x8000; + return res; +} + +inline float16& operator+=(float16& a, const float16& b) { // NOLINT + a = float16(static_cast(a) + static_cast(b)); + return a; +} + +inline float16& operator-=(float16& a, const float16& b) { // NOLINT + a = float16(static_cast(a) - static_cast(b)); + return a; +} + +inline float16& operator*=(float16& a, const float16& b) { // NOLINT + a = float16(static_cast(a) * static_cast(b)); + return a; +} + +inline float16& operator/=(float16& a, const float16& b) { // NOLINT + a = float16(static_cast(a) / static_cast(b)); + return a; +} + +inline bool operator==(const float16& a, const float16& b) { + return static_cast(a) == static_cast(b); +} + +inline bool operator!=(const float16& a, const float16& b) { + return static_cast(a) != static_cast(b); +} + +inline bool operator<(const float16& a, const float16& b) { + return static_cast(a) < static_cast(b); +} + +inline bool operator<=(const float16& a, const float16& b) { + return static_cast(a) <= static_cast(b); +} + +inline bool operator>(const float16& a, const float16& b) { + return static_cast(a) > static_cast(b); +} + +inline bool operator>=(const float16& a, const float16& b) { + return static_cast(a) >= static_cast(b); +} +#endif + + template ::value || + std::is_same::value, + bool>::type = true> + inline T& operator+=(T& a, const float16& b) { // NOLINT + auto c = static_cast(a) + static_cast(b); + a = static_cast(c); + return a; + } + + inline double& operator+=(double& a, const float16& b) { // NOLINT + a = a + static_cast(b); + return a; + } + + inline float16 raw_uint16_to_float16(uint16_t a) { + float16 res; + res.x = a; + return res; + } + + inline bool(isnan)(const float16& a) { return (a.x & 0x7fff) > 0x7c00; } + + inline bool(isinf)(const float16& a) { return (a.x & 0x7fff) == 0x7c00; } + + inline bool(isfinite)(const float16& a) { + return !((isnan)(a)) && !((isinf)(a)); + } + + inline float16(abs)(const float16& a) { + return float16(std::abs(static_cast(a))); + } + + inline std::ostream& operator<<(std::ostream& os, const float16& a) { + os << static_cast(a); + return os; + } +} // namespace fastdeploy + +namespace std { + +// Override the std::is_pod::value for float16 +// The reason is that different compilers implemented std::is_pod based on +// different C++ standards. float16 class is a plain old data in C++11 given +// that it is both trivial and standard_layout. +// However, std::is_pod in nvcc 8.0 host c++ compiler follows C++0x and is +// more restricted in that you cannot provide any customized +// constructor in float16. Hence, we override is_pod here following C++11 +// so that .cu files can be successfully compiled by nvcc. +template <> +struct is_pod { + static const bool value = is_trivial::value && + is_standard_layout::value; +}; + +template <> +struct is_floating_point + : std::integral_constant< + bool, std::is_same::type>::value> {}; +template <> +struct is_signed { + static const bool value = true; +}; + +template <> +struct is_unsigned { + static const bool value = false; +}; + +inline bool isnan(const fastdeploy::float16& a) { return fastdeploy::isnan(a); } + +inline bool isinf(const fastdeploy::float16& a) { return fastdeploy::isinf(a); } + +template <> +struct numeric_limits { + static const bool is_specialized = true; + static const bool is_signed = true; + static const bool is_integer = false; + static const bool is_exact = false; + static const bool has_infinity = true; + static const bool has_quiet_NaN = true; + static const bool has_signaling_NaN = true; + static const float_denorm_style has_denorm = denorm_present; + static const bool has_denorm_loss = false; + static const std::float_round_style round_style = std::round_to_nearest; + static const bool is_iec559 = false; + static const bool is_bounded = false; + static const bool is_modulo = false; + static const int digits = 11; + static const int digits10 = 3; + static const int max_digits10 = 5; + static const int radix = 2; + static const int min_exponent = -13; + static const int min_exponent10 = -4; + static const int max_exponent = 16; + static const int max_exponent10 = 4; + static const bool traps = true; + static const bool tinyness_before = false; + + static fastdeploy::float16(min)() { + return fastdeploy::raw_uint16_to_float16(0x400); + } + static fastdeploy::float16 lowest() { + return fastdeploy::raw_uint16_to_float16(0xfbff); + } + static fastdeploy::float16(max)() { + return fastdeploy::raw_uint16_to_float16(0x7bff); + } + static fastdeploy::float16 epsilon() { + return fastdeploy::raw_uint16_to_float16(0x0800); + } + static fastdeploy::float16 round_error() { return fastdeploy::float16(0.5); } + static fastdeploy::float16 infinity() { + return fastdeploy::raw_uint16_to_float16(0x7c00); + } + static fastdeploy::float16 quiet_NaN() { + return fastdeploy::raw_uint16_to_float16(0x7e00); + } + static fastdeploy::float16 signaling_NaN() { + return fastdeploy::raw_uint16_to_float16(0x7e00); + } + static fastdeploy::float16 denorm_min() { + return fastdeploy::raw_uint16_to_float16(0x1); + } +}; + +inline fastdeploy::float16 abs(const fastdeploy::float16& a) { + return fastdeploy::abs(a); +} + +} // namespace std diff --git a/3rdparty/include/fastdeploy/fastdeploy_model.h b/3rdparty/include/fastdeploy/fastdeploy_model.h new file mode 100644 index 0000000000..717639694e --- /dev/null +++ b/3rdparty/include/fastdeploy/fastdeploy_model.h @@ -0,0 +1,142 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once +#include "fastdeploy/runtime.h" + +namespace fastdeploy { + +/*! @brief Base model object for all the vision models + */ +class FASTDEPLOY_DECL FastDeployModel { + public: + /// Get model's name + virtual std::string ModelName() const { return "NameUndefined"; } + + /** \brief Inference the model by the runtime. This interface is included in the `Predict()` function, so we don't call `Infer()` directly in most common situation + */ + virtual bool Infer(std::vector& input_tensors, + std::vector* output_tensors); + + /** \brief Inference the model by the runtime. This interface is using class member reused_input_tensors_ to do inference and writing results to reused_output_tensors_ + */ + virtual bool Infer(); + + RuntimeOption runtime_option; + /** \brief Model's valid cpu backends. This member defined all the cpu backends have successfully tested for the model + */ + std::vector valid_cpu_backends = {Backend::ORT}; + /** Model's valid gpu backends. This member defined all the gpu backends have successfully tested for the model + */ + std::vector valid_gpu_backends = {Backend::ORT}; + /** Model's valid ipu backends. This member defined all the ipu backends have successfully tested for the model + */ + std::vector valid_ipu_backends = {}; + /** Model's valid timvx backends. This member defined all the timvx backends have successfully tested for the model + */ + std::vector valid_timvx_backends = {}; + /** Model's valid hardware backends. This member defined all the gpu backends have successfully tested for the model + */ + std::vector valid_rknpu_backends = {}; + + /// Get number of inputs for this model + virtual int NumInputsOfRuntime() { return runtime_->NumInputs(); } + /// Get number of outputs for this model + virtual int NumOutputsOfRuntime() { return runtime_->NumOutputs(); } + /// Get input information for this model + virtual TensorInfo InputInfoOfRuntime(int index) { + return runtime_->GetInputInfo(index); + } + /// Get output information for this model + virtual TensorInfo OutputInfoOfRuntime(int index) { + return runtime_->GetOutputInfo(index); + } + /// Check if the model is initialized successfully + virtual bool Initialized() const { + return runtime_initialized_ && initialized; + } + + /** \brief This is a debug interface, used to record the time of backend runtime + * + * example code @code + * auto model = fastdeploy::vision::PPYOLOE("model.pdmodel", "model.pdiparams", "infer_cfg.yml"); + * if (!model.Initialized()) { + * std::cerr << "Failed to initialize." << std::endl; + * return -1; + * } + * model.EnableRecordTimeOfRuntime(); + * cv::Mat im = cv::imread("test.jpg"); + * for (auto i = 0; i < 1000; ++i) { + * fastdeploy::vision::DetectionResult result; + * model.Predict(&im, &result); + * } + * model.PrintStatisInfoOfRuntime(); + * @endcode After called the `PrintStatisInfoOfRuntime()`, the statistical information of runtime will be printed in the console + */ + virtual void EnableRecordTimeOfRuntime() { + time_of_runtime_.clear(); + std::vector().swap(time_of_runtime_); + enable_record_time_of_runtime_ = true; + } + + /** \brief Disable to record the time of backend runtime, see `EnableRecordTimeOfRuntime()` for more detail + */ + virtual void DisableRecordTimeOfRuntime() { + enable_record_time_of_runtime_ = false; + } + + /** \brief Print the statistic information of runtime in the console, see function `EnableRecordTimeOfRuntime()` for more detail + */ + virtual std::map PrintStatisInfoOfRuntime(); + + /** \brief Check if the `EnableRecordTimeOfRuntime()` method is enabled. + */ + virtual bool EnabledRecordTimeOfRuntime() { + return enable_record_time_of_runtime_; + } + + /** \brief Release reused input/output buffers + */ + virtual void ReleaseReusedBuffer() { + std::vector().swap(reused_input_tensors_); + std::vector().swap(reused_output_tensors_); + } + + protected: + virtual bool InitRuntime(); + + bool initialized = false; + // Reused input tensors + std::vector reused_input_tensors_; + // Reused output tensors + std::vector reused_output_tensors_; + + private: + bool InitRuntimeWithSpecifiedBackend(); + bool InitRuntimeWithSpecifiedDevice(); + bool CreateCpuBackend(); + bool CreateGpuBackend(); + bool CreateIpuBackend(); + bool CreateRKNPUBackend(); + bool CreateTimVXBackend(); + + std::shared_ptr runtime_; + bool runtime_initialized_ = false; + // whether to record inference time + bool enable_record_time_of_runtime_ = false; + + // record inference time for backend + std::vector time_of_runtime_; +}; + +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/function/concat.h b/3rdparty/include/fastdeploy/function/concat.h new file mode 100644 index 0000000000..2a89300bc8 --- /dev/null +++ b/3rdparty/include/fastdeploy/function/concat.h @@ -0,0 +1,32 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "fastdeploy/core/fd_tensor.h" + +namespace fastdeploy { +namespace function { + +/** Excute the concatenate operation for input FDTensor along given axis. + @param x The input tensor. + @param out The output tensor which stores the result. + @param axisi Axis which will be concatenated. +*/ + +FASTDEPLOY_DECL void Concat(const std::vector& x, FDTensor* out, + int axis = 0); + +} // namespace function +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/function/cuda_cast.h b/3rdparty/include/fastdeploy/function/cuda_cast.h new file mode 100644 index 0000000000..0d6d65ca08 --- /dev/null +++ b/3rdparty/include/fastdeploy/function/cuda_cast.h @@ -0,0 +1,29 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "fastdeploy/core/fd_tensor.h" + +namespace fastdeploy { +namespace function { +/** Cast the type of the data in GPU buffer. + @param in The input tensor. + @param out The output tensor + @param stream CUDA stream +*/ +FASTDEPLOY_DECL void CudaCast(const FDTensor& in, FDTensor* out, + cudaStream_t stream); +} // namespace function +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/function/eigen.h b/3rdparty/include/fastdeploy/function/eigen.h new file mode 100644 index 0000000000..cde0700d8e --- /dev/null +++ b/3rdparty/include/fastdeploy/function/eigen.h @@ -0,0 +1,140 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include "fastdeploy/core/fd_tensor.h" +#include "fastdeploy/utils/axis_utils.h" +#include "unsupported/Eigen/CXX11/Tensor" + +namespace fastdeploy { +namespace function { +// EigenDim converts shape into Eigen::DSizes. +template +struct EigenDim { + using Type = Eigen::DSizes; + + static Type From(const std::vector& dims) { + Type ret; + for (int64_t d = 0; d < dims.size(); d++) { + ret[d] = dims[d]; + } + return ret; + } +}; + +// Interpret FDTensor as EigenTensor and EigenConstTensor. +template +struct EigenTensor { + using Type = Eigen::TensorMap>; + + using ConstType = + Eigen::TensorMap>; + + static Type From(FDTensor& tensor, + const std::vector& dims) { // NOLINT + return Type(reinterpret_cast(tensor.Data()), EigenDim::From(dims)); + } + + static Type From(FDTensor& tensor) { // NOLINT + return From(tensor, tensor.shape); + } // NOLINT + + static ConstType From(const FDTensor& tensor, + const std::vector& dims) { + return ConstType(reinterpret_cast(tensor.Data()), + EigenDim::From(dims)); + } + + static ConstType From(const FDTensor& tensor) { + return From(tensor, tensor.shape); + } +}; + +template +struct EigenScalar { + // Scalar tensor (implemented as a rank-0 tensor) of scalar type T. + using Type = Eigen::TensorMap< + Eigen::TensorFixedSize, MajorType, IndexType>>; + using ConstType = Eigen::TensorMap< + Eigen::TensorFixedSize, MajorType, IndexType>>; + + static Type From(FDTensor& tensor) { + return Type(reinterpret_cast(tensor.Data())); + } // NOLINT + + static ConstType From(const FDTensor& tensor) { + return ConstType(reinterpret_cast(tensor.Data())); + } +}; + +template +struct EigenVector : public EigenTensor { + // Flatten reshapes a Tensor into an EigenVector. + static typename EigenVector::Type Flatten(FDTensor& tensor) { // NOLINT + return EigenVector::From(tensor, {tensor.Numel()}); + } + + static typename EigenVector::ConstType Flatten( + const FDTensor& tensor) { // NOLINT + return EigenVector::From(tensor, {tensor.Numel()}); + } +}; + +template +struct EigenMatrix : public EigenTensor { + static typename EigenMatrix::Type Reshape(FDTensor& tensor, // NOLINT + int num_col_dims) { + int rank = tensor.shape.size(); + FDASSERT((num_col_dims > 0 && num_col_dims < rank), + "Input dimension number(num_col_dims) must be between 0 and %d, " + "but received number is %d.", + rank, num_col_dims); + const int n = SizeToAxis(num_col_dims, tensor.shape); + const int d = SizeFromAxis(num_col_dims, tensor.shape); + return EigenMatrix::From(tensor, {n, d}); + } + + static typename EigenMatrix::ConstType Reshape(const FDTensor& tensor, + int num_col_dims) { + int rank = tensor.shape.size(); + FDASSERT((num_col_dims > 0 && num_col_dims < rank), + "Input dimension number(num_col_dims) must be between 0 and %d, " + "but received number is %d.", + rank, num_col_dims); + const int n = SizeToAxis(num_col_dims, tensor.shape); + const int d = SizeFromAxis(num_col_dims, tensor.shape); + return EigenMatrix::From(tensor, {n, d}); + } +}; + +class EigenDeviceWrapper { + public: + static std::shared_ptr GetInstance(); + const Eigen::DefaultDevice* GetDevice() const; + + private: + Eigen::DefaultDevice device_; + static std::shared_ptr instance_; +}; + +} // namespace function +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/function/pad.h b/3rdparty/include/fastdeploy/function/pad.h new file mode 100644 index 0000000000..4995ba584d --- /dev/null +++ b/3rdparty/include/fastdeploy/function/pad.h @@ -0,0 +1,31 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "fastdeploy/core/fd_tensor.h" + +namespace fastdeploy { +namespace function { +/** Excute the pad operation for input FDTensor along given dims. + @param x The input tensor. + @param out The output tensor which stores the result. + @param pads The size of padding for each dimension, for 3-D tensor, the pads should be [1d-left, 1d-right, 2d-left, 2d-right, 3d-left, 3d-right] + @param pad_value The value which will fill into out tensor +*/ +FASTDEPLOY_DECL void Pad(const FDTensor& x, FDTensor* out, + const std::vector& pads, float pad_value = 0); + +} +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/function/reduce.h b/3rdparty/include/fastdeploy/function/reduce.h new file mode 100644 index 0000000000..3521f16f11 --- /dev/null +++ b/3rdparty/include/fastdeploy/function/reduce.h @@ -0,0 +1,127 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "fastdeploy/core/fd_tensor.h" + +namespace fastdeploy { +namespace function { +/** Excute the maximum operation for input FDTensor along given dims. + @param x The input tensor. + @param out The output tensor which stores the result. + @param dims The vector of axis which will be reduced. + @param keep_dim Whether to keep the reduced dims, default false. + @param reduce_all Whether to reduce all dims, default false. +*/ +FASTDEPLOY_DECL void Max(const FDTensor& x, FDTensor* out, + const std::vector& dims, + bool keep_dim = false, bool reduce_all = false); + +/** Excute the minimum operation for input FDTensor along given dims. + @param x The input tensor. + @param out The output tensor which stores the result. + @param dims The vector of axis which will be reduced. + @param keep_dim Whether to keep the reduced dims, default false. + @param reduce_all Whether to reduce all dims, default false. +*/ +FASTDEPLOY_DECL void Min(const FDTensor& x, FDTensor* out, + const std::vector& dims, + bool keep_dim = false, bool reduce_all = false); + +/** Excute the sum operation for input FDTensor along given dims. + @param x The input tensor. + @param out The output tensor which stores the result. + @param dims The vector of axis which will be reduced. + @param keep_dim Whether to keep the reduced dims, default false. + @param reduce_all Whether to reduce all dims, default false. +*/ +FASTDEPLOY_DECL void Sum(const FDTensor& x, FDTensor* out, + const std::vector& dims, + bool keep_dim = false, bool reduce_all = false); + +/** Excute the all operation for input FDTensor along given dims. + @param x The input tensor. + @param out The output tensor which stores the result. + @param dims The vector of axis which will be reduced. + @param keep_dim Whether to keep the reduced dims, default false. + @param reduce_all Whether to reduce all dims, default false. +*/ +FASTDEPLOY_DECL void All(const FDTensor& x, FDTensor* out, + const std::vector& dims, + bool keep_dim = false, bool reduce_all = false); + +/** Excute the any operation for input FDTensor along given dims. + @param x The input tensor. + @param out The output tensor which stores the result. + @param dims The vector of axis which will be reduced. + @param keep_dim Whether to keep the reduced dims, default false. + @param reduce_all Whether to reduce all dims, default false. +*/ +FASTDEPLOY_DECL void Any(const FDTensor& x, FDTensor* out, + const std::vector& dims, + bool keep_dim = false, bool reduce_all = false); + +/** Excute the mean operation for input FDTensor along given dims. + @param x The input tensor. + @param out The output tensor which stores the result. + @param dims The vector of axis which will be reduced. + @param keep_dim Whether to keep the reduced dims, default false. + @param reduce_all Whether to reduce all dims, default false. +*/ +FASTDEPLOY_DECL void Mean(const FDTensor& x, FDTensor* out, + const std::vector& dims, + bool keep_dim = false, bool reduce_all = false); + +/** Excute the product operation for input FDTensor along given dims. + @param x The input tensor. + @param out The output tensor which stores the result. + @param dims The vector of axis which will be reduced. + @param keep_dim Whether to keep the reduced dims, default false. + @param reduce_all Whether to reduce all dims, default false. +*/ +FASTDEPLOY_DECL void Prod(const FDTensor& x, FDTensor* out, + const std::vector& dims, + bool keep_dim = false, bool reduce_all = false); + +/** Excute the argmax operation for input FDTensor along given dims. + @param x The input tensor. + @param out The output tensor which stores the result. + @param axis The axis which will be reduced. + @param output_dtype The data type of output FDTensor, INT64 or INT32, + default to INT64. + @param keep_dim Whether to keep the reduced dims, default false. + @param flatten Whether to flatten FDTensor to get the argmin index, default + false. +*/ +FASTDEPLOY_DECL void ArgMax(const FDTensor& x, FDTensor* out, int64_t axis, + FDDataType output_dtype = FDDataType::INT64, + bool keep_dim = false, bool flatten = false); + +/** Excute the argmin operation for input FDTensor along given dims. + @param x The input tensor. + @param out The output tensor which stores the result. + @param axis The axis which will be reduced. + @param output_dtype The data type of output FDTensor, INT64 or INT32, + default to INT64. + @param keep_dim Whether to keep the reduced dims, default false. + @param flatten Whether to flatten FDTensor to get the argmin index, default + false. +*/ +FASTDEPLOY_DECL void ArgMin(const FDTensor& x, FDTensor* out, int64_t axis, + FDDataType output_dtype = FDDataType::INT64, + bool keep_dim = false, bool flatten = false); + +} // namespace function +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/function/reduce_functor.h b/3rdparty/include/fastdeploy/function/reduce_functor.h new file mode 100644 index 0000000000..d048beca94 --- /dev/null +++ b/3rdparty/include/fastdeploy/function/reduce_functor.h @@ -0,0 +1,77 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "fastdeploy/function/eigen.h" +namespace fastdeploy { +namespace function { +//////// Max Functor /////// +struct MaxFunctor { + template + void operator()(const Eigen::DefaultDevice& dev, X* x, Y* y, const Dim& dim) { + y->device(dev) = x->maximum(dim); + } +}; + +//////// Min Functor /////// +struct MinFunctor { + template + void operator()(const Eigen::DefaultDevice& dev, X* x, Y* y, const Dim& dim) { + y->device(dev) = x->minimum(dim); + } +}; + +//////// Sum Functor /////// +struct SumFunctor { + template + void operator()(const Eigen::DefaultDevice& dev, X* x, Y* y, const Dim& dim) { + y->device(dev) = x->sum(dim); + } +}; + +//////// All Functor /////// +struct AllFunctor { + template + void operator()(const Eigen::DefaultDevice& dev, X* x, Y* y, const Dim& dim) { + y->device(dev) = x->all(dim); + } +}; + +//////// Any Functor /////// +struct AnyFunctor { + template + void operator()(const Eigen::DefaultDevice& dev, X* x, Y* y, const Dim& dim) { + y->device(dev) = x->any(dim); + } +}; + +//////// Mean Functor /////// +struct MeanFunctor { + template + void operator()(const Eigen::DefaultDevice& dev, X* x, Y* y, const Dim& dim) { + y->device(dev) = x->mean(dim); + } +}; + +//////// Prod Functor /////// +struct ProdFunctor { + template + void operator()(const Eigen::DefaultDevice& dev, X* x, Y* y, const Dim& dim) { + y->device(dev) = x->prod(dim); + } +}; + +} // namespace function +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/function/softmax.h b/3rdparty/include/fastdeploy/function/softmax.h new file mode 100644 index 0000000000..3cd2ed3a77 --- /dev/null +++ b/3rdparty/include/fastdeploy/function/softmax.h @@ -0,0 +1,29 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "fastdeploy/core/fd_tensor.h" + +namespace fastdeploy { +namespace function { +/** Excute the softmax operation for input FDTensor along given dims. + @param x The input tensor. + @param out The output tensor which stores the result. + @param axis The axis to be computed softmax value. +*/ +FASTDEPLOY_DECL void Softmax(const FDTensor& x, FDTensor* out, int axis = -1); + +} // namespace function +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/function/transpose.h b/3rdparty/include/fastdeploy/function/transpose.h new file mode 100644 index 0000000000..9bae6e66c7 --- /dev/null +++ b/3rdparty/include/fastdeploy/function/transpose.h @@ -0,0 +1,29 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "fastdeploy/core/fd_tensor.h" + +namespace fastdeploy { +namespace function { +/** Excute the transpose operation for input FDTensor along given dims. + @param x The input tensor. + @param out The output tensor which stores the result. + @param dims The vector of axis which the input tensor will transpose. +*/ +FASTDEPLOY_DECL void Transpose(const FDTensor& x, FDTensor* out, + const std::vector& dims); +} // namespace function +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/pipeline.h b/3rdparty/include/fastdeploy/pipeline.h new file mode 100644 index 0000000000..5d16935ccc --- /dev/null +++ b/3rdparty/include/fastdeploy/pipeline.h @@ -0,0 +1,21 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once + +#include "fastdeploy/core/config.h" +#ifdef ENABLE_VISION +#include "fastdeploy/pipeline/pptinypose/pipeline.h" +#endif + +#include "fastdeploy/vision/visualize/visualize.h" diff --git a/3rdparty/include/fastdeploy/pipeline/pptinypose/pipeline.h b/3rdparty/include/fastdeploy/pipeline/pptinypose/pipeline.h new file mode 100644 index 0000000000..0cdbee3995 --- /dev/null +++ b/3rdparty/include/fastdeploy/pipeline/pptinypose/pipeline.h @@ -0,0 +1,67 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "fastdeploy/fastdeploy_model.h" +#include "fastdeploy/vision/common/result.h" +#include "fastdeploy/vision/detection/ppdet/model.h" +#include "fastdeploy/vision/keypointdet/pptinypose/pptinypose.h" + +namespace fastdeploy { +/** \brief All pipeline model APIs are defined inside this namespace + * + */ +namespace pipeline { + +/*! @brief PPTinyPose Pipeline object used when to load a detection model + pptinypose model + */ +class FASTDEPLOY_DECL PPTinyPose { + public: + /** \brief Set initialized detection model object and pptinypose model object + * + * \param[in] det_model Initialized detection model object + * \param[in] pptinypose_model Initialized pptinypose model object + */ + PPTinyPose( + fastdeploy::vision::detection::PicoDet* det_model, + fastdeploy::vision::keypointdetection::PPTinyPose* pptinypose_model); + + /** \brief Predict the keypoint detection result for an input image + * + * \param[in] img The input image data, comes from cv::imread() + * \param[in] result The output keypoint detection result will be writen to this structure + * \return true if the prediction successed, otherwise false + */ + virtual bool Predict(cv::Mat* img, + fastdeploy::vision::KeyPointDetectionResult* result); + + /* \brief The score threshold for detectin model to filter bbox before inputting pptinypose model + */ + float detection_model_score_threshold = 0; + + protected: + fastdeploy::vision::detection::PicoDet* detector_ = nullptr; + fastdeploy::vision::keypointdetection::PPTinyPose* pptinypose_model_ = + nullptr; + + virtual bool Detect(cv::Mat* img, + fastdeploy::vision::DetectionResult* result); + virtual bool KeypointDetect( + cv::Mat* img, fastdeploy::vision::KeyPointDetectionResult* result, + fastdeploy::vision::DetectionResult& detection_result); +}; + +} // namespace pipeline +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/runtime.h b/3rdparty/include/fastdeploy/runtime.h new file mode 100644 index 0000000000..8822308ec7 --- /dev/null +++ b/3rdparty/include/fastdeploy/runtime.h @@ -0,0 +1,432 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/*! \file runtime.h + \brief A brief file description. + + More details + */ + +#pragma once + +#include +#include +#include + +#include "fastdeploy/backends/backend.h" +#include "fastdeploy/utils/perf.h" +#include "backends/rknpu/rknpu2/rknpu2_config.h" + +/** \brief All C++ FastDeploy APIs are defined inside this namespace +* +*/ +namespace fastdeploy { + +/*! Inference backend supported in FastDeploy */ +enum Backend { + UNKNOWN, ///< Unknown inference backend + ORT, ///< ONNX Runtime, support Paddle/ONNX format model, CPU / Nvidia GPU + TRT, ///< TensorRT, support Paddle/ONNX format model, Nvidia GPU only + PDINFER, ///< Paddle Inference, support Paddle format model, CPU / Nvidia GPU + POROS, ///< Poros, support TorchScript format model, CPU / Nvidia GPU + OPENVINO, ///< Intel OpenVINO, support Paddle/ONNX format, CPU only + LITE, ///< Paddle Lite, support Paddle format model, ARM CPU only + RKNPU2, ///< RKNPU2, support RKNN format model, Rockchip NPU only +}; + +FASTDEPLOY_DECL std::ostream& operator<<(std::ostream& out, + const Backend& backend); + +/*! Paddle Lite power mode for mobile device. */ +enum LitePowerMode { + LITE_POWER_HIGH = 0, ///< Use Lite Backend with high power mode + LITE_POWER_LOW = 1, ///< Use Lite Backend with low power mode + LITE_POWER_FULL = 2, ///< Use Lite Backend with full power mode + LITE_POWER_NO_BIND = 3, ///< Use Lite Backend with no bind power mode + LITE_POWER_RAND_HIGH = 4, ///< Use Lite Backend with rand high mode + LITE_POWER_RAND_LOW = 5 ///< Use Lite Backend with rand low power mode +}; + +FASTDEPLOY_DECL std::string Str(const Backend& b); +FASTDEPLOY_DECL std::string Str(const ModelFormat& f); + +/** + * @brief Get all the available inference backend in FastDeploy + */ +FASTDEPLOY_DECL std::vector GetAvailableBackends(); + +/** + * @brief Check if the inference backend available + */ +FASTDEPLOY_DECL bool IsBackendAvailable(const Backend& backend); + +bool CheckModelFormat(const std::string& model_file, + const ModelFormat& model_format); +ModelFormat GuessModelFormat(const std::string& model_file); + +/*! @brief Option object used when create a new Runtime object + */ +struct FASTDEPLOY_DECL RuntimeOption { + /** \brief Set path of model file and parameter file + * + * \param[in] model_path Path of model file, e.g ResNet50/model.pdmodel for Paddle format model / ResNet50/model.onnx for ONNX format model + * \param[in] params_path Path of parameter file, this only used when the model format is Paddle, e.g Resnet50/model.pdiparams + * \param[in] format Format of the loaded model + */ + void SetModelPath(const std::string& model_path, + const std::string& params_path = "", + const ModelFormat& format = ModelFormat::PADDLE); + + /// Use cpu to inference, the runtime will inference on CPU by default + void UseCpu(); + + /// Use Nvidia GPU to inference + void UseGpu(int gpu_id = 0); + + void UseRKNPU2(fastdeploy::rknpu2::CpuName rknpu2_name + = fastdeploy::rknpu2::CpuName::RK3588, + fastdeploy::rknpu2::CoreMask rknpu2_core + = fastdeploy::rknpu2::CoreMask::RKNN_NPU_CORE_0); + + /// Use TimVX to inference + void UseTimVX(); + + void SetExternalStream(void* external_stream); + + /* + * @brief Set number of cpu threads while inference on CPU, by default it will decided by the different backends + */ + void SetCpuThreadNum(int thread_num); + + /// Set ORT graph opt level, default is decide by ONNX Runtime itself + void SetOrtGraphOptLevel(int level = -1); + + /// Set Paddle Inference as inference backend, support CPU/GPU + void UsePaddleBackend(); + + /// Wrapper function of UsePaddleBackend() + void UsePaddleInferBackend() { + return UsePaddleBackend(); + } + + /// Set ONNX Runtime as inference backend, support CPU/GPU + void UseOrtBackend(); + + /// Set TensorRT as inference backend, only support GPU + void UseTrtBackend(); + + /// Set Poros backend as inference backend, support CPU/GPU + void UsePorosBackend(); + + /// Set OpenVINO as inference backend, only support CPU + void UseOpenVINOBackend(); + + /// Set Paddle Lite as inference backend, only support arm cpu + void UseLiteBackend(); + + /// Wrapper function of UseLiteBackend() + void UsePaddleLiteBackend() { + return UseLiteBackend(); + } + + /// Set mkldnn switch while using Paddle Inference as inference backend + void SetPaddleMKLDNN(bool pd_mkldnn = true); + + /* + * @brief If TensorRT backend is used, EnablePaddleToTrt will change to use Paddle Inference backend, and use its integrated TensorRT instead. + */ + void EnablePaddleToTrt(); + + /** + * @brief Delete pass by name while using Paddle Inference as inference backend, this can be called multiple times to delete a set of passes + */ + void DeletePaddleBackendPass(const std::string& delete_pass_name); + + /** + * @brief Enable print debug information while using Paddle Inference as inference backend, the backend disable the debug information by default + */ + void EnablePaddleLogInfo(); + + /** + * @brief Disable print debug information while using Paddle Inference as inference backend + */ + void DisablePaddleLogInfo(); + + /** + * @brief Set shape cache size while using Paddle Inference with mkldnn, by default it will cache all the difference shape + */ + void SetPaddleMKLDNNCacheSize(int size); + + /** + * @brief Set optimzed model dir for Paddle Lite backend. + */ + void SetLiteOptimizedModelDir(const std::string& optimized_model_dir); + + /** + * @brief Set nnadapter subgraph partition path for Paddle Lite backend. + */ + void SetLiteSubgraphPartitionPath( + const std::string& nnadapter_subgraph_partition_config_path); + + /** + * @brief enable half precision while use paddle lite backend + */ + void EnableLiteFP16(); + + /** + * @brief disable half precision, change to full precision(float32) + */ + void DisableLiteFP16(); + + /** + * @brief enable int8 precision while use paddle lite backend + */ + void EnableLiteInt8(); + + /** + * @brief disable int8 precision, change to full precision(float32) + */ + void DisableLiteInt8(); + + /** + * @brief Set power mode while using Paddle Lite as inference backend, mode(0: LITE_POWER_HIGH; 1: LITE_POWER_LOW; 2: LITE_POWER_FULL; 3: LITE_POWER_NO_BIND, 4: LITE_POWER_RAND_HIGH; 5: LITE_POWER_RAND_LOW, refer [paddle lite](https://paddle-lite.readthedocs.io/zh/latest/api_reference/cxx_api_doc.html#set-power-mode) for more details) + */ + void SetLitePowerMode(LitePowerMode mode); + + /** \brief Set shape range of input tensor for the model that contain dynamic input shape while using TensorRT backend + * + * \param[in] input_name The name of input for the model which is dynamic shape + * \param[in] min_shape The minimal shape for the input tensor + * \param[in] opt_shape The optimized shape for the input tensor, just set the most common shape, if set as default value, it will keep same with min_shape + * \param[in] max_shape The maximum shape for the input tensor, if set as default value, it will keep same with min_shape + */ + void SetTrtInputShape( + const std::string& input_name, const std::vector& min_shape, + const std::vector& opt_shape = std::vector(), + const std::vector& max_shape = std::vector()); + + /// Set max_workspace_size for TensorRT, default 1<<30 + void SetTrtMaxWorkspaceSize(size_t trt_max_workspace_size); + + /// Set max_batch_size for TensorRT, default 32 + void SetTrtMaxBatchSize(size_t max_batch_size); + + /** + * @brief Enable FP16 inference while using TensorRT backend. Notice: not all the GPU device support FP16, on those device doesn't support FP16, FastDeploy will fallback to FP32 automaticly + */ + void EnableTrtFP16(); + + /// Disable FP16 inference while using TensorRT backend + void DisableTrtFP16(); + + /** + * @brief Set cache file path while use TensorRT backend. Loadding a Paddle/ONNX model and initialize TensorRT will take a long time, by this interface it will save the tensorrt engine to `cache_file_path`, and load it directly while execute the code again + */ + void SetTrtCacheFile(const std::string& cache_file_path); + + /** + * @brief Enable pinned memory. Pinned memory can be utilized to speedup the data transfer between CPU and GPU. Currently it's only suppurted in TRT backend and Paddle Inference backend. + */ + void EnablePinnedMemory(); + + /** + * @brief Disable pinned memory + */ + void DisablePinnedMemory(); + + /** + * @brief Enable to collect shape in paddle trt backend + */ + void EnablePaddleTrtCollectShape(); + + /** + * @brief Disable to collect shape in paddle trt backend + */ + void DisablePaddleTrtCollectShape(); + + /* + * @brief Set number of streams by the OpenVINO backends + */ + void SetOpenVINOStreams(int num_streams); + + /** \Use Graphcore IPU to inference. + * + * \param[in] device_num the number of IPUs. + * \param[in] micro_batch_size the batch size in the graph, only work when graph has no batch shape info. + * \param[in] enable_pipelining enable pipelining. + * \param[in] batches_per_step the number of batches per run in pipelining. + */ + void UseIpu(int device_num = 1, int micro_batch_size = 1, + bool enable_pipelining = false, int batches_per_step = 1); + + /** \brief Set IPU config. + * + * \param[in] enable_fp16 enable fp16. + * \param[in] replica_num the number of graph replication. + * \param[in] available_memory_proportion the available memory proportion for matmul/conv. + * \param[in] enable_half_partial enable fp16 partial for matmul, only work with fp16. + */ + void SetIpuConfig(bool enable_fp16 = false, int replica_num = 1, + float available_memory_proportion = 1.0, + bool enable_half_partial = false); + + Backend backend = Backend::UNKNOWN; + // for cpu inference and preprocess + // default will let the backend choose their own default value + int cpu_thread_num = -1; + int device_id = 0; + + Device device = Device::CPU; + + void* external_stream_ = nullptr; + + bool enable_pinned_memory = false; + + // ======Only for ORT Backend======== + // -1 means use default value by ort + // 0: ORT_DISABLE_ALL 1: ORT_ENABLE_BASIC 2: ORT_ENABLE_EXTENDED 3: + // ORT_ENABLE_ALL + int ort_graph_opt_level = -1; + int ort_inter_op_num_threads = -1; + // 0: ORT_SEQUENTIAL 1: ORT_PARALLEL + int ort_execution_mode = -1; + + // ======Only for Paddle Backend===== + bool pd_enable_mkldnn = true; + bool pd_enable_log_info = false; + bool pd_enable_trt = false; + bool pd_collect_shape = false; + int pd_mkldnn_cache_size = 1; + std::vector pd_delete_pass_names; + + // ======Only for Paddle IPU Backend ======= + int ipu_device_num = 1; + int ipu_micro_batch_size = 1; + bool ipu_enable_pipelining = false; + int ipu_batches_per_step = 1; + bool ipu_enable_fp16 = false; + int ipu_replica_num = 1; + float ipu_available_memory_proportion = 1.0; + bool ipu_enable_half_partial = false; + + // ======Only for Paddle-Lite Backend===== + // 0: LITE_POWER_HIGH 1: LITE_POWER_LOW 2: LITE_POWER_FULL + // 3: LITE_POWER_NO_BIND 4: LITE_POWER_RAND_HIGH + // 5: LITE_POWER_RAND_LOW + LitePowerMode lite_power_mode = LitePowerMode::LITE_POWER_NO_BIND; + // enable int8 or not + bool lite_enable_int8 = false; + // enable fp16 or not + bool lite_enable_fp16 = false; + // optimized model dir for CxxConfig + std::string lite_optimized_model_dir = ""; + std::string lite_nnadapter_subgraph_partition_config_path = ""; + bool enable_timvx = false; + + // ======Only for Trt Backend======= + std::map> trt_max_shape; + std::map> trt_min_shape; + std::map> trt_opt_shape; + std::string trt_serialize_file = ""; + bool trt_enable_fp16 = false; + bool trt_enable_int8 = false; + size_t trt_max_batch_size = 32; + size_t trt_max_workspace_size = 1 << 30; + + // ======Only for Poros Backend======= + bool is_dynamic = false; + bool long_to_int = true; + bool use_nvidia_tf32 = false; + int unconst_ops_thres = -1; + std::string poros_file = ""; + + // ======Only for OpenVINO Backend======= + int ov_num_streams = 1; + + // ======Only for RKNPU2 Backend======= + fastdeploy::rknpu2::CpuName rknpu2_cpu_name_ + = fastdeploy::rknpu2::CpuName::RK3588; + fastdeploy::rknpu2::CoreMask rknpu2_core_mask_ + = fastdeploy::rknpu2::CoreMask::RKNN_NPU_CORE_AUTO; + + std::string model_file = ""; // Path of model file + std::string params_file = ""; // Path of parameters file, can be empty + // format of input model + ModelFormat model_format = ModelFormat::AUTOREC; +}; + +/*! @brief Runtime object used to inference the loaded model on different devices + */ +struct FASTDEPLOY_DECL Runtime { + public: + /// Intialize a Runtime object with RuntimeOption + bool Init(const RuntimeOption& _option); + + /** \brief Inference the model by the input data, and write to the output + * + * \param[in] input_tensors Notice the FDTensor::name should keep same with the model's input + * \param[in] output_tensors Inference results + * \return true if the inference successed, otherwise false + */ + bool Infer(std::vector& input_tensors, + std::vector* output_tensors); + + /** \brief Compile TorchScript Module, only for Poros backend + * + * \param[in] prewarm_tensors Prewarm datas for compile + * \param[in] _option Runtime option + * \return true if compile successed, otherwise false + */ + bool Compile(std::vector>& prewarm_tensors, + const RuntimeOption& _option); + + /** \brief Get number of inputs + */ + int NumInputs() { return backend_->NumInputs(); } + /** \brief Get number of outputs + */ + int NumOutputs() { return backend_->NumOutputs(); } + /** \brief Get input information by index + */ + TensorInfo GetInputInfo(int index); + /** \brief Get output information by index + */ + TensorInfo GetOutputInfo(int index); + /** \brief Get all the input information + */ + std::vector GetInputInfos(); + /** \brief Get all the output information + */ + std::vector GetOutputInfos(); + + /** \brief Clone new Runtime when multiple instances of the same model are created + * + * \param[in] stream CUDA Stream, defualt param is nullptr + * \return new Runtime* by this clone + */ + Runtime* Clone(void* stream = nullptr, + int device_id = -1); + + RuntimeOption option; + + private: + void CreateOrtBackend(); + void CreatePaddleBackend(); + void CreateTrtBackend(); + void CreateOpenVINOBackend(); + void CreateLiteBackend(); + void CreateRKNPU2Backend(); + std::unique_ptr backend_; +}; +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/text.h b/3rdparty/include/fastdeploy/text.h new file mode 100644 index 0000000000..85563a658f --- /dev/null +++ b/3rdparty/include/fastdeploy/text.h @@ -0,0 +1,19 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once + +#include "fastdeploy/core/config.h" +#ifdef ENABLE_TEXT +#include "fastdeploy/text/uie/model.h" +#endif diff --git a/3rdparty/include/fastdeploy/utils/axis_utils.h b/3rdparty/include/fastdeploy/utils/axis_utils.h new file mode 100644 index 0000000000..5d82a49fd2 --- /dev/null +++ b/3rdparty/include/fastdeploy/utils/axis_utils.h @@ -0,0 +1,52 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +namespace fastdeploy { + +static inline int CanonicalAxis(const int axis, const int rank) { + if (axis < 0) { + return axis + rank; + } + return axis; +} + +static inline int SizeToAxis(const int axis, const std::vector& dims) { + int size = 1; + for (int i = 0; i < axis; i++) { + size *= dims[i]; + } + return size; +} + +static inline int SizeFromAxis(const int axis, + const std::vector& dims) { + int size = 1; + for (int i = axis; i < dims.size(); i++) { + size *= dims[i]; + } + return size; +} + +static inline int SizeOutAxis(const int axis, + const std::vector& dims) { + int size = 1; + for (int i = axis + 1; i < dims.size(); i++) { + size *= dims[i]; + } + return size; +} + +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/utils/path.h b/3rdparty/include/fastdeploy/utils/path.h new file mode 100644 index 0000000000..5c9ed70877 --- /dev/null +++ b/3rdparty/include/fastdeploy/utils/path.h @@ -0,0 +1,74 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#ifdef _MSC_VER +#define PATH_SEP "\\" +#else +#define PATH_SEP "/" +#endif + +namespace fastdeploy { + +inline std::string PathJoin(const std::vector& paths, + const std::string& sep = PATH_SEP) { + if (paths.size() == 1) { + return paths[0]; + } + std::string filepath = ""; + for (const auto& path : paths) { + if (filepath == "") { + filepath += path; + continue; + } + if (path[0] == sep[0] || filepath.back() == sep[0]) { + filepath += path; + } else { + filepath += sep + path; + } + } + return filepath; +} + +inline std::string PathJoin(const std::string& folder, + const std::string& filename, + const std::string& sep = PATH_SEP) { + return PathJoin(std::vector{folder, filename}, sep); +} + +inline std::string GetDirFromPath(const std::string& path) { + auto pos = path.find_last_of(PATH_SEP); + if (pos == std::string::npos) { + return ""; + } + // The root path in UNIX systems + if (pos == 0) { + return "/"; + } + return path.substr(0, pos); +} + +inline bool CheckFileExists(const std::string& path) { + std::fstream fin(path, std::ios::in); + if (!fin) { + return false; + } + return true; +} + +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/utils/perf.h b/3rdparty/include/fastdeploy/utils/perf.h new file mode 100644 index 0000000000..783459064a --- /dev/null +++ b/3rdparty/include/fastdeploy/utils/perf.h @@ -0,0 +1,49 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "fastdeploy/utils/utils.h" +#include // NOLINT + +namespace fastdeploy { + +class FASTDEPLOY_DECL TimeCounter { + public: + void Start() { begin_ = std::chrono::system_clock::now(); } + + void End() { end_ = std::chrono::system_clock::now(); } + + double Duration() { + auto duration = + std::chrono::duration_cast(end_ - begin_); + return static_cast(duration.count()) * + std::chrono::microseconds::period::num / + std::chrono::microseconds::period::den; + } + + void PrintInfo(const std::string& prefix = "TimeCounter: ", + bool print_out = true) { + if (!print_out) { + return; + } + FDLogger() << prefix << " duration = " << Duration() << "s." << std::endl; + } + + private: + std::chrono::time_point begin_; + std::chrono::time_point end_; +}; + +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/utils/unique_ptr.h b/3rdparty/include/fastdeploy/utils/unique_ptr.h new file mode 100644 index 0000000000..2f24ef70c6 --- /dev/null +++ b/3rdparty/include/fastdeploy/utils/unique_ptr.h @@ -0,0 +1,58 @@ +/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include + +namespace fastdeploy { +namespace utils { +// Trait to select overloads and return types for MakeUnique. +template +struct MakeUniqueResult { + using scalar = std::unique_ptr; +}; +template +struct MakeUniqueResult { + using array = std::unique_ptr; +}; +template +struct MakeUniqueResult { + using invalid = void; +}; + +// MakeUnique(...) is an early implementation of C++14 std::make_unique. +// It is designed to be 100% compatible with std::make_unique so that the +// eventual switchover will be a simple renaming operation. +template +typename MakeUniqueResult::scalar make_unique(Args &&... args) { // NOLINT + return std::unique_ptr( + new T(std::forward(args)...)); // NOLINT(build/c++11) +} + +// Overload for array of unknown bound. +// The allocation of arrays needs to use the array form of new, +// and cannot take element constructor arguments. +template +typename MakeUniqueResult::array make_unique(size_t n) { + return std::unique_ptr(new typename std::remove_extent::type[n]()); +} + +// Reject arrays of known bound. +template +typename MakeUniqueResult::invalid make_unique(Args &&... /* args */) = + delete; // NOLINT + +} // namespace utils +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/utils/utils.h b/3rdparty/include/fastdeploy/utils/utils.h new file mode 100644 index 0000000000..2c72697634 --- /dev/null +++ b/3rdparty/include/fastdeploy/utils/utils.h @@ -0,0 +1,189 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include + +#include +#include +#include +#include +#include + +#if defined(_WIN32) +#ifdef FASTDEPLOY_LIB +#define FASTDEPLOY_DECL __declspec(dllexport) +#else +#define FASTDEPLOY_DECL __declspec(dllimport) +#endif // FASTDEPLOY_LIB +#else +#define FASTDEPLOY_DECL __attribute__((visibility("default"))) +#endif // _WIN32 + +namespace fastdeploy { + +class FASTDEPLOY_DECL FDLogger { + public: + FDLogger() { + line_ = ""; + prefix_ = "[FastDeploy]"; + verbose_ = true; + } + explicit FDLogger(bool verbose, const std::string& prefix = "[FastDeploy]"); + + template + FDLogger& operator<<(const T& val) { + if (!verbose_) { + return *this; + } + std::stringstream ss; + ss << val; + line_ += ss.str(); + return *this; + } + FDLogger& operator<<(std::ostream& (*os)(std::ostream&)); + ~FDLogger() { + if (!verbose_ && line_ != "") { + std::cout << line_ << std::endl; + } + } + + private: + std::string line_; + std::string prefix_; + bool verbose_ = true; +}; + +FASTDEPLOY_DECL bool ReadBinaryFromFile(const std::string& file, + std::string* contents); + +#ifndef __REL_FILE__ +#define __REL_FILE__ __FILE__ +#endif + +#define FDERROR \ + FDLogger(true, "[ERROR]") << __REL_FILE__ << "(" << __LINE__ \ + << ")::" << __FUNCTION__ << "\t" + +#define FDWARNING \ + FDLogger(true, "[WARNING]") << __REL_FILE__ << "(" << __LINE__ \ + << ")::" << __FUNCTION__ << "\t" + +#define FDINFO \ + FDLogger(true, "[INFO]") << __REL_FILE__ << "(" << __LINE__ \ + << ")::" << __FUNCTION__ << "\t" + +#define FDASSERT(condition, format, ...) \ + if (!(condition)) { \ + int n = std::snprintf(nullptr, 0, format, ##__VA_ARGS__); \ + std::vector buffer(n + 1); \ + std::snprintf(buffer.data(), n + 1, format, ##__VA_ARGS__); \ + FDERROR << buffer.data() << std::endl; \ + std::abort(); \ + } + +///////// Basic Marco /////////// + +#define FD_PRIVATE_CASE_TYPE_USING_HINT(NAME, enum_type, type, HINT, ...) \ + case enum_type: { \ + using HINT = type; \ + __VA_ARGS__(); \ + break; \ + } + +#define FD_PRIVATE_CASE_TYPE(NAME, enum_type, type, ...) \ + FD_PRIVATE_CASE_TYPE_USING_HINT(NAME, enum_type, type, data_t, __VA_ARGS__) + +// Visit different data type to match the corresponding function of FDTensor +#define FD_VISIT_ALL_TYPES(TYPE, NAME, ...) \ + [&] { \ + const auto& __dtype__ = TYPE; \ + switch (__dtype__) { \ + FD_PRIVATE_CASE_TYPE(NAME, ::fastdeploy::FDDataType::BOOL, bool, \ + __VA_ARGS__) \ + FD_PRIVATE_CASE_TYPE(NAME, ::fastdeploy::FDDataType::INT32, int32_t, \ + __VA_ARGS__) \ + FD_PRIVATE_CASE_TYPE(NAME, ::fastdeploy::FDDataType::INT64, int64_t, \ + __VA_ARGS__) \ + FD_PRIVATE_CASE_TYPE(NAME, ::fastdeploy::FDDataType::FP32, float, \ + __VA_ARGS__) \ + FD_PRIVATE_CASE_TYPE(NAME, ::fastdeploy::FDDataType::FP64, double, \ + __VA_ARGS__) \ + default: \ + FDASSERT( \ + false, \ + "Invalid enum data type. Expect to accept data type BOOL, INT32, " \ + "INT64, FP32, FP64, but receive type %s.", \ + Str(__dtype__).c_str()); \ + } \ + }() + +#define FD_VISIT_INT_FLOAT_TYPES(TYPE, NAME, ...) \ + [&] { \ + const auto& __dtype__ = TYPE; \ + switch (__dtype__) { \ + FD_PRIVATE_CASE_TYPE(NAME, ::fastdeploy::FDDataType::INT32, int32_t, \ + __VA_ARGS__) \ + FD_PRIVATE_CASE_TYPE(NAME, ::fastdeploy::FDDataType::INT64, int64_t, \ + __VA_ARGS__) \ + FD_PRIVATE_CASE_TYPE(NAME, ::fastdeploy::FDDataType::FP32, float, \ + __VA_ARGS__) \ + FD_PRIVATE_CASE_TYPE(NAME, ::fastdeploy::FDDataType::FP64, double, \ + __VA_ARGS__) \ + default: \ + FDASSERT(false, \ + "Invalid enum data type. Expect to accept data type INT32, " \ + "INT64, FP32, FP64, but receive type %s.", \ + Str(__dtype__).c_str()); \ + } \ + }() + +#define FD_VISIT_FLOAT_TYPES(TYPE, NAME, ...) \ + [&] { \ + const auto& __dtype__ = TYPE; \ + switch (__dtype__) { \ + FD_PRIVATE_CASE_TYPE(NAME, ::fastdeploy::FDDataType::FP32, float, \ + __VA_ARGS__) \ + FD_PRIVATE_CASE_TYPE(NAME, ::fastdeploy::FDDataType::FP64, double, \ + __VA_ARGS__) \ + default: \ + FDASSERT(false, \ + "Invalid enum data type. Expect to accept data type FP32, " \ + "FP64, but receive type %s.", \ + Str(__dtype__).c_str()); \ + } \ + }() + +#define FD_VISIT_INT_TYPES(TYPE, NAME, ...) \ + [&] { \ + const auto& __dtype__ = TYPE; \ + switch (__dtype__) { \ + FD_PRIVATE_CASE_TYPE(NAME, ::fastdeploy::FDDataType::INT32, int32_t, \ + __VA_ARGS__) \ + FD_PRIVATE_CASE_TYPE(NAME, ::fastdeploy::FDDataType::INT64, int64_t, \ + __VA_ARGS__) \ + default: \ + FDASSERT(false, \ + "Invalid enum data type. Expect to accept data type INT32, " \ + "INT64, but receive type %s.", \ + Str(__dtype__).c_str()); \ + } \ + }() + +FASTDEPLOY_DECL std::vector GetStride( + const std::vector& dims); + +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/vision.h b/3rdparty/include/fastdeploy/vision.h new file mode 100644 index 0000000000..2f8c706613 --- /dev/null +++ b/3rdparty/include/fastdeploy/vision.h @@ -0,0 +1,57 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once + +#include "fastdeploy/core/config.h" +#ifdef ENABLE_VISION +#include "fastdeploy/vision/classification/contrib/yolov5cls.h" +#include "fastdeploy/vision/classification/ppcls/model.h" +#include "fastdeploy/vision/classification/contrib/resnet.h" +#include "fastdeploy/vision/detection/contrib/nanodet_plus.h" +#include "fastdeploy/vision/detection/contrib/scaledyolov4.h" +#include "fastdeploy/vision/detection/contrib/yolor.h" +#include "fastdeploy/vision/detection/contrib/yolov5/yolov5.h" +#include "fastdeploy/vision/detection/contrib/yolov5lite.h" +#include "fastdeploy/vision/detection/contrib/yolov6.h" +#include "fastdeploy/vision/detection/contrib/yolov7.h" +#include "fastdeploy/vision/detection/contrib/yolov7end2end_ort.h" +#include "fastdeploy/vision/detection/contrib/yolov7end2end_trt.h" +#include "fastdeploy/vision/detection/contrib/yolox.h" +#include "fastdeploy/vision/detection/ppdet/model.h" +#include "fastdeploy/vision/facedet/contrib/retinaface.h" +#include "fastdeploy/vision/facedet/contrib/scrfd.h" +#include "fastdeploy/vision/facedet/contrib/ultraface.h" +#include "fastdeploy/vision/facedet/contrib/yolov5face.h" +#include "fastdeploy/vision/facealign/contrib/pfld.h" +#include "fastdeploy/vision/faceid/contrib/adaface.h" +#include "fastdeploy/vision/faceid/contrib/arcface.h" +#include "fastdeploy/vision/faceid/contrib/cosface.h" +#include "fastdeploy/vision/faceid/contrib/insightface_rec.h" +#include "fastdeploy/vision/faceid/contrib/partial_fc.h" +#include "fastdeploy/vision/faceid/contrib/vpl.h" +#include "fastdeploy/vision/keypointdet/pptinypose/pptinypose.h" +#include "fastdeploy/vision/matting/contrib/modnet.h" +#include "fastdeploy/vision/matting/contrib/rvm.h" +#include "fastdeploy/vision/matting/ppmatting/ppmatting.h" +#include "fastdeploy/vision/ocr/ppocr/classifier.h" +#include "fastdeploy/vision/ocr/ppocr/dbdetector.h" +#include "fastdeploy/vision/ocr/ppocr/ppocr_v2.h" +#include "fastdeploy/vision/ocr/ppocr/ppocr_v3.h" +#include "fastdeploy/vision/ocr/ppocr/recognizer.h" +#include "fastdeploy/vision/segmentation/ppseg/model.h" +#include "fastdeploy/vision/tracking/pptracking/model.h" +#include "fastdeploy/vision/headpose/contrib/fsanet.h" +#endif + +#include "fastdeploy/vision/visualize/visualize.h" diff --git a/3rdparty/include/fastdeploy/vision/common/processors/base.h b/3rdparty/include/fastdeploy/vision/common/processors/base.h new file mode 100644 index 0000000000..c833ef8a37 --- /dev/null +++ b/3rdparty/include/fastdeploy/vision/common/processors/base.h @@ -0,0 +1,76 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "fastdeploy/utils/utils.h" +#include "fastdeploy/vision/common/processors/mat.h" +#include "opencv2/highgui/highgui.hpp" +#include "opencv2/imgproc/imgproc.hpp" +#include + +namespace fastdeploy { +namespace vision { + +/*! @brief Enable using FlyCV to process image while deploy vision models. + * Currently, FlyCV in only available on ARM(Linux aarch64/Android), so will + * fallback to using OpenCV in other platform + */ +FASTDEPLOY_DECL void EnableFlyCV(); + +/// Disable using FlyCV to process image while deploy vision models. +FASTDEPLOY_DECL void DisableFlyCV(); + +/*! @brief Set the cpu num threads of ProcLib. The cpu num threads + * of FlyCV and OpenCV is 2 by default. + */ +FASTDEPLOY_DECL void SetProcLibCpuNumThreads(int threads); + +class FASTDEPLOY_DECL Processor { + public: + // default_lib has the highest priority + // all the function in `processor` will force to use + // default_lib if this flag is set. + // DEFAULT means this flag is not set + // static ProcLib default_lib; + + virtual std::string Name() = 0; + + virtual bool ImplByOpenCV(Mat* mat) { + FDERROR << Name() << " Not Implement Yet." << std::endl; + return false; + } + + virtual bool ImplByFlyCV(Mat* mat) { + return ImplByOpenCV(mat); + } + + virtual bool ImplByCuda(Mat* mat) { + return ImplByOpenCV(mat); + } + + virtual bool operator()(Mat* mat, ProcLib lib = ProcLib::DEFAULT); + + protected: + FDTensor* UpdateAndGetReusedBuffer( + const std::vector& new_shape, const int& opencv_dtype, + const std::string& buffer_name, const Device& new_device = Device::CPU, + const bool& use_pinned_memory = false); + + private: + std::unordered_map reused_buffers_; +}; + +} // namespace vision +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/vision/common/processors/cast.h b/3rdparty/include/fastdeploy/vision/common/processors/cast.h new file mode 100644 index 0000000000..891ae334c8 --- /dev/null +++ b/3rdparty/include/fastdeploy/vision/common/processors/cast.h @@ -0,0 +1,39 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "fastdeploy/vision/common/processors/base.h" + +namespace fastdeploy { +namespace vision { + +class FASTDEPLOY_DECL Cast : public Processor { + public: + explicit Cast(const std::string& dtype = "float") : dtype_(dtype) {} + bool ImplByOpenCV(Mat* mat); +#ifdef ENABLE_FLYCV + bool ImplByFlyCV(Mat* mat); +#endif + std::string Name() { return "Cast"; } + static bool Run(Mat* mat, const std::string& dtype, + ProcLib lib = ProcLib::DEFAULT); + + std::string GetDtype() const { return dtype_; } + + private: + std::string dtype_; +}; +} // namespace vision +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/vision/common/processors/center_crop.h b/3rdparty/include/fastdeploy/vision/common/processors/center_crop.h new file mode 100644 index 0000000000..05f594249c --- /dev/null +++ b/3rdparty/include/fastdeploy/vision/common/processors/center_crop.h @@ -0,0 +1,40 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "fastdeploy/vision/common/processors/base.h" + +namespace fastdeploy { +namespace vision { + +class FASTDEPLOY_DECL CenterCrop : public Processor { + public: + CenterCrop(int width, int height) : height_(height), width_(width) {} + bool ImplByOpenCV(Mat* mat); +#ifdef ENABLE_FLYCV + bool ImplByFlyCV(Mat* mat); +#endif + std::string Name() { return "CenterCrop"; } + + static bool Run(Mat* mat, const int& width, const int& height, + ProcLib lib = ProcLib::DEFAULT); + + private: + int height_; + int width_; +}; + +} // namespace vision +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/vision/common/processors/color_space_convert.h b/3rdparty/include/fastdeploy/vision/common/processors/color_space_convert.h new file mode 100644 index 0000000000..e090bc62d5 --- /dev/null +++ b/3rdparty/include/fastdeploy/vision/common/processors/color_space_convert.h @@ -0,0 +1,44 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "fastdeploy/vision/common/processors/base.h" + +namespace fastdeploy { +namespace vision { + +class FASTDEPLOY_DECL BGR2RGB : public Processor { + public: + bool ImplByOpenCV(Mat* mat); +#ifdef ENABLE_FLYCV + bool ImplByFlyCV(Mat* mat); +#endif + virtual std::string Name() { return "BGR2RGB"; } + + static bool Run(Mat* mat, ProcLib lib = ProcLib::DEFAULT); +}; + +class FASTDEPLOY_DECL RGB2BGR : public Processor { + public: + bool ImplByOpenCV(Mat* mat); +#ifdef ENABLE_FLYCV + bool ImplByFlyCV(Mat* mat); +#endif + std::string Name() { return "RGB2BGR"; } + + static bool Run(Mat* mat, ProcLib lib = ProcLib::DEFAULT); +}; +} // namespace vision +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/vision/common/processors/convert.h b/3rdparty/include/fastdeploy/vision/common/processors/convert.h new file mode 100644 index 0000000000..6f26c45282 --- /dev/null +++ b/3rdparty/include/fastdeploy/vision/common/processors/convert.h @@ -0,0 +1,42 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "fastdeploy/vision/common/processors/base.h" + +namespace fastdeploy { +namespace vision { +class FASTDEPLOY_DECL Convert : public Processor { + public: + Convert(const std::vector& alpha, const std::vector& beta); + + bool ImplByOpenCV(Mat* mat); +#ifdef ENABLE_FLYCV + bool ImplByFlyCV(Mat* mat); +#endif + std::string Name() { return "Convert"; } + + // Compute `result = mat * alpha + beta` directly by channel. + // The default behavior is the same as OpenCV's convertTo method. + static bool Run(Mat* mat, const std::vector& alpha, + const std::vector& beta, + ProcLib lib = ProcLib::DEFAULT); + + private: + std::vector alpha_; + std::vector beta_; +}; +} // namespace vision +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/vision/common/processors/convert_and_permute.h b/3rdparty/include/fastdeploy/vision/common/processors/convert_and_permute.h new file mode 100644 index 0000000000..a36dfd5638 --- /dev/null +++ b/3rdparty/include/fastdeploy/vision/common/processors/convert_and_permute.h @@ -0,0 +1,66 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "fastdeploy/vision/common/processors/base.h" + +namespace fastdeploy { +namespace vision { +class FASTDEPLOY_DECL ConvertAndPermute : public Processor { + public: + ConvertAndPermute(const std::vector& alpha = std::vector(), + const std::vector& beta = std::vector(), + bool swap_rb = false); + bool ImplByOpenCV(FDMat* mat); +#ifdef ENABLE_FLYCV + bool ImplByFlyCV(FDMat* mat); +#endif + std::string Name() { return "ConvertAndPermute"; } + + static bool Run(FDMat* mat, const std::vector& alpha, + const std::vector& beta, bool swap_rb = false, + ProcLib lib = ProcLib::DEFAULT); + + std::vector GetAlpha() const { return alpha_; } + + void SetAlpha(const std::vector& alpha) { + alpha_.clear(); + std::vector().swap(alpha_); + alpha_.assign(alpha.begin(), alpha.end()); + } + + std::vector GetBeta() const { return beta_; } + + void SetBeta(const std::vector& beta) { + beta_.clear(); + std::vector().swap(beta_); + beta_.assign(beta.begin(), beta.end()); + } + + bool GetSwapRB() { + return swap_rb_; + } + + void SetSwapRB(bool swap_rb) { + swap_rb_ = swap_rb; + } + + private: + std::vector alpha_; + std::vector beta_; + bool swap_rb_; +}; +} // namespace vision +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/vision/common/processors/crop.h b/3rdparty/include/fastdeploy/vision/common/processors/crop.h new file mode 100644 index 0000000000..369edb610f --- /dev/null +++ b/3rdparty/include/fastdeploy/vision/common/processors/crop.h @@ -0,0 +1,49 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "fastdeploy/vision/common/processors/base.h" + +namespace fastdeploy { +namespace vision { + +class FASTDEPLOY_DECL Crop : public Processor { + public: + Crop(int offset_w, int offset_h, int width, int height) { + offset_w_ = offset_w; + offset_h_ = offset_h; + width_ = width; + height_ = height; + } + + bool ImplByOpenCV(Mat* mat); + +#ifdef ENABLE_FLYCV + bool ImplByFlyCV(Mat* mat); +#endif + std::string Name() { return "Crop"; } + + static bool Run(Mat* mat, int offset_w, int offset_h, int width, int height, + ProcLib lib = ProcLib::DEFAULT); + + private: + int offset_w_; + int offset_h_; + int height_; + int width_; +}; + +} // namespace vision +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/vision/common/processors/hwc2chw.h b/3rdparty/include/fastdeploy/vision/common/processors/hwc2chw.h new file mode 100644 index 0000000000..535a1887b5 --- /dev/null +++ b/3rdparty/include/fastdeploy/vision/common/processors/hwc2chw.h @@ -0,0 +1,33 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "fastdeploy/vision/common/processors/base.h" + +namespace fastdeploy { +namespace vision { + +class FASTDEPLOY_DECL HWC2CHW : public Processor { + public: + bool ImplByOpenCV(Mat* mat); +#ifdef ENABLE_FLYCV + bool ImplByFlyCV(Mat* mat); +#endif + std::string Name() { return "HWC2CHW"; } + + static bool Run(Mat* mat, ProcLib lib = ProcLib::DEFAULT); +}; +} // namespace vision +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/vision/common/processors/limit_by_stride.h b/3rdparty/include/fastdeploy/vision/common/processors/limit_by_stride.h new file mode 100644 index 0000000000..a3751df2e9 --- /dev/null +++ b/3rdparty/include/fastdeploy/vision/common/processors/limit_by_stride.h @@ -0,0 +1,44 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "fastdeploy/vision/common/processors/base.h" + +namespace fastdeploy { +namespace vision { + +class FASTDEPLOY_DECL LimitByStride : public Processor { + public: + explicit LimitByStride(int stride = 32, int interp = 1) { + stride_ = stride; + interp_ = interp; + } + + // Resize Mat* mat to make the size divisible by stride_. + bool ImplByOpenCV(Mat* mat); +#ifdef ENABLE_FLYCV + bool ImplByFlyCV(Mat* mat); +#endif + std::string Name() { return "LimitByStride"; } + + static bool Run(Mat* mat, int stride = 32, int interp = 1, + ProcLib lib = ProcLib::DEFAULT); + + private: + int interp_; + int stride_; +}; +} // namespace vision +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/vision/common/processors/limit_short.h b/3rdparty/include/fastdeploy/vision/common/processors/limit_short.h new file mode 100644 index 0000000000..268fa088fe --- /dev/null +++ b/3rdparty/include/fastdeploy/vision/common/processors/limit_short.h @@ -0,0 +1,51 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "fastdeploy/vision/common/processors/base.h" + +namespace fastdeploy { +namespace vision { + +class LimitShort : public Processor { + public: + explicit LimitShort(int max_short = -1, int min_short = -1, int interp = 1) { + max_short_ = max_short; + min_short_ = min_short; + interp_ = interp; + } + + // Limit the short edge of image. + // If the short edge is larger than max_short_, resize the short edge + // to max_short_, while scale the long edge proportionally. + // If the short edge is smaller than min_short_, resize the short edge + // to min_short_, while scale the long edge proportionally. + bool ImplByOpenCV(Mat* mat); +#ifdef ENABLE_FLYCV + bool ImplByFlyCV(Mat* mat); +#endif + std::string Name() { return "LimitShort"; } + + static bool Run(Mat* mat, int max_short = -1, int min_short = -1, + int interp = 1, ProcLib lib = ProcLib::DEFAULT); + int GetMaxShort() const { return max_short_; } + + private: + int max_short_; + int min_short_; + int interp_; +}; +} // namespace vision +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/vision/common/processors/mat.h b/3rdparty/include/fastdeploy/vision/common/processors/mat.h new file mode 100644 index 0000000000..8c19080301 --- /dev/null +++ b/3rdparty/include/fastdeploy/vision/common/processors/mat.h @@ -0,0 +1,162 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once +#include "fastdeploy/core/fd_tensor.h" +#include "fastdeploy/vision/common/processors/utils.h" +#include "fastdeploy/vision/common/processors/proc_lib.h" +#include "opencv2/core/core.hpp" + +namespace fastdeploy { +namespace vision { + +enum Layout { HWC, CHW }; + + +struct FASTDEPLOY_DECL Mat { + explicit Mat(const cv::Mat& mat) { + cpu_mat = mat; + layout = Layout::HWC; + height = cpu_mat.rows; + width = cpu_mat.cols; + channels = cpu_mat.channels(); + mat_type = ProcLib::OPENCV; + } + +#ifdef ENABLE_FLYCV + explicit Mat(const fcv::Mat& mat) { + fcv_mat = mat; + layout = Layout::HWC; + height = fcv_mat.height(); + width = fcv_mat.width(); + channels = fcv_mat.channels(); + mat_type = ProcLib::FLYCV; + } +#endif + + Mat(const Mat& mat) = default; + Mat& operator=(const Mat& mat) = default; + + // Careful if you use this interface + // this only used if you don't want to write + // the original data, and write to a new cv::Mat + // then replace the old cv::Mat of this structure + void SetMat(const cv::Mat& mat) { + cpu_mat = mat; + mat_type = ProcLib::OPENCV; + } + + cv::Mat* GetOpenCVMat() { + if (mat_type == ProcLib::OPENCV) { + return &cpu_mat; + } else if (mat_type == ProcLib::FLYCV) { +#ifdef ENABLE_FLYCV + // Just a reference to fcv_mat, zero copy. After you + // call this method, cpu_mat and fcv_mat will point + // to the same memory buffer. + cpu_mat = ConvertFlyCVMatToOpenCV(fcv_mat); + mat_type = ProcLib::OPENCV; + return &cpu_mat; +#else + FDASSERT(false, "FastDeploy didn't compiled with FlyCV!"); +#endif + } else { + FDASSERT(false, "The mat_type of custom Mat can not be ProcLib::DEFAULT"); + } + } + +#ifdef ENABLE_FLYCV + void SetMat(const fcv::Mat& mat) { + fcv_mat = mat; + mat_type = ProcLib::FLYCV; + } + + fcv::Mat* GetFlyCVMat() { + if (mat_type == ProcLib::FLYCV) { + return &fcv_mat; + } else if (mat_type == ProcLib::OPENCV) { + // Just a reference to cpu_mat, zero copy. After you + // call this method, fcv_mat and cpu_mat will point + // to the same memory buffer. + fcv_mat = ConvertOpenCVMatToFlyCV(cpu_mat); + mat_type = ProcLib::FLYCV; + return &fcv_mat; + } else { + FDASSERT(false, "The mat_type of custom Mat can not be ProcLib::DEFAULT"); + } + } +#endif + + void* Data(); + + private: + int channels; + int height; + int width; + cv::Mat cpu_mat; +#ifdef ENABLE_FLYCV + fcv::Mat fcv_mat; +#endif + + public: + FDDataType Type(); + int Channels() const { return channels; } + int Width() const { return width; } + int Height() const { return height; } + void SetChannels(int s) { channels = s; } + void SetWidth(int w) { width = w; } + void SetHeight(int h) { height = h; } + + // Transfer the vision::Mat to FDTensor + void ShareWithTensor(FDTensor* tensor); + // Only support copy to cpu tensor now + bool CopyToTensor(FDTensor* tensor); + + // Debug functions + // TODO(jiangjiajun) Develop a right process pipeline with c++ + // is not a easy things, Will add more debug function here to + // help debug processed image. This function will print shape + // and mean of each channels of the Mat + void PrintInfo(const std::string& flag); + + ProcLib mat_type = ProcLib::OPENCV; + Layout layout = Layout::HWC; + Device device = Device::CPU; + + // Create FD Mat from FD Tensor. This method only create a + // new FD Mat with zero copy and it's data pointer is reference + // to the original memory buffer of input FD Tensor. Carefully, + // any operation on this Mat may change memory that points to + // FDTensor. We assume that the memory Mat points to is mutable. + // This method will create a FD Mat according to current global + // default ProcLib (OPENCV,FLYCV,...). + static Mat Create(const FDTensor& tensor); + static Mat Create(const FDTensor& tensor, ProcLib lib); + static Mat Create(int height, int width, int channels, + FDDataType type, void* data); + static Mat Create(int height, int width, int channels, + FDDataType type, void* data, ProcLib lib); +}; + +typedef Mat FDMat; +/* + * @brief Wrap a cv::Mat to FDMat, there's no memory copy, memory buffer is managed by user + */ +FASTDEPLOY_DECL FDMat WrapMat(const cv::Mat& image); +/* + * Warp a vector to vector, there's no memory copy, memory buffer is managed by user + */ +FASTDEPLOY_DECL std::vector WrapMat(const std::vector& images); + +} // namespace vision +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/vision/common/processors/normalize.h b/3rdparty/include/fastdeploy/vision/common/processors/normalize.h new file mode 100644 index 0000000000..c489207df3 --- /dev/null +++ b/3rdparty/include/fastdeploy/vision/common/processors/normalize.h @@ -0,0 +1,67 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "fastdeploy/vision/common/processors/base.h" + +namespace fastdeploy { +namespace vision { +class FASTDEPLOY_DECL Normalize : public Processor { + public: + Normalize(const std::vector& mean, const std::vector& std, + bool is_scale = true, + const std::vector& min = std::vector(), + const std::vector& max = std::vector(), + bool swap_rb = false); + bool ImplByOpenCV(Mat* mat); +#ifdef ENABLE_FLYCV + bool ImplByFlyCV(Mat* mat); +#endif + std::string Name() { return "Normalize"; } + + // While use normalize, it is more recommend not use this function + // this function will need to compute result = ((mat / 255) - mean) / std + // if we use the following method + // ``` + // auto norm = Normalize(...) + // norm(mat) + // ``` + // There will be some precomputation in contruct function + // and the `norm(mat)` only need to compute result = mat * alpha + beta + // which will reduce lots of time + static bool Run(Mat* mat, const std::vector& mean, + const std::vector& std, bool is_scale = true, + const std::vector& min = std::vector(), + const std::vector& max = std::vector(), + ProcLib lib = ProcLib::DEFAULT, bool swap_rb = false); + + std::vector GetAlpha() const { return alpha_; } + std::vector GetBeta() const { return beta_; } + + bool GetSwapRB() { + return swap_rb_; + } + + void SetSwapRB(bool swap_rb) { + swap_rb_ = swap_rb; + } + + private: + std::vector alpha_; + std::vector beta_; + bool swap_rb_; +}; +} // namespace vision +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/vision/common/processors/normalize_and_permute.h b/3rdparty/include/fastdeploy/vision/common/processors/normalize_and_permute.h new file mode 100644 index 0000000000..ea7649d928 --- /dev/null +++ b/3rdparty/include/fastdeploy/vision/common/processors/normalize_and_permute.h @@ -0,0 +1,79 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "fastdeploy/vision/common/processors/base.h" + +namespace fastdeploy { +namespace vision { +class FASTDEPLOY_DECL NormalizeAndPermute : public Processor { + public: + NormalizeAndPermute(const std::vector& mean, + const std::vector& std, bool is_scale = true, + const std::vector& min = std::vector(), + const std::vector& max = std::vector(), + bool swap_rb = false); + bool ImplByOpenCV(Mat* mat); +#ifdef ENABLE_FLYCV + bool ImplByFlyCV(Mat* mat); +#endif +#ifdef WITH_GPU + bool ImplByCuda(Mat* mat); +#endif + std::string Name() { return "NormalizeAndPermute"; } + + // While use normalize, it is more recommend not use this function + // this function will need to compute result = ((mat / 255) - mean) / std + // if we use the following method + // ``` + // auto norm = Normalize(...) + // norm(mat) + // ``` + // There will be some precomputation in contruct function + // and the `norm(mat)` only need to compute result = mat * alpha + beta + // which will reduce lots of time + static bool Run(Mat* mat, const std::vector& mean, + const std::vector& std, bool is_scale = true, + const std::vector& min = std::vector(), + const std::vector& max = std::vector(), + ProcLib lib = ProcLib::DEFAULT, bool swap_rb = false); + + void SetAlpha(const std::vector& alpha) { + alpha_.clear(); + std::vector().swap(alpha_); + alpha_.assign(alpha.begin(), alpha.end()); + } + + void SetBeta(const std::vector& beta) { + beta_.clear(); + std::vector().swap(beta_); + beta_.assign(beta.begin(), beta.end()); + } + + bool GetSwapRB() { + return swap_rb_; + } + + void SetSwapRB(bool swap_rb) { + swap_rb_ = swap_rb; + } + + private: + std::vector alpha_; + std::vector beta_; + bool swap_rb_; +}; +} // namespace vision +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/vision/common/processors/pad.h b/3rdparty/include/fastdeploy/vision/common/processors/pad.h new file mode 100644 index 0000000000..661632e77c --- /dev/null +++ b/3rdparty/include/fastdeploy/vision/common/processors/pad.h @@ -0,0 +1,50 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "fastdeploy/vision/common/processors/base.h" + +namespace fastdeploy { +namespace vision { + +class FASTDEPLOY_DECL Pad : public Processor { + public: + Pad(int top, int bottom, int left, int right, + const std::vector& value) { + top_ = top; + bottom_ = bottom; + left_ = left; + right_ = right; + value_ = value; + } + bool ImplByOpenCV(Mat* mat); +#ifdef ENABLE_FLYCV + bool ImplByFlyCV(Mat* mat); +#endif + std::string Name() { return "Pad"; } + + static bool Run(Mat* mat, const int& top, const int& bottom, const int& left, + const int& right, const std::vector& value, + ProcLib lib = ProcLib::DEFAULT); + + private: + int top_; + int bottom_; + int left_; + int right_; + std::vector value_; +}; +} // namespace vision +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/vision/common/processors/pad_to_size.h b/3rdparty/include/fastdeploy/vision/common/processors/pad_to_size.h new file mode 100644 index 0000000000..c73cee3c24 --- /dev/null +++ b/3rdparty/include/fastdeploy/vision/common/processors/pad_to_size.h @@ -0,0 +1,46 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "fastdeploy/vision/common/processors/base.h" + +namespace fastdeploy { +namespace vision { + +class FASTDEPLOY_DECL PadToSize : public Processor { + public: + // only support pad with right-bottom padding mode + PadToSize(int width, int height, const std::vector& value) { + width_ = width; + height_ = height; + value_ = value; + } + bool ImplByOpenCV(Mat* mat); +#ifdef ENABLE_FLYCV + bool ImplByFlyCV(Mat* mat); +#endif + std::string Name() { return "PadToSize"; } + + static bool Run(Mat* mat, int width, int height, + const std::vector& value, + ProcLib lib = ProcLib::DEFAULT); + + private: + int width_; + int height_; + std::vector value_; +}; +} // namespace vision +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/vision/common/processors/proc_lib.h b/3rdparty/include/fastdeploy/vision/common/processors/proc_lib.h new file mode 100644 index 0000000000..512ed9f837 --- /dev/null +++ b/3rdparty/include/fastdeploy/vision/common/processors/proc_lib.h @@ -0,0 +1,34 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#include "fastdeploy/utils/utils.h" + +namespace fastdeploy { +namespace vision { + +enum class FASTDEPLOY_DECL ProcLib { DEFAULT, OPENCV, FLYCV, CUDA }; + +FASTDEPLOY_DECL std::ostream& operator<<(std::ostream& out, const ProcLib& p); + +struct FASTDEPLOY_DECL DefaultProcLib { + // default_lib has the highest priority + // all the function in `processor` will force to use + // default_lib if this flag is set. + // DEFAULT means this flag is not set + static ProcLib default_lib; +}; + +} // namespace vision +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/vision/common/processors/resize.h b/3rdparty/include/fastdeploy/vision/common/processors/resize.h new file mode 100644 index 0000000000..e6a4ba1b05 --- /dev/null +++ b/3rdparty/include/fastdeploy/vision/common/processors/resize.h @@ -0,0 +1,63 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "fastdeploy/vision/common/processors/base.h" + +namespace fastdeploy { +namespace vision { + +class FASTDEPLOY_DECL Resize : public Processor { + public: + Resize(int width, int height, float scale_w = -1.0, float scale_h = -1.0, + int interp = 1, bool use_scale = false) { + width_ = width; + height_ = height; + scale_w_ = scale_w; + scale_h_ = scale_h; + interp_ = interp; + use_scale_ = use_scale; + } + + bool ImplByOpenCV(Mat* mat); +#ifdef ENABLE_FLYCV + bool ImplByFlyCV(Mat* mat); +#endif + std::string Name() { return "Resize"; } + + static bool Run(Mat* mat, int width, int height, float scale_w = -1.0, + float scale_h = -1.0, int interp = 1, bool use_scale = false, + ProcLib lib = ProcLib::DEFAULT); + + bool SetWidthAndHeight(int width, int height) { + width_ = width; + height_ = height; + return true; + } + + std::tuple GetWidthAndHeight() { + return std::make_tuple(width_, height_); + } + + private: + int width_; + int height_; + float scale_w_ = -1.0; + float scale_h_ = -1.0; + int interp_ = 1; + bool use_scale_ = false; +}; +} // namespace vision +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/vision/common/processors/resize_by_short.h b/3rdparty/include/fastdeploy/vision/common/processors/resize_by_short.h new file mode 100644 index 0000000000..151605bebd --- /dev/null +++ b/3rdparty/include/fastdeploy/vision/common/processors/resize_by_short.h @@ -0,0 +1,50 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "fastdeploy/vision/common/processors/base.h" + +namespace fastdeploy { +namespace vision { + +class FASTDEPLOY_DECL ResizeByShort : public Processor { + public: + ResizeByShort(int target_size, int interp = 1, bool use_scale = true, + const std::vector& max_hw = std::vector()) { + target_size_ = target_size; + max_hw_ = max_hw; + interp_ = interp; + use_scale_ = use_scale; + } + bool ImplByOpenCV(Mat* mat); +#ifdef ENABLE_FLYCV + bool ImplByFlyCV(Mat* mat); +#endif + std::string Name() { return "ResizeByShort"; } + + static bool Run(Mat* mat, int target_size, int interp = 1, + bool use_scale = true, + const std::vector& max_hw = std::vector(), + ProcLib lib = ProcLib::DEFAULT); + + private: + double GenerateScale(const int origin_w, const int origin_h); + int target_size_; + std::vector max_hw_; + int interp_; + bool use_scale_; +}; +} // namespace vision +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/vision/common/processors/stride_pad.h b/3rdparty/include/fastdeploy/vision/common/processors/stride_pad.h new file mode 100644 index 0000000000..18eebd54e1 --- /dev/null +++ b/3rdparty/include/fastdeploy/vision/common/processors/stride_pad.h @@ -0,0 +1,44 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "fastdeploy/vision/common/processors/base.h" + +namespace fastdeploy { +namespace vision { + +class FASTDEPLOY_DECL StridePad : public Processor { + public: + // only support pad with left-top padding mode + StridePad(int stride, const std::vector& value) { + stride_ = stride; + value_ = value; + } + bool ImplByOpenCV(Mat* mat); +#ifdef ENABLE_FLYCV + bool ImplByFlyCV(Mat* mat); +#endif + std::string Name() { return "StridePad"; } + + static bool Run(Mat* mat, int stride, + const std::vector& value = std::vector(), + ProcLib lib = ProcLib::DEFAULT); + + private: + int stride_ = 32; + std::vector value_; +}; +} // namespace vision +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/vision/common/processors/transform.h b/3rdparty/include/fastdeploy/vision/common/processors/transform.h new file mode 100644 index 0000000000..e3f7146365 --- /dev/null +++ b/3rdparty/include/fastdeploy/vision/common/processors/transform.h @@ -0,0 +1,50 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "fastdeploy/vision/common/processors/cast.h" +#include "fastdeploy/vision/common/processors/center_crop.h" +#include "fastdeploy/vision/common/processors/color_space_convert.h" +#include "fastdeploy/vision/common/processors/convert.h" +#include "fastdeploy/vision/common/processors/convert_and_permute.h" +#include "fastdeploy/vision/common/processors/crop.h" +#include "fastdeploy/vision/common/processors/hwc2chw.h" +#include "fastdeploy/vision/common/processors/limit_by_stride.h" +#include "fastdeploy/vision/common/processors/limit_short.h" +#include "fastdeploy/vision/common/processors/normalize.h" +#include "fastdeploy/vision/common/processors/normalize_and_permute.h" +#include "fastdeploy/vision/common/processors/pad.h" +#include "fastdeploy/vision/common/processors/pad_to_size.h" +#include "fastdeploy/vision/common/processors/resize.h" +#include "fastdeploy/vision/common/processors/resize_by_short.h" +#include "fastdeploy/vision/common/processors/stride_pad.h" +#include "fastdeploy/vision/common/processors/warp_affine.h" +#include + +namespace fastdeploy { +namespace vision { + +void FuseTransforms(std::vector>* processors); + +// Fuse Normalize + Cast(Float) to Normalize +void FuseNormalizeCast(std::vector>* processors); +// Fuse Normalize + HWC2CHW to NormalizeAndPermute +void FuseNormalizeHWC2CHW(std::vector>* processors); +// Fuse Normalize + Color Convert +void FuseNormalizeColorConvert( + std::vector>* processors); + +} // namespace vision +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/vision/common/processors/utils.h b/3rdparty/include/fastdeploy/vision/common/processors/utils.h new file mode 100644 index 0000000000..50074c3a0c --- /dev/null +++ b/3rdparty/include/fastdeploy/vision/common/processors/utils.h @@ -0,0 +1,53 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "fastdeploy/core/fd_tensor.h" +#include "fastdeploy/utils/utils.h" +#include "opencv2/core/core.hpp" + +#ifdef ENABLE_FLYCV +#include "flycv.h" // NOLINT +#endif + +namespace fastdeploy { +namespace vision { + +// Convert data type of opencv to FDDataType +FDDataType OpenCVDataTypeToFD(int type); +// Create data type of opencv by FDDataType +int CreateOpenCVDataType(FDDataType type, int channel = 1); +#ifdef ENABLE_FLYCV +// Convert data type of flycv to FDDataType +FDDataType FlyCVDataTypeToFD(fcv::FCVImageType type); +// Create data type of flycv by FDDataType +fcv::FCVImageType CreateFlyCVDataType(FDDataType type, int channel = 1); +// Convert cv::Mat to fcv::Mat +fcv::Mat ConvertOpenCVMatToFlyCV(cv::Mat& im); +// Convert fcv::Mat to fcv::mat +cv::Mat ConvertFlyCVMatToOpenCV(fcv::Mat& fim); +#endif + +// Create zero copy OpenCV/FlyCV Mat from FD Tensor / Buffer +cv::Mat CreateZeroCopyOpenCVMatFromBuffer(int height, int width, + int channels, FDDataType type, void* data); +cv::Mat CreateZeroCopyOpenCVMatFromTensor(const FDTensor& tensor); +#ifdef ENABLE_FLYCV +fcv::Mat CreateZeroCopyFlyCVMatFromBuffer(int height, int width, + int channels, FDDataType type, void* data); +fcv::Mat CreateZeroCopyFlyCVMatFromTensor(const FDTensor& tensor); +#endif +} // namespace vision +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/vision/common/processors/warp_affine.h b/3rdparty/include/fastdeploy/vision/common/processors/warp_affine.h new file mode 100644 index 0000000000..4349ee206b --- /dev/null +++ b/3rdparty/include/fastdeploy/vision/common/processors/warp_affine.h @@ -0,0 +1,61 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "fastdeploy/vision/common/processors/base.h" + +namespace fastdeploy { +namespace vision { + +class WarpAffine : public Processor { + public: + WarpAffine(const cv::Mat& trans_matrix, int width, int height, int interp = 1, + int border_mode = 0, + const cv::Scalar& borderValue = cv::Scalar()) { + trans_matrix_ = trans_matrix; + width_ = width; + height_ = height; + interp_ = interp; + border_mode_ = border_mode; + borderValue_ = borderValue; + } + + bool ImplByOpenCV(Mat* mat); + std::string Name() { return "WarpAffine"; } + + bool SetTransformMatrix(const cv::Mat& trans_matrix) { + trans_matrix_ = trans_matrix; + return true; + } + + std::tuple GetWidthAndHeight() { + return std::make_tuple(width_, height_); + } + + static bool Run(Mat* mat, const cv::Mat& trans_matrix, int width, int height, + int interp = 1, int border_mode = 0, + const cv::Scalar& borderValue = cv::Scalar(), + ProcLib lib = ProcLib::DEFAULT); + + private: + cv::Mat trans_matrix_; + int width_; + int height_; + int interp_; + int border_mode_; + cv::Scalar borderValue_; +}; +} // namespace vision +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/vision/common/result.h b/3rdparty/include/fastdeploy/vision/common/result.h new file mode 100644 index 0000000000..59690ab6d8 --- /dev/null +++ b/3rdparty/include/fastdeploy/vision/common/result.h @@ -0,0 +1,347 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once +#include "fastdeploy/fastdeploy_model.h" +#include "opencv2/core/core.hpp" +#include + +namespace fastdeploy { +/** \brief All C++ FastDeploy Vision Models APIs are defined inside this namespace +* +*/ +namespace vision { +enum FASTDEPLOY_DECL ResultType { + UNKNOWN_RESULT, + CLASSIFY, + DETECTION, + SEGMENTATION, + OCR, + MOT, + FACE_DETECTION, + FACE_ALIGNMENT, + FACE_RECOGNITION, + MATTING, + MASK, + KEYPOINT_DETECTION, + HEADPOSE, +}; + +struct FASTDEPLOY_DECL BaseResult { + ResultType type = ResultType::UNKNOWN_RESULT; +}; + +/*! @brief Classify result structure for all the image classify models + */ +struct FASTDEPLOY_DECL ClassifyResult : public BaseResult { + ClassifyResult() = default; + /// Classify result for an image + std::vector label_ids; + /// The confidence for each classify result + std::vector scores; + ResultType type = ResultType::CLASSIFY; + + /// Clear result + void Clear(); + + /// Copy constructor + ClassifyResult(const ClassifyResult& other) = default; + /// Move assignment + ClassifyResult& operator=(ClassifyResult&& other); + + /// Debug function, convert the result to string to print + std::string Str(); +}; + +/*! Mask structure, used in DetectionResult for instance segmentation models + */ +struct FASTDEPLOY_DECL Mask : public BaseResult { + /// Mask data buffer + std::vector data; + /// Shape of mask + std::vector shape; // (H,W) ... + ResultType type = ResultType::MASK; + + /// clear mask + void Clear(); + + /// Return a mutable pointer of the mask data buffer + void* Data() { return data.data(); } + + /// Return a pointer of the mask data buffer for read only + const void* Data() const { return data.data(); } + + /// Reserve size for mask data buffer + void Reserve(int size); + + /// Resize the mask data buffer + void Resize(int size); + + /// Debug function, convert the result to string to print + std::string Str(); +}; + + +/*! @brief Detection result structure for all the object detection models and instance segmentation models + */ +struct FASTDEPLOY_DECL DetectionResult : public BaseResult { + /** \brief All the detected object boxes for an input image, the size of `boxes` is the number of detected objects, and the element of `boxes` is a array of 4 float values, means [xmin, ymin, xmax, ymax] + */ + std::vector> boxes; + /** \brief The confidence for all the detected objects + */ + std::vector scores; + /// The classify label for all the detected objects + std::vector label_ids; + /** \brief For instance segmentation model, `masks` is the predict mask for all the deteced objects + */ + std::vector masks; + //// Shows if the DetectionResult has mask + bool contain_masks = false; + + ResultType type = ResultType::DETECTION; + + DetectionResult() {} + DetectionResult(const DetectionResult& res); + + /// Clear detection result + void Clear(); + + void Reserve(int size); + + void Resize(int size); + + /// Debug function, convert the result to string to print + std::string Str(); +}; + +/*! @brief KeyPoint Detection result structure for all the keypoint detection models + */ +struct FASTDEPLOY_DECL KeyPointDetectionResult : public BaseResult { + /** \brief All the coordinates of detected keypoints for an input image, the size of `keypoints` is num_detected_objects * num_joints, and the element of `keypoint` is a array of 2 float values, means [x, y] + */ + std::vector> keypoints; + //// The confidence for all the detected points + std::vector scores; + //// Number of joints for a detected object + int num_joints = -1; + + ResultType type = ResultType::KEYPOINT_DETECTION; + /// Clear detection result + void Clear(); + + void Reserve(int size); + + void Resize(int size); + + /// Debug function, convert the result to string to print + std::string Str(); +}; + +struct FASTDEPLOY_DECL OCRResult : public BaseResult { + std::vector> boxes; + + std::vector text; + std::vector rec_scores; + + std::vector cls_scores; + std::vector cls_labels; + + ResultType type = ResultType::OCR; + + void Clear(); + + std::string Str(); +}; + +/*! @brief MOT(Multi-Object Tracking) result structure for all the MOT models + */ +struct FASTDEPLOY_DECL MOTResult : public BaseResult { + /** \brief All the tracking object boxes for an input image, the size of `boxes` is the number of tracking objects, and the element of `boxes` is a array of 4 float values, means [xmin, ymin, xmax, ymax] + */ + std::vector> boxes; + /** \brief All the tracking object ids + */ + std::vector ids; + /** \brief The confidence for all the tracking objects + */ + std::vector scores; + /** \brief The classify label id for all the tracking object + */ + std::vector class_ids; + + ResultType type = ResultType::MOT; + /// Clear MOT result + void Clear(); + /// Debug function, convert the result to string to print + std::string Str(); +}; + + + +/*! @brief Face detection result structure for all the face detection models + */ +struct FASTDEPLOY_DECL FaceDetectionResult : public BaseResult { + /** \brief All the detected object boxes for an input image, the size of `boxes` is the number of detected objects, and the element of `boxes` is a array of 4 float values, means [xmin, ymin, xmax, ymax] + */ + std::vector> boxes; + /** \brief + * If the model detect face with landmarks, every detected object box correspoing to a landmark, which is a array of 2 float values, means location [x,y] + */ + std::vector> landmarks; + /** \brief + * Indicates the confidence of all targets detected from a single image, and the number of elements is consistent with boxes.size() + */ + std::vector scores; + ResultType type = ResultType::FACE_DETECTION; + /** \brief + * `landmarks_per_face` indicates the number of face landmarks for each detected face + * if the model's output contains face landmarks (such as YOLOv5Face, SCRFD, ...) + */ + int landmarks_per_face; + + FaceDetectionResult() { landmarks_per_face = 0; } + FaceDetectionResult(const FaceDetectionResult& res); + /// Clear detection result + void Clear(); + + void Reserve(int size); + + void Resize(int size); + /// Debug function, convert the result to string to print + std::string Str(); +}; + +/*! @brief Face Alignment result structure for all the face alignment models + */ +struct FASTDEPLOY_DECL FaceAlignmentResult : public BaseResult { + /** \brief All the coordinates of detected landmarks for an input image, and the element of `landmarks` is a array of 2 float values, means [x, y] + */ + std::vector> landmarks; + + ResultType type = ResultType::FACE_ALIGNMENT; + /// Clear facealignment result + void Clear(); + + void Reserve(int size); + + void Resize(int size); + + /// Debug function, convert the result to string to print + std::string Str(); +}; + +/*! @brief Segmentation result structure for all the segmentation models + */ +struct FASTDEPLOY_DECL SegmentationResult : public BaseResult { + /** \brief + * `label_map` stores the pixel-level category labels for input image. the number of pixels is equal to label_map.size() + */ + std::vector label_map; + /** \brief + * `score_map` stores the probability of the predicted label for each pixel of input image. + */ + std::vector score_map; + /// The output shape, means [H, W] + std::vector shape; + bool contain_score_map = false; + + ResultType type = ResultType::SEGMENTATION; + /// Clear detection result + void Clear(); + + void Reserve(int size); + + void Resize(int size); + + /// Debug function, convert the result to string to print + std::string Str(); +}; + +/*! @brief Face recognition result structure for all the Face recognition models + */ +struct FASTDEPLOY_DECL FaceRecognitionResult : public BaseResult { + /** \brief The feature embedding that represents the final extraction of the face recognition model can be used to calculate the feature similarity between faces. + */ + std::vector embedding; + + ResultType type = ResultType::FACE_RECOGNITION; + + FaceRecognitionResult() {} + FaceRecognitionResult(const FaceRecognitionResult& res); + /// Clear detection result + void Clear(); + + void Reserve(int size); + + void Resize(int size); + /// Debug function, convert the result to string to print + std::string Str(); +}; + +/*! @brief Matting result structure for all the Matting models + */ +struct FASTDEPLOY_DECL MattingResult : public BaseResult { + /** \brief + `alpha` is a one-dimensional vector, which is the predicted alpha transparency value. The range of values is [0., 1.], and the length is hxw. h, w are the height and width of the input image + */ + std::vector alpha; // h x w + /** \brief + If the model can predict foreground, `foreground` save the predicted foreground image, the shape is [hight,width,channel] generally. + */ + std::vector foreground; // h x w x c (c=3 default) + /** \brief + * The shape of output result, when contain_foreground == false, shape only contains (h, w), when contain_foreground == true, shape contains (h, w, c), and c is generally 3 + */ + std::vector shape; + /** \brief + If the model can predict alpha matte and foreground, contain_foreground = true, default false + */ + bool contain_foreground = false; + + ResultType type = ResultType::MATTING; + + MattingResult() {} + MattingResult(const MattingResult& res); + /// Clear detection result + void Clear(); + + void Reserve(int size); + + void Resize(int size); + /// Debug function, convert the result to string to print + std::string Str(); +}; + +/*! @brief HeadPose result structure for all the headpose models + */ +struct FASTDEPLOY_DECL HeadPoseResult : public BaseResult { + /** \brief EulerAngles for an input image, and the element of `euler_angles` is a vector, contains {yaw, pitch, roll} + */ + std::vector euler_angles; + + ResultType type = ResultType::HEADPOSE; + /// Clear headpose result + void Clear(); + + void Reserve(int size); + + void Resize(int size); + + /// Debug function, convert the result to string to print + std::string Str(); +}; + + +} // namespace vision +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/vision/ocr/ppocr/classifier.h b/3rdparty/include/fastdeploy/vision/ocr/ppocr/classifier.h new file mode 100644 index 0000000000..d87fec6fa6 --- /dev/null +++ b/3rdparty/include/fastdeploy/vision/ocr/ppocr/classifier.h @@ -0,0 +1,71 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#include "fastdeploy/fastdeploy_model.h" +#include "fastdeploy/vision/common/processors/transform.h" +#include "fastdeploy/vision/common/result.h" +#include "fastdeploy/vision/ocr/ppocr/utils/ocr_postprocess_op.h" + +namespace fastdeploy { +namespace vision { +/** \brief All OCR series model APIs are defined inside this namespace + * + */ +namespace ocr { +/*! @brief Classifier object is used to load the classification model provided by PaddleOCR. + */ +class FASTDEPLOY_DECL Classifier : public FastDeployModel { + public: + Classifier(); + /** \brief Set path of model file, and the configuration of runtime + * + * \param[in] model_file Path of model file, e.g ./ch_ppocr_mobile_v2.0_cls_infer/model.pdmodel. + * \param[in] params_file Path of parameter file, e.g ./ch_ppocr_mobile_v2.0_cls_infer/model.pdiparams, if the model format is ONNX, this parameter will be ignored. + * \param[in] custom_option RuntimeOption for inference, the default will use cpu, and choose the backend defined in `valid_cpu_backends`. + * \param[in] model_format Model format of the loaded model, default is Paddle format. + */ + Classifier(const std::string& model_file, const std::string& params_file = "", + const RuntimeOption& custom_option = RuntimeOption(), + const ModelFormat& model_format = ModelFormat::PADDLE); + /// Get model's name + std::string ModelName() const { return "ppocr/ocr_cls"; } + /** \brief Predict the input image and get OCR classification model result. + * + * \param[in] im The input image data, comes from cv::imread(), is a 3-D array with layout HWC, BGR format. + * \param[in] result The output of OCR classification model result will be writen to this structure. + * \return true if the prediction is successed, otherwise false. + */ + virtual bool Predict(cv::Mat* img, std::tuple* result); + + // Pre & Post parameters + float cls_thresh; + std::vector cls_image_shape; + int cls_batch_num; + + std::vector mean; + std::vector scale; + bool is_scale; + + private: + bool Initialize(); + /// Preprocess the input data, and set the preprocessed results to `outputs` + bool Preprocess(Mat* img, FDTensor* output); + /// Postprocess the inferenced results, and set the final result to `result` + bool Postprocess(FDTensor& infer_result, std::tuple* result); +}; + +} // namespace ocr +} // namespace vision +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/vision/ocr/ppocr/dbdetector.h b/3rdparty/include/fastdeploy/vision/ocr/ppocr/dbdetector.h new file mode 100644 index 0000000000..e0baf319cb --- /dev/null +++ b/3rdparty/include/fastdeploy/vision/ocr/ppocr/dbdetector.h @@ -0,0 +1,85 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#include "fastdeploy/fastdeploy_model.h" +#include "fastdeploy/vision/common/processors/transform.h" +#include "fastdeploy/vision/common/result.h" +#include "fastdeploy/vision/ocr/ppocr/utils/ocr_postprocess_op.h" + +namespace fastdeploy { +namespace vision { +/** \brief All OCR series model APIs are defined inside this namespace + * + */ +namespace ocr { + +/*! @brief DBDetector object is used to load the detection model provided by PaddleOCR. + */ +class FASTDEPLOY_DECL DBDetector : public FastDeployModel { + public: + DBDetector(); + /** \brief Set path of model file, and the configuration of runtime + * + * \param[in] model_file Path of model file, e.g ./ch_PP-OCRv3_det_infer/model.pdmodel. + * \param[in] params_file Path of parameter file, e.g ./ch_PP-OCRv3_det_infer/model.pdiparams, if the model format is ONNX, this parameter will be ignored. + * \param[in] custom_option RuntimeOption for inference, the default will use cpu, and choose the backend defined in `valid_cpu_backends`. + * \param[in] model_format Model format of the loaded model, default is Paddle format. + */ + DBDetector(const std::string& model_file, const std::string& params_file = "", + const RuntimeOption& custom_option = RuntimeOption(), + const ModelFormat& model_format = ModelFormat::PADDLE); + /// Get model's name + std::string ModelName() const { return "ppocr/ocr_det"; } + /** \brief Predict the input image and get OCR detection model result. + * + * \param[in] im The input image data, comes from cv::imread(), is a 3-D array with layout HWC, BGR format. + * \param[in] boxes_result The output of OCR detection model result will be writen to this structure. + * \return true if the prediction is successed, otherwise false. + */ + virtual bool Predict(cv::Mat* im, + std::vector>* boxes_result); + + // Pre & Post process parameters + int max_side_len; + + float ratio_h{}; + float ratio_w{}; + + double det_db_thresh; + double det_db_box_thresh; + double det_db_unclip_ratio; + std::string det_db_score_mode; + bool use_dilation; + + std::vector mean; + std::vector scale; + bool is_scale; + + private: + bool Initialize(); + /// Preprocess the input data, and set the preprocessed results to `outputs` + bool Preprocess(Mat* mat, FDTensor* outputs, + std::map>* im_info); + /*! @brief Postprocess the inferenced results, and set the final result to `boxes_result` + */ + bool Postprocess(FDTensor& infer_result, + std::vector>* boxes_result, + const std::map>& im_info); + PostProcessor post_processor_; +}; + +} // namespace ocr +} // namespace vision +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/vision/ocr/ppocr/ppocr_v2.h b/3rdparty/include/fastdeploy/vision/ocr/ppocr/ppocr_v2.h new file mode 100644 index 0000000000..bf53000202 --- /dev/null +++ b/3rdparty/include/fastdeploy/vision/ocr/ppocr/ppocr_v2.h @@ -0,0 +1,83 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +#include "fastdeploy/fastdeploy_model.h" +#include "fastdeploy/vision/common/processors/transform.h" +#include "fastdeploy/vision/common/result.h" + +#include "fastdeploy/vision/ocr/ppocr/classifier.h" +#include "fastdeploy/vision/ocr/ppocr/dbdetector.h" +#include "fastdeploy/vision/ocr/ppocr/recognizer.h" +#include "fastdeploy/vision/ocr/ppocr/utils/ocr_postprocess_op.h" + +namespace fastdeploy { +/** \brief This pipeline can launch detection model, classification model and recognition model sequentially. All OCR pipeline APIs are defined inside this namespace. + * + */ +namespace pipeline { +/*! @brief PPOCRv2 is used to load PP-OCRv2 series models provided by PaddleOCR. + */ +class FASTDEPLOY_DECL PPOCRv2 : public FastDeployModel { + public: + /** \brief Set up the detection model path, classification model path and recognition model path respectively. + * + * \param[in] det_model Path of detection model, e.g ./ch_PP-OCRv2_det_infer + * \param[in] cls_model Path of classification model, e.g ./ch_ppocr_mobile_v2.0_cls_infer + * \param[in] rec_model Path of recognition model, e.g ./ch_PP-OCRv2_rec_infer + */ + PPOCRv2(fastdeploy::vision::ocr::DBDetector* det_model, + fastdeploy::vision::ocr::Classifier* cls_model, + fastdeploy::vision::ocr::Recognizer* rec_model); + + /** \brief Classification model is optional, so this function is set up the detection model path and recognition model path respectively. + * + * \param[in] det_model Path of detection model, e.g ./ch_PP-OCRv2_det_infer + * \param[in] rec_model Path of recognition model, e.g ./ch_PP-OCRv2_rec_infer + */ + PPOCRv2(fastdeploy::vision::ocr::DBDetector* det_model, + fastdeploy::vision::ocr::Recognizer* rec_model); + + /** \brief Predict the input image and get OCR result. + * + * \param[in] im The input image data, comes from cv::imread(), is a 3-D array with layout HWC, BGR format. + * \param[in] result The output OCR result will be writen to this structure. + * \return true if the prediction successed, otherwise false. + */ + virtual bool Predict(cv::Mat* img, fastdeploy::vision::OCRResult* result); + bool Initialized() const override; + + protected: + fastdeploy::vision::ocr::DBDetector* detector_ = nullptr; + fastdeploy::vision::ocr::Classifier* classifier_ = nullptr; + fastdeploy::vision::ocr::Recognizer* recognizer_ = nullptr; + /// Launch the detection process in OCR. + virtual bool Detect(cv::Mat* img, fastdeploy::vision::OCRResult* result); + /// Launch the recognition process in OCR. + virtual bool Recognize(cv::Mat* img, fastdeploy::vision::OCRResult* result); + /// Launch the classification process in OCR. + virtual bool Classify(cv::Mat* img, fastdeploy::vision::OCRResult* result); +}; + +namespace application { +namespace ocrsystem { + typedef pipeline::PPOCRv2 PPOCRSystemv2; +} // namespace ocrsystem +} // namespace application + +} // namespace pipeline +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/vision/ocr/ppocr/ppocr_v3.h b/3rdparty/include/fastdeploy/vision/ocr/ppocr/ppocr_v3.h new file mode 100644 index 0000000000..e248eca75e --- /dev/null +++ b/3rdparty/include/fastdeploy/vision/ocr/ppocr/ppocr_v3.h @@ -0,0 +1,62 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "fastdeploy/vision/ocr/ppocr/ppocr_v2.h" + +namespace fastdeploy { +/** \brief This pipeline can launch detection model, classification model and recognition model sequentially. All OCR pipeline APIs are defined inside this namespace. + * + */ +namespace pipeline { +/*! @brief PPOCRv3 is used to load PP-OCRv3 series models provided by PaddleOCR. + */ +class FASTDEPLOY_DECL PPOCRv3 : public PPOCRv2 { + public: + /** \brief Set up the detection model path, classification model path and recognition model path respectively. + * + * \param[in] det_model Path of detection model, e.g ./ch_PP-OCRv3_det_infer + * \param[in] cls_model Path of classification model, e.g ./ch_ppocr_mobile_v2.0_cls_infer + * \param[in] rec_model Path of recognition model, e.g ./ch_PP-OCRv3_rec_infer + */ + PPOCRv3(fastdeploy::vision::ocr::DBDetector* det_model, + fastdeploy::vision::ocr::Classifier* cls_model, + fastdeploy::vision::ocr::Recognizer* rec_model) + : PPOCRv2(det_model, cls_model, rec_model) { + // The only difference between v2 and v3 + recognizer_->rec_image_shape[1] = 48; + } + /** \brief Classification model is optional, so this function is set up the detection model path and recognition model path respectively. + * + * \param[in] det_model Path of detection model, e.g ./ch_PP-OCRv3_det_infer + * \param[in] rec_model Path of recognition model, e.g ./ch_PP-OCRv3_rec_infer + */ + PPOCRv3(fastdeploy::vision::ocr::DBDetector* det_model, + fastdeploy::vision::ocr::Recognizer* rec_model) + : PPOCRv2(det_model, rec_model) { + // The only difference between v2 and v3 + recognizer_->rec_image_shape[1] = 48; + } +}; + +} // namespace pipeline + +namespace application { +namespace ocrsystem { + typedef pipeline::PPOCRv3 PPOCRSystemv3; +} // namespace ocrsystem +} // namespace application + +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/vision/ocr/ppocr/recognizer.h b/3rdparty/include/fastdeploy/vision/ocr/ppocr/recognizer.h new file mode 100644 index 0000000000..d3c5fcc9d4 --- /dev/null +++ b/3rdparty/include/fastdeploy/vision/ocr/ppocr/recognizer.h @@ -0,0 +1,79 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#include "fastdeploy/fastdeploy_model.h" +#include "fastdeploy/vision/common/processors/transform.h" +#include "fastdeploy/vision/common/result.h" +#include "fastdeploy/vision/ocr/ppocr/utils/ocr_postprocess_op.h" + +namespace fastdeploy { +namespace vision { +/** \brief All OCR series model APIs are defined inside this namespace + * + */ +namespace ocr { +/*! @brief Recognizer object is used to load the recognition model provided by PaddleOCR. + */ +class FASTDEPLOY_DECL Recognizer : public FastDeployModel { + public: + Recognizer(); + /** \brief Set path of model file, and the configuration of runtime + * + * \param[in] model_file Path of model file, e.g ./ch_PP-OCRv3_rec_infer/model.pdmodel. + * \param[in] params_file Path of parameter file, e.g ./ch_PP-OCRv3_rec_infer/model.pdiparams, if the model format is ONNX, this parameter will be ignored. + * \param[in] label_path Path of label file used by OCR recognition model. e.g ./ppocr_keys_v1.txt + * \param[in] custom_option RuntimeOption for inference, the default will use cpu, and choose the backend defined in `valid_cpu_backends`. + * \param[in] model_format Model format of the loaded model, default is Paddle format. + */ + Recognizer(const std::string& model_file, const std::string& params_file = "", + const std::string& label_path = "", + const RuntimeOption& custom_option = RuntimeOption(), + const ModelFormat& model_format = ModelFormat::PADDLE); + /// Get model's name + std::string ModelName() const { return "ppocr/ocr_rec"; } + /** \brief Predict the input image and get OCR recognition model result. + * + * \param[in] im The input image data, comes from cv::imread(), is a 3-D array with layout HWC, BGR format. + * \param[in] rec_result The output of OCR recognition model result will be writen to this structure. + * \return true if the prediction is successed, otherwise false. + */ + virtual bool Predict(cv::Mat* img, + std::tuple* rec_result); + + // Pre & Post parameters + std::vector label_list; + int rec_batch_num; + int rec_img_h; + int rec_img_w; + std::vector rec_image_shape; + + std::vector mean; + std::vector scale; + bool is_scale; + + private: + bool Initialize(); + /// Preprocess the input data, and set the preprocessed results to `outputs` + bool Preprocess(Mat* img, FDTensor* outputs, + const std::vector& rec_image_shape); + /*! @brief Postprocess the inferenced results, and set the final result to `rec_result` + */ + bool Postprocess(FDTensor& infer_result, + std::tuple* rec_result); +}; + +} // namespace ocr +} // namespace vision +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/vision/ocr/ppocr/utils/clipper.h b/3rdparty/include/fastdeploy/vision/ocr/ppocr/utils/clipper.h new file mode 100644 index 0000000000..60b03cee38 --- /dev/null +++ b/3rdparty/include/fastdeploy/vision/ocr/ppocr/utils/clipper.h @@ -0,0 +1,425 @@ +/******************************************************************************* +* * +* Author : Angus Johnson * +* Version : 6.4.2 * +* Date : 27 February 2017 * +* Website : http://www.angusj.com * +* Copyright : Angus Johnson 2010-2017 * +* * +* License: * +* Use, modification & distribution is subject to Boost Software License Ver 1. * +* http://www.boost.org/LICENSE_1_0.txt * +* * +* Attributions: * +* The code in this library is an extension of Bala Vatti's clipping algorithm: * +* "A generic solution to polygon clipping" * +* Communications of the ACM, Vol 35, Issue 7 (July 1992) pp 56-63. * +* http://portal.acm.org/citation.cfm?id=129906 * +* * +* Computer graphics and geometric modeling: implementation and algorithms * +* By Max K. Agoston * +* Springer; 1 edition (January 4, 2005) * +* http://books.google.com/books?q=vatti+clipping+agoston * +* * +* See also: * +* "Polygon Offsetting by Computing Winding Numbers" * +* Paper no. DETC2005-85513 pp. 565-575 * +* ASME 2005 International Design Engineering Technical Conferences * +* and Computers and Information in Engineering Conference (IDETC/CIE2005) * +* September 24-28, 2005 , Long Beach, California, USA * +* http://www.me.berkeley.edu/~mcmains/pubs/DAC05OffsetPolygon.pdf * +* * +*******************************************************************************/ + +#pragma once + +#ifndef clipper_hpp +#define clipper_hpp + +#define CLIPPER_VERSION "6.4.2" + +// use_int32: When enabled 32bit ints are used instead of 64bit ints. This +// improve performance but coordinate values are limited to the range +/- 46340 +//#define use_int32 + +// use_xyz: adds a Z member to IntPoint. Adds a minor cost to perfomance. +//#define use_xyz + +// use_lines: Enables line clipping. Adds a very minor cost to performance. +#define use_lines + +// use_deprecated: Enables temporary support for the obsolete functions +//#define use_deprecated + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ClipperLib { + +enum ClipType { ctIntersection, ctUnion, ctDifference, ctXor }; +enum PolyType { ptSubject, ptClip }; +// By far the most widely used winding rules for polygon filling are +// EvenOdd & NonZero (GDI, GDI+, XLib, OpenGL, Cairo, AGG, Quartz, SVG, Gr32) +// Others rules include Positive, Negative and ABS_GTR_EQ_TWO (only in OpenGL) +// see http://glprogramming.com/red/chapter11.html +enum PolyFillType { pftEvenOdd, pftNonZero, pftPositive, pftNegative }; + +#ifdef use_int32 +typedef int cInt; +static cInt const loRange = 0x7FFF; +static cInt const hiRange = 0x7FFF; +#else +typedef signed long long cInt; +static cInt const loRange = 0x3FFFFFFF; +static cInt const hiRange = 0x3FFFFFFFFFFFFFFFLL; +typedef signed long long long64; // used by Int128 class +typedef unsigned long long ulong64; + +#endif + +struct IntPoint { + cInt X; + cInt Y; +#ifdef use_xyz + cInt Z; + IntPoint(cInt x = 0, cInt y = 0, cInt z = 0) : X(x), Y(y), Z(z){}; +#else + IntPoint(cInt x = 0, cInt y = 0) : X(x), Y(y){}; +#endif + + friend inline bool operator==(const IntPoint &a, const IntPoint &b) { + return a.X == b.X && a.Y == b.Y; + } + friend inline bool operator!=(const IntPoint &a, const IntPoint &b) { + return a.X != b.X || a.Y != b.Y; + } +}; +//------------------------------------------------------------------------------ + +typedef std::vector Path; +typedef std::vector Paths; + +inline Path &operator<<(Path &poly, const IntPoint &p) { + poly.push_back(p); + return poly; +} +inline Paths &operator<<(Paths &polys, const Path &p) { + polys.push_back(p); + return polys; +} + +std::ostream &operator<<(std::ostream &s, const IntPoint &p); +std::ostream &operator<<(std::ostream &s, const Path &p); +std::ostream &operator<<(std::ostream &s, const Paths &p); + +struct DoublePoint { + double X; + double Y; + DoublePoint(double x = 0, double y = 0) : X(x), Y(y) {} + DoublePoint(IntPoint ip) : X((double)ip.X), Y((double)ip.Y) {} +}; +//------------------------------------------------------------------------------ + +#ifdef use_xyz +typedef void (*ZFillCallback)(IntPoint &e1bot, IntPoint &e1top, IntPoint &e2bot, + IntPoint &e2top, IntPoint &pt); +#endif + +enum InitOptions { + ioReverseSolution = 1, + ioStrictlySimple = 2, + ioPreserveCollinear = 4 +}; +enum JoinType { jtSquare, jtRound, jtMiter }; +enum EndType { + etClosedPolygon, + etClosedLine, + etOpenButt, + etOpenSquare, + etOpenRound +}; + +class PolyNode; +typedef std::vector PolyNodes; + +class PolyNode { + public: + PolyNode(); + virtual ~PolyNode(){}; + Path Contour; + PolyNodes Childs; + PolyNode *Parent; + PolyNode *GetNext() const; + bool IsHole() const; + bool IsOpen() const; + int ChildCount() const; + + private: + // PolyNode& operator =(PolyNode& other); + unsigned Index; // node index in Parent.Childs + bool m_IsOpen; + JoinType m_jointype; + EndType m_endtype; + PolyNode *GetNextSiblingUp() const; + void AddChild(PolyNode &child); + friend class Clipper; // to access Index + friend class ClipperOffset; +}; + +class PolyTree : public PolyNode { + public: + ~PolyTree() { Clear(); }; + PolyNode *GetFirst() const; + void Clear(); + int Total() const; + + private: + // PolyTree& operator =(PolyTree& other); + PolyNodes AllNodes; + friend class Clipper; // to access AllNodes +}; + +bool Orientation(const Path &poly); +double Area(const Path &poly); +int PointInPolygon(const IntPoint &pt, const Path &path); + +void SimplifyPolygon(const Path &in_poly, Paths &out_polys, + PolyFillType fillType = pftEvenOdd); +void SimplifyPolygons(const Paths &in_polys, Paths &out_polys, + PolyFillType fillType = pftEvenOdd); +void SimplifyPolygons(Paths &polys, PolyFillType fillType = pftEvenOdd); + +void CleanPolygon(const Path &in_poly, Path &out_poly, double distance = 1.415); +void CleanPolygon(Path &poly, double distance = 1.415); +void CleanPolygons(const Paths &in_polys, Paths &out_polys, + double distance = 1.415); +void CleanPolygons(Paths &polys, double distance = 1.415); + +void MinkowskiSum(const Path &pattern, const Path &path, Paths &solution, + bool pathIsClosed); +void MinkowskiSum(const Path &pattern, const Paths &paths, Paths &solution, + bool pathIsClosed); +void MinkowskiDiff(const Path &poly1, const Path &poly2, Paths &solution); + +void PolyTreeToPaths(const PolyTree &polytree, Paths &paths); +void ClosedPathsFromPolyTree(const PolyTree &polytree, Paths &paths); +void OpenPathsFromPolyTree(PolyTree &polytree, Paths &paths); + +void ReversePath(Path &p); +void ReversePaths(Paths &p); + +struct IntRect { + cInt left; + cInt top; + cInt right; + cInt bottom; +}; + +// enums that are used internally ... +enum EdgeSide { esLeft = 1, esRight = 2 }; + +// forward declarations (for stuff used internally) ... +struct TEdge; +struct IntersectNode; +struct LocalMinimum; +struct OutPt; +struct OutRec; +struct Join; + +typedef std::vector PolyOutList; +typedef std::vector EdgeList; +typedef std::vector JoinList; +typedef std::vector IntersectList; + +//------------------------------------------------------------------------------ + +// ClipperBase is the ancestor to the Clipper class. It should not be +// instantiated directly. This class simply abstracts the conversion of sets of +// polygon coordinates into edge objects that are stored in a LocalMinima list. +class ClipperBase { + public: + ClipperBase(); + virtual ~ClipperBase(); + virtual bool AddPath(const Path &pg, PolyType PolyTyp, bool Closed); + bool AddPaths(const Paths &ppg, PolyType PolyTyp, bool Closed); + virtual void Clear(); + IntRect GetBounds(); + bool PreserveCollinear() { return m_PreserveCollinear; }; + void PreserveCollinear(bool value) { m_PreserveCollinear = value; }; + + protected: + void DisposeLocalMinimaList(); + TEdge *AddBoundsToLML(TEdge *e, bool IsClosed); + virtual void Reset(); + TEdge *ProcessBound(TEdge *E, bool IsClockwise); + void InsertScanbeam(const cInt Y); + bool PopScanbeam(cInt &Y); + bool LocalMinimaPending(); + bool PopLocalMinima(cInt Y, const LocalMinimum *&locMin); + OutRec *CreateOutRec(); + void DisposeAllOutRecs(); + void DisposeOutRec(PolyOutList::size_type index); + void SwapPositionsInAEL(TEdge *edge1, TEdge *edge2); + void DeleteFromAEL(TEdge *e); + void UpdateEdgeIntoAEL(TEdge *&e); + + typedef std::vector MinimaList; + MinimaList::iterator m_CurrentLM; + MinimaList m_MinimaList; + + bool m_UseFullRange; + EdgeList m_edges; + bool m_PreserveCollinear; + bool m_HasOpenPaths; + PolyOutList m_PolyOuts; + TEdge *m_ActiveEdges; + + typedef std::priority_queue ScanbeamList; + ScanbeamList m_Scanbeam; +}; +//------------------------------------------------------------------------------ + +class Clipper : public virtual ClipperBase { + public: + Clipper(int initOptions = 0); + bool Execute(ClipType clipType, Paths &solution, + PolyFillType fillType = pftEvenOdd); + bool Execute(ClipType clipType, Paths &solution, PolyFillType subjFillType, + PolyFillType clipFillType); + bool Execute(ClipType clipType, PolyTree &polytree, + PolyFillType fillType = pftEvenOdd); + bool Execute(ClipType clipType, PolyTree &polytree, PolyFillType subjFillType, + PolyFillType clipFillType); + bool ReverseSolution() { return m_ReverseOutput; }; + void ReverseSolution(bool value) { m_ReverseOutput = value; }; + bool StrictlySimple() { return m_StrictSimple; }; + void StrictlySimple(bool value) { m_StrictSimple = value; }; +// set the callback function for z value filling on intersections (otherwise Z +// is 0) +#ifdef use_xyz + void ZFillFunction(ZFillCallback zFillFunc); +#endif + protected: + virtual bool ExecuteInternal(); + + private: + JoinList m_Joins; + JoinList m_GhostJoins; + IntersectList m_IntersectList; + ClipType m_ClipType; + typedef std::list MaximaList; + MaximaList m_Maxima; + TEdge *m_SortedEdges; + bool m_ExecuteLocked; + PolyFillType m_ClipFillType; + PolyFillType m_SubjFillType; + bool m_ReverseOutput; + bool m_UsingPolyTree; + bool m_StrictSimple; +#ifdef use_xyz + ZFillCallback m_ZFill; // custom callback +#endif + void SetWindingCount(TEdge &edge); + bool IsEvenOddFillType(const TEdge &edge) const; + bool IsEvenOddAltFillType(const TEdge &edge) const; + void InsertLocalMinimaIntoAEL(const cInt botY); + void InsertEdgeIntoAEL(TEdge *edge, TEdge *startEdge); + void AddEdgeToSEL(TEdge *edge); + bool PopEdgeFromSEL(TEdge *&edge); + void CopyAELToSEL(); + void DeleteFromSEL(TEdge *e); + void SwapPositionsInSEL(TEdge *edge1, TEdge *edge2); + bool IsContributing(const TEdge &edge) const; + bool IsTopHorz(const cInt XPos); + void DoMaxima(TEdge *e); + void ProcessHorizontals(); + void ProcessHorizontal(TEdge *horzEdge); + void AddLocalMaxPoly(TEdge *e1, TEdge *e2, const IntPoint &pt); + OutPt *AddLocalMinPoly(TEdge *e1, TEdge *e2, const IntPoint &pt); + OutRec *GetOutRec(int idx); + void AppendPolygon(TEdge *e1, TEdge *e2); + void IntersectEdges(TEdge *e1, TEdge *e2, IntPoint &pt); + OutPt *AddOutPt(TEdge *e, const IntPoint &pt); + OutPt *GetLastOutPt(TEdge *e); + bool ProcessIntersections(const cInt topY); + void BuildIntersectList(const cInt topY); + void ProcessIntersectList(); + void ProcessEdgesAtTopOfScanbeam(const cInt topY); + void BuildResult(Paths &polys); + void BuildResult2(PolyTree &polytree); + void SetHoleState(TEdge *e, OutRec *outrec); + void DisposeIntersectNodes(); + bool FixupIntersectionOrder(); + void FixupOutPolygon(OutRec &outrec); + void FixupOutPolyline(OutRec &outrec); + bool IsHole(TEdge *e); + bool FindOwnerFromSplitRecs(OutRec &outRec, OutRec *&currOrfl); + void FixHoleLinkage(OutRec &outrec); + void AddJoin(OutPt *op1, OutPt *op2, const IntPoint offPt); + void ClearJoins(); + void ClearGhostJoins(); + void AddGhostJoin(OutPt *op, const IntPoint offPt); + bool JoinPoints(Join *j, OutRec *outRec1, OutRec *outRec2); + void JoinCommonEdges(); + void DoSimplePolygons(); + void FixupFirstLefts1(OutRec *OldOutRec, OutRec *NewOutRec); + void FixupFirstLefts2(OutRec *InnerOutRec, OutRec *OuterOutRec); + void FixupFirstLefts3(OutRec *OldOutRec, OutRec *NewOutRec); +#ifdef use_xyz + void SetZ(IntPoint &pt, TEdge &e1, TEdge &e2); +#endif +}; +//------------------------------------------------------------------------------ + +class ClipperOffset { + public: + ClipperOffset(double miterLimit = 2.0, double roundPrecision = 0.25); + ~ClipperOffset(); + void AddPath(const Path &path, JoinType joinType, EndType endType); + void AddPaths(const Paths &paths, JoinType joinType, EndType endType); + void Execute(Paths &solution, double delta); + void Execute(PolyTree &solution, double delta); + void Clear(); + double MiterLimit; + double ArcTolerance; + + private: + Paths m_destPolys; + Path m_srcPoly; + Path m_destPoly; + std::vector m_normals; + double m_delta, m_sinA, m_sin, m_cos; + double m_miterLim, m_StepsPerRad; + IntPoint m_lowest; + PolyNode m_polyNodes; + + void FixOrientations(); + void DoOffset(double delta); + void OffsetPoint(int j, int &k, JoinType jointype); + void DoSquare(int j, int k); + void DoMiter(int j, int k, double r); + void DoRound(int j, int k); +}; +//------------------------------------------------------------------------------ + +class clipperException : public std::exception { + public: + clipperException(const char *description) : m_descr(description) {} + virtual ~clipperException() throw() {} + virtual const char *what() const throw() { return m_descr.c_str(); } + + private: + std::string m_descr; +}; +//------------------------------------------------------------------------------ + +} // ClipperLib namespace + +#endif // clipper_hpp diff --git a/3rdparty/include/fastdeploy/vision/ocr/ppocr/utils/ocr_postprocess_op.h b/3rdparty/include/fastdeploy/vision/ocr/ppocr/utils/ocr_postprocess_op.h new file mode 100644 index 0000000000..8422704fd5 --- /dev/null +++ b/3rdparty/include/fastdeploy/vision/ocr/ppocr/utils/ocr_postprocess_op.h @@ -0,0 +1,91 @@ +// Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include "opencv2/core.hpp" +#include "opencv2/imgcodecs.hpp" +#include "opencv2/imgproc.hpp" + +#include +#include +#include + +#include "fastdeploy/vision/ocr/ppocr/utils/clipper.h" + +namespace fastdeploy { +namespace vision { +namespace ocr { + +class PostProcessor { + public: + void GetContourArea(const std::vector> &box, + float unclip_ratio, float &distance); + + cv::RotatedRect UnClip(std::vector> box, + const float &unclip_ratio); + + float **Mat2Vec(cv::Mat mat); + + std::vector> OrderPointsClockwise( + std::vector> pts); + + std::vector> GetMiniBoxes(cv::RotatedRect box, + float &ssid); + + float BoxScoreFast(std::vector> box_array, cv::Mat pred); + float PolygonScoreAcc(std::vector contour, cv::Mat pred); + + std::vector>> BoxesFromBitmap( + const cv::Mat pred, const cv::Mat bitmap, const float &box_thresh, + const float &det_db_unclip_ratio, const std::string &det_db_score_mode); + + std::vector>> FilterTagDetRes( + std::vector>> boxes, float ratio_h, + float ratio_w, + const std::map> &im_info); + + private: + static bool XsortInt(std::vector a, std::vector b); + + static bool XsortFp32(std::vector a, std::vector b); + + std::vector> Mat2Vector(cv::Mat mat); + + inline int _max(int a, int b) { return a >= b ? a : b; } + + inline int _min(int a, int b) { return a >= b ? b : a; } + + template + inline T clamp(T x, T min, T max) { + if (x > max) return max; + if (x < min) return min; + return x; + } + + inline float clampf(float x, float min, float max) { + if (x > max) return max; + if (x < min) return min; + return x; + } +}; + +} // namespace ocr +} // namespace vision +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/vision/ocr/ppocr/utils/ocr_utils.h b/3rdparty/include/fastdeploy/vision/ocr/ppocr/utils/ocr_utils.h new file mode 100644 index 0000000000..36994389ba --- /dev/null +++ b/3rdparty/include/fastdeploy/vision/ocr/ppocr/utils/ocr_utils.h @@ -0,0 +1,38 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include "fastdeploy/core/fd_tensor.h" +#include "fastdeploy/utils/utils.h" +#include "fastdeploy/vision/common/result.h" + +#include "opencv2/core.hpp" +#include "opencv2/imgcodecs.hpp" +#include "opencv2/imgproc.hpp" + +namespace fastdeploy { +namespace vision { +namespace ocr { + +cv::Mat GetRotateCropImage(const cv::Mat& srcimage, + const std::array& box); + +void SortBoxes(OCRResult* result); + +} // namespace ocr +} // namespace vision +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/vision/utils/cuda_utils.h b/3rdparty/include/fastdeploy/vision/utils/cuda_utils.h new file mode 100644 index 0000000000..0f0f414e95 --- /dev/null +++ b/3rdparty/include/fastdeploy/vision/utils/cuda_utils.h @@ -0,0 +1,42 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include + +#ifndef CUDA_CHECK +#define CUDA_CHECK(callstr)\ + {\ + cudaError_t error_code = callstr;\ + if (error_code != cudaSuccess) {\ + std::cerr << "CUDA error " << error_code << " at " << __FILE__ << ":";\ + std::cerr << __LINE__;\ + assert(0);\ + }\ + } +#endif // CUDA_CHECK + +namespace fastdeploy { +namespace vision { +namespace utils { +void CudaYoloPreprocess(uint8_t* src, int src_width, int src_height, + float* dst, int dst_width, int dst_height, + const std::vector padding_value, + cudaStream_t stream); +} // namespace utils +} // namespace vision +} // namespace fastdeploy diff --git a/3rdparty/include/fastdeploy/vision/utils/utils.h b/3rdparty/include/fastdeploy/vision/utils/utils.h new file mode 100644 index 0000000000..1590922d8a --- /dev/null +++ b/3rdparty/include/fastdeploy/vision/utils/utils.h @@ -0,0 +1,104 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include + +#include "fastdeploy/core/fd_tensor.h" +#include "fastdeploy/utils/utils.h" +#include "fastdeploy/vision/common/result.h" + +// #include "unsupported/Eigen/CXX11/Tensor" +#include "fastdeploy/function/reduce.h" +#include "fastdeploy/function/softmax.h" +#include "fastdeploy/function/transpose.h" +#include "fastdeploy/vision/common/processors/mat.h" + +namespace fastdeploy { +namespace vision { +namespace utils { +// topk sometimes is a very small value +// so this implementation is simple but I don't think it will +// cost too much time +// Also there may be cause problem since we suppose the minimum value is +// -99999999 +// Do not use this function on array which topk contains value less than +// -99999999 +template +std::vector TopKIndices(const T* array, int array_size, int topk) { + topk = std::min(array_size, topk); + std::vector res(topk); + std::set searched; + for (int32_t i = 0; i < topk; ++i) { + T min = static_cast(-99999999); + for (int32_t j = 0; j < array_size; ++j) { + if (searched.find(j) != searched.end()) { + continue; + } + if (*(array + j) > min) { + res[i] = j; + min = *(array + j); + } + } + searched.insert(res[i]); + } + return res; +} + +void NMS(DetectionResult* output, float iou_threshold = 0.5); + +void NMS(FaceDetectionResult* result, float iou_threshold = 0.5); + +// MergeSort +void SortDetectionResult(DetectionResult* output); + +void SortDetectionResult(FaceDetectionResult* result); + +// L2 Norm / cosine similarity (for face recognition, ...) +FASTDEPLOY_DECL std::vector L2Normalize( + const std::vector& values); + +FASTDEPLOY_DECL float CosineSimilarity(const std::vector& a, + const std::vector& b, + bool normalized = true); + +bool CropImageByBox(Mat& src_im, Mat* dst_im, + const std::vector& box, std::vector* center, + std::vector* scale, const float expandratio = 0.3); + +/** + * Function: for keypoint detection model, fine positioning of keypoints in + * postprocess + * Parameters: + * heatmap: model inference results for keypoint detection models + * dim: shape information of the inference result + * coords: coordinates after refined positioning + * px: px = int(coords[ch * 2] + 0.5) , refer to API detection::GetFinalPredictions + * py: px = int(coords[ch * 2 + 1] + 0.5), refer to API detection::GetFinalPredictions + * index: index information of heatmap pixels + * ch: channel + * Paper reference: DARK postpocessing, Zhang et al. + * Distribution-Aware Coordinate Representation for Human Pose Estimation (CVPR + * 2020). + */ +void DarkParse(const std::vector& heatmap, const std::vector& dim, + std::vector* coords, const int px, const int py, + const int index, const int ch); + +} // namespace utils +} // namespace vision +} // namespace fastdeploy diff --git a/3rdparty/lib/ppocr.lib b/3rdparty/lib/ppocr.lib deleted file mode 100644 index e850d785c4..0000000000 Binary files a/3rdparty/lib/ppocr.lib and /dev/null differ diff --git a/3rdparty/resource/PaddleCharOCR/det/version.txt b/3rdparty/resource/PaddleCharOCR/det/version.txt index cd6b757802..89a056fb9c 100644 --- a/3rdparty/resource/PaddleCharOCR/det/version.txt +++ b/3rdparty/resource/PaddleCharOCR/det/version.txt @@ -1 +1 @@ -en_PP-OCRv3_det_slim_infer +en_PP-OCRv3_det_infer diff --git a/3rdparty/resource/PaddleCharOCR/rec/version.txt b/3rdparty/resource/PaddleCharOCR/rec/version.txt index 23a40fac52..5ef569e1f8 100644 --- a/3rdparty/resource/PaddleCharOCR/rec/version.txt +++ b/3rdparty/resource/PaddleCharOCR/rec/version.txt @@ -1 +1 @@ -en_PP-OCRv3_rec_slim_infer +en_PP-OCRv3_rec_infer diff --git a/resource/config.json b/resource/config.json index 495ad341d0..419eb8e101 100644 --- a/resource/config.json +++ b/resource/config.json @@ -30,9 +30,7 @@ "Doc": "仓库识别导出结果模板", "arkPlanner": "{\"@type\": \"@penguin-statistics/depot\",\"items\": []}", "arkPlanner_Doc": "https://penguin-stats.cn/planner" - }, - "ocrWithRawData": true, - "ocrWithRawData_Doc": "是否使用图片原始数据进行OCR，可以提高性能，但在部分设备上可能有兼容性问题" + } }, "intent": { "Official": "com.hypergryph.arknights/com.u8.sdk.U8UnityContext", diff --git a/src/MeoAssistant/Common/AsstConf.h b/src/MeoAssistant/Common/AsstConf.h index b1e755b35b..8b42d1658e 100644 --- a/src/MeoAssistant/Common/AsstConf.h +++ b/src/MeoAssistant/Common/AsstConf.h @@ -19,7 +19,7 @@ #ifdef _MSC_VER #define ASST_SUPPRESS_CV_WARNINGS_START \ ASST_DO_PRAGMA(warning(push)) \ - ASST_DO_PRAGMA(warning(disable : 5054)) + ASST_DO_PRAGMA(warning(disable : 5054 4251 4305 4275 4100)) #define ASST_SUPPRESS_CV_WARNINGS_END ASST_DO_PRAGMA(warning(pop)) #elif defined(__clang__) #define ASST_SUPPRESS_CV_WARNINGS_START \ diff --git a/src/MeoAssistant/MeoAssistant.vcxproj b/src/MeoAssistant/MeoAssistant.vcxproj index f41e9a5d5d..d2fe8ce28a 100644 --- a/src/MeoAssistant/MeoAssistant.vcxproj +++ b/src/MeoAssistant/MeoAssistant.vcxproj @@ -294,7 +294,7 @@ true true true - ppocr.lib;opencv_world453.lib;zlibstatic.lib;ws2_32.lib;%(AdditionalDependencies) + fastdeploy.lib;opencv_world453.lib;zlibstatic.lib;ws2_32.lib;%(AdditionalDependencies) RequireAdministrator @@ -347,7 +347,7 @@ true true true - ppocr.lib;opencv_world453.lib;zlibstatic.lib;ws2_32.lib;%(AdditionalDependencies) + fastdeploy.lib;opencv_world453.lib;zlibstatic.lib;ws2_32.lib;%(AdditionalDependencies) RequireAdministrator diff --git a/src/MeoAssistant/Resource/GeneralConfig.cpp b/src/MeoAssistant/Resource/GeneralConfig.cpp index d4a01d8158..5465537298 100644 --- a/src/MeoAssistant/Resource/GeneralConfig.cpp +++ b/src/MeoAssistant/Resource/GeneralConfig.cpp @@ -22,7 +22,6 @@ bool asst::GeneralConfig::parse(const json::value& json) m_options.yituliu_report.cmd_format = options_json.get("yituliuReport", "cmdFormat", std::string()); m_options.depot_export_template.ark_planner = options_json.get("depotExportTemplate", "arkPlanner", std::string()); - m_options.ocr_with_rawdata = options_json.get("ocrWithRawData", true); } for (const auto& [client_type, intent_name] : json.at("intent").as_object()) { diff --git a/src/MeoAssistant/Resource/GeneralConfig.h b/src/MeoAssistant/Resource/GeneralConfig.h index 983d15d63c..9768864701 100644 --- a/src/MeoAssistant/Resource/GeneralConfig.h +++ b/src/MeoAssistant/Resource/GeneralConfig.h @@ -46,7 +46,6 @@ namespace asst // 每次到结算界面，汇报掉落数据至企鹅物流 https://penguin-stats.cn/ DepotExportTemplate depot_export_template; // 仓库识别结果导出模板 yituliuReportCfg yituliu_report; // 一图流大数据汇报：目前只有公招功能，https://yituliu.site/maarecruitdata - bool ocr_with_rawdata = true; // 使用原始数据进行OCR识别，可以提高性能，但会带来少量兼容性问题 }; struct AdbCfg diff --git a/src/MeoAssistant/Resource/Miscellaneous/OcrPack.cpp b/src/MeoAssistant/Resource/Miscellaneous/OcrPack.cpp index 1b298a20c5..297fb92b53 100644 --- a/src/MeoAssistant/Resource/Miscellaneous/OcrPack.cpp +++ b/src/MeoAssistant/Resource/Miscellaneous/OcrPack.cpp @@ -3,7 +3,11 @@ #include #include "Utils/NoWarningCV.h" -#include +ASST_SUPPRESS_CV_WARNINGS_START +#include "fastdeploy/vision/ocr/ppocr/dbdetector.h" +#include "fastdeploy/vision/ocr/ppocr/ppocr_v3.h" +#include "fastdeploy/vision/ocr/ppocr/recognizer.h" +ASST_SUPPRESS_CV_WARNINGS_END #include "Resource/GeneralConfig.h" #include "Utils/Demangle.hpp" @@ -24,22 +28,11 @@ static std::filesystem::path prepare_paddle_dir(const std::filesystem::path& dir asst::OcrPack::OcrPack() { - Log.info("hardware_concurrency:", std::thread::hardware_concurrency()); - for (size_t i = 0; i != MaxBoxSize; ++i) { - static constexpr size_t MaxTextSize = 4096; - *(m_strs_buffer + i) = new char[MaxTextSize]; - // memset(*(m_strs_buffer + i), 0, MaxTextSize); - } + m_ocr_option = std::make_unique(); + m_ocr_option->UseOrtBackend(); } -asst::OcrPack::~OcrPack() -{ - for (size_t i = 0; i != MaxBoxSize; ++i) { - delete[] *(m_strs_buffer + i); - } - - PaddleOcrDestroy(m_ocr); -} +asst::OcrPack::~OcrPack() {} bool asst::OcrPack::load(const std::filesystem::path& path) { @@ -50,29 +43,26 @@ bool asst::OcrPack::load(const std::filesystem::path& path) return false; } - constexpr static auto DetName = "det"; - // constexpr static const char* ClsName = "cls"; - constexpr static auto RecName = "rec"; - constexpr static auto KeysName = "ppocr_keys_v1.txt"; + using namespace asst::utils::path_literals; + const auto dst_model_file = paddle_dir / "det"_p / "inference.pdmodel"_p; + const auto dst_params_file = paddle_dir / "det"_p / "inference.pdiparams"_p; + const auto rec_model_file = paddle_dir / "rec"_p / "inference.pdmodel"_p; + const auto rec_params_file = paddle_dir / "rec"_p / "inference.pdiparams"_p; + const auto rec_label_file = paddle_dir / "ppocr_keys_v1.txt"_p; - const auto dst_filename = paddle_dir / asst::utils::path(DetName); - // const std::string cls_filename = dir + ClsName; - const auto rec_filename = paddle_dir / asst::utils::path(RecName); - const auto keys_filename = paddle_dir / asst::utils::path(KeysName); - - if (m_ocr != nullptr) { - PaddleOcrDestroy(m_ocr); - } - - const auto det4paddle = asst::utils::path_to_ansi_string(dst_filename); - const auto rec4paddle = asst::utils::path_to_ansi_string(rec_filename); - const auto keys4paddle = asst::utils::path_to_ansi_string(keys_filename); - - if (det4paddle.empty() || rec4paddle.empty() || keys4paddle.empty()) { + if (!std::filesystem::exists(dst_model_file) || !std::filesystem::exists(dst_params_file) || + !std::filesystem::exists(rec_model_file) || !std::filesystem::exists(rec_params_file) || + !std::filesystem::exists(rec_label_file)) { return false; } - m_ocr = PaddleOcrCreate(det4paddle.c_str(), rec4paddle.c_str(), keys4paddle.c_str(), nullptr); + m_det = std::make_unique(asst::utils::path_to_ansi_string(dst_model_file), + asst::utils::path_to_ansi_string(dst_params_file), + *m_ocr_option); + m_rec = std::make_unique( + asst::utils::path_to_ansi_string(rec_model_file), asst::utils::path_to_ansi_string(rec_params_file), + asst::utils::path_to_ansi_string(rec_label_file), *m_ocr_option); + m_ocr = std::make_unique(m_det.get(), m_rec.get()); if (use_temp_dir) { // files can be removed after load @@ -84,87 +74,8 @@ bool asst::OcrPack::load(const std::filesystem::path& path) }).detach(); } - return m_ocr != nullptr; -} - -std::vector asst::OcrPack::recognize(const cv::Mat& image, const asst::TextRectProc& pred, - bool without_det, bool trim) -{ - size_t size = 0; - std::string class_type = utils::demangle(typeid(*this).name()); - - if (Config.get_options().ocr_with_rawdata) { - // 如果是带 ROI 的 cv::Mat, data 仍是指向完整的图片数据，仅通过内部的一些其他参数标识 ROI - // 直接取 data 拿到的不是正确的图，所以拷贝一份出来 - cv::Mat copied = image.clone(); - if (!without_det) { - Log.trace("Ocr System with RawData and", class_type); - PaddleOcrSystemWithData(m_ocr, copied.rows, copied.cols, copied.type(), copied.data, false, m_boxes_buffer, - m_strs_buffer, m_scores_buffer, &size, nullptr, nullptr); - } - else { - Log.trace("Ocr Rec with RawData and", class_type); - PaddleOcrRecWithData(m_ocr, copied.rows, copied.cols, copied.type(), copied.data, m_strs_buffer, - m_scores_buffer, &size, nullptr, nullptr); - } - } - else { - std::vector buf; - cv::imencode(".png", image, buf); - - if (!without_det) { - Log.trace("Ocr System with Encode and", class_type); - PaddleOcrSystem(m_ocr, buf.data(), buf.size(), false, m_boxes_buffer, m_strs_buffer, m_scores_buffer, &size, - nullptr, nullptr); - } - else { - Log.trace("Ocr Rec with Encode and", class_type); - PaddleOcrRec(m_ocr, buf.data(), buf.size(), m_strs_buffer, m_scores_buffer, &size, nullptr, nullptr); - } - } - - std::vector result; - std::vector raw_result; - -#ifdef ASST_DEBUG - cv::Mat draw = image.clone(); -#endif - - for (size_t i = 0; i != size; ++i) { - // the box rect like ↓ - // 0 - 1 - // 3 - 2 - Rect rect; - if (!without_det) { - int* box = m_boxes_buffer + i * 8; - int x_collect[4] = { *(box + 0), *(box + 2), *(box + 4), *(box + 6) }; - int y_collect[4] = { *(box + 1), *(box + 3), *(box + 5), *(box + 7) }; - auto [left, right] = ranges::minmax(x_collect); - auto [top, bottom] = ranges::minmax(y_collect); - rect = Rect(left, top, right - left, bottom - top); - } - std::string text(*(m_strs_buffer + i)); - float score = *(m_scores_buffer + i); - if (score > 2.0) { - score = 0; - } - - TextRect tr { score, rect, text }; -#ifdef ASST_DEBUG - cv::rectangle(draw, make_rect(rect), cv::Scalar(0, 0, 255), 2); -#endif - raw_result.emplace_back(tr); - if (trim) { - utils::string_trim(tr.text); - } - if (!pred || pred(tr)) { - result.emplace_back(std::move(tr)); - } - } - - Log.trace("OcrPack::recognize | raw:", raw_result); - Log.trace("OcrPack::recognize | proc:", result); - return result; + return m_det != nullptr && m_rec != nullptr && m_ocr != nullptr && m_det->Initialized() && m_rec->Initialized() && + m_ocr->Initialized(); } std::vector asst::OcrPack::recognize(const cv::Mat& image, const Rect& roi, @@ -185,6 +96,77 @@ std::vector asst::OcrPack::recognize(const cv::Mat& image, const return recognize(roi_img, rect_cor, without_det, trim); } +std::vector asst::OcrPack::recognize(const cv::Mat& image, const asst::TextRectProc& pred, + bool without_det, bool trim) +{ + std::string class_type = utils::demangle(typeid(*this).name()); + // 如果是带 ROI 的 cv::Mat, data 仍是指向完整的图片数据，仅通过内部的一些其他参数标识 ROI + // 直接取 data 拿到的不是正确的图，所以拷贝一份出来 + cv::Mat copied = image.clone(); + fastdeploy::vision::OCRResult ocr_result; + if (!without_det) { + LogTraceScope("Ocr System with " + class_type); + + m_ocr->Predict(&copied, &ocr_result); + } + else { + LogTraceScope("Ocr Rec with " + class_type); + + std::tuple rec_result; + m_rec->Predict(&copied, &rec_result); + + auto& [text, score] = rec_result; + ocr_result.text.emplace_back(std::move(text)); + ocr_result.rec_scores.emplace_back(score); + } + +#ifdef ASST_DEBUG + cv::Mat draw = image.clone(); +#endif + + std::vector raw_result; + std::vector proced_result; + for (size_t i = 0; i != ocr_result.text.size(); ++i) { + // the box rect like ↓ + // 0 - 1 + // 3 - 2 + Rect det_rect; + if (!without_det && i < ocr_result.boxes.size()) { + const auto& box = ocr_result.boxes.at(i); + int x_collect[] = { box[0], box[2], box[4], box[6] }; + int y_collect[] = { box[1], box[3], box[5], box[7] }; + auto [left, right] = ranges::minmax(x_collect); + auto [top, bottom] = ranges::minmax(y_collect); + det_rect = Rect(left, top, right - left, bottom - top); + } + else { + det_rect = Rect(0, 0, image.cols, image.rows); + } + std::string text = ocr_result.text.at(i); + double score = ocr_result.rec_scores.at(i); + if (score > 2.0) { + score = 0; + } + +#ifdef ASST_DEBUG + cv::rectangle(draw, make_rect(det_rect), cv::Scalar(0, 0, 255), 2); +#endif + + TextRect tr(score, det_rect, std::move(text)); + raw_result.emplace_back(tr); + if (trim) { + utils::string_trim(tr.text); + } + if (!pred || pred(tr)) { + proced_result.emplace_back(std::move(tr)); + } + } + + Log.trace("OcrPack::recognize | raw:", raw_result); + Log.trace("OcrPack::recognize | proc:", proced_result); + return proced_result; +} + #ifdef _WIN32 static std::filesystem::path prepare_paddle_dir(const std::filesystem::path& dir, bool* is_temp) @@ -235,4 +217,4 @@ static std::filesystem::path prepare_paddle_dir(const std::filesystem::path& dir } } -#endif +#endif \ No newline at end of file diff --git a/src/MeoAssistant/Resource/Miscellaneous/OcrPack.h b/src/MeoAssistant/Resource/Miscellaneous/OcrPack.h index 87325a2320..1e781db092 100644 --- a/src/MeoAssistant/Resource/Miscellaneous/OcrPack.h +++ b/src/MeoAssistant/Resource/Miscellaneous/OcrPack.h @@ -9,7 +9,22 @@ namespace cv { class Mat; } -struct paddle_ocr_t; + +namespace fastdeploy +{ + namespace vision::ocr + { + class DBDetector; + class Recognizer; + } + + namespace pipeline + { + class PPOCRv3; + } + + struct RuntimeOption; +} namespace asst { @@ -31,12 +46,10 @@ namespace asst protected: OcrPack(); - paddle_ocr_t* m_ocr = nullptr; - - // each box has 8 value ( 4 points, x and y ) - int m_boxes_buffer[MaxBoxSize * 8] = { 0 }; - char* m_strs_buffer[MaxBoxSize] = { nullptr }; - float m_scores_buffer[MaxBoxSize] = { 0 }; + std::unique_ptr m_ocr_option = nullptr; + std::unique_ptr m_det = nullptr; + std::unique_ptr m_rec = nullptr; + std::unique_ptr m_ocr = nullptr; }; class WordOcr final : public SingletonHolder, public OcrPack