chore: 更新fastdeploy

83824f7969

3rdparty/include/fastdeploy/backends/backend.h

@@ -62,8 +62,11 @@ class BaseBackend {
   virtual TensorInfo GetOutputInfo(int index) = 0;
   virtual std::vector<TensorInfo> GetInputInfos() = 0;
   virtual std::vector<TensorInfo> GetOutputInfos() = 0;
+  // if copy_to_fd is true, copy memory data to FDTensor
+  // else share memory to FDTensor(only Paddle、ORT、TRT、OpenVINO support it)
   virtual bool Infer(std::vector<FDTensor>& inputs,
-                     std::vector<FDTensor>* outputs) = 0;
+                     std::vector<FDTensor>* outputs,
+                     bool copy_to_fd = true) = 0;
   virtual std::unique_ptr<BaseBackend> Clone(void *stream = nullptr,
                                              int device_id = -1) {
     FDERROR << "Clone no support" << std::endl;

3rdparty/include/fastdeploy/backends/ort/ops/adaptive_pool2d.h

@@ -0,0 +1,89 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <map>
+#include <string>
+#include <algorithm>
+#include <cmath>
+#include "fastdeploy/core/fd_tensor.h"
+#include "fastdeploy/utils/utils.h"
+
+#ifndef NON_64_PLATFORM
+#include "onnxruntime_cxx_api.h"  // NOLINT
+
+#ifdef WITH_GPU
+#include "fastdeploy/backends/op_cuda_kernels/adaptive_pool2d_kernel.h"
+#endif
+
+namespace fastdeploy {
+struct AdaptivePool2dKernel {
+ protected:
+  std::string pooling_type_ = "avg";
+  std::vector<int64_t> output_size_ = {};
+  Ort::CustomOpApi ort_;
+  void* compute_stream_;
+  const char* provider_;
+
+ public:
+  AdaptivePool2dKernel(Ort::CustomOpApi ort,
+                        const OrtKernelInfo* info,
+                        const char* provider)
+      : ort_(ort) {
+    GetAttribute(info);
+    provider_ = provider;
+  }
+
+  void GetAttribute(const OrtKernelInfo* info);
+
+  void Compute(OrtKernelContext* context);
+
+  void CpuAdaptivePool(const std::vector<int64_t>& input_size,
+                        const std::vector<int64_t>& output_size,
+                        const float* input_data,
+                        float* output_data);
+};
+
+struct AdaptivePool2dOp
+    : Ort::CustomOpBase<AdaptivePool2dOp, AdaptivePool2dKernel> {
+  explicit AdaptivePool2dOp(const char* provider) : provider_(provider) {}
+  void* CreateKernel(Ort::CustomOpApi api, const OrtKernelInfo* info) const {
+    return new AdaptivePool2dKernel(api, info, provider_);
+  }
+
+  const char* GetName() const { return "AdaptivePool2d"; }
+
+  size_t GetInputTypeCount() const { return 1; }
+
+  ONNXTensorElementDataType GetInputType(size_t index) const {
+    return ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT;
+  }
+
+  size_t GetOutputTypeCount() const { return 1; }
+
+  ONNXTensorElementDataType GetOutputType(size_t index) const {
+    return ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT;
+  }
+
+  const char* GetExecutionProviderType() const {
+    return provider_;
+  }
+ private:
+  const char* provider_;
+};
+
+}  // namespace fastdeploy
+
+#endif

3rdparty/include/fastdeploy/backends/ort/ort_backend.h

@@ -68,7 +68,8 @@ class OrtBackend : public BaseBackend {
                     bool from_memory_buffer = false);
 
   bool Infer(std::vector<FDTensor>& inputs,
-             std::vector<FDTensor>* outputs) override;
+             std::vector<FDTensor>* outputs,
+             bool copy_to_fd = true) override;
 
   int NumInputs() const override { return inputs_desc_.size(); }
 
@@ -92,7 +93,7 @@ class OrtBackend : public BaseBackend {
   Ort::CustomOpDomain custom_op_domain_ = Ort::CustomOpDomain("Paddle");
 #endif
   OrtBackendOption option_;
-  void CopyToCpu(const Ort::Value& value, FDTensor* tensor,
-                 const std::string& name);
+  void OrtValueToFDTensor(const Ort::Value& value, FDTensor* tensor,
+                          const std::string& name, bool copy_to_fd);
 };
 }  // namespace fastdeploy

3rdparty/include/fastdeploy/backends/rknpu/rknpu2/rknpu2_backend.h

@@ -72,7 +72,8 @@ class RKNPU2Backend : public BaseBackend {
   std::vector<TensorInfo> GetInputInfos() override;
   std::vector<TensorInfo> GetOutputInfos() override;
   bool Infer(std::vector<FDTensor>& inputs,
-             std::vector<FDTensor>* outputs) override;
+             std::vector<FDTensor>* outputs,
+             bool copy_to_fd = true) override;
 
  private:
   // The object of rknn context.

3rdparty/include/fastdeploy/core/fd_scalar.h

@@ -0,0 +1,121 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#pragma once
+
+#include <cstdint>
+#include <limits>
+
+#include "fastdeploy/core/fd_type.h"
+#include "fastdeploy/core/float16.h"
+
+namespace fastdeploy {
+
+class Scalar {
+ public:
+  // Constructor support implicit
+  Scalar() : Scalar(0) {}
+  Scalar(double val) : dtype_(FDDataType::FP64) {  // NOLINT
+    data_.f64 = val;
+  }
+
+  Scalar(float val) : dtype_(FDDataType::FP32) {  // NOLINT
+    data_.f32 = val;
+  }
+
+  Scalar(float16 val) : dtype_(FDDataType::FP16) {  // NOLINT
+    data_.f16 = val;
+  }
+
+  Scalar(int64_t val) : dtype_(FDDataType::INT64) {  // NOLINT
+    data_.i64 = val;
+  }
+
+  Scalar(int32_t val) : dtype_(FDDataType::INT32) {  // NOLINT
+    data_.i32 = val;
+  }
+
+  Scalar(int16_t val) : dtype_(FDDataType::INT16) {  // NOLINT
+    data_.i16 = val;
+  }
+
+  Scalar(int8_t val) : dtype_(FDDataType::INT8) {  // NOLINT
+    data_.i8 = val;
+  }
+
+  Scalar(uint8_t val) : dtype_(FDDataType::UINT8) {  // NOLINT
+    data_.ui8 = val;
+  }
+
+  Scalar(bool val) : dtype_(FDDataType::BOOL) {  // NOLINT
+    data_.b = val;
+  }
+
+  // The compatible method for fliud operators,
+  // and it will be removed in the future.
+  explicit Scalar(const std::string& str_value) : dtype_(FDDataType::FP64) {
+    if (str_value == "inf") {
+      data_.f64 = std::numeric_limits<double>::infinity();
+    } else if (str_value == "-inf") {
+      data_.f64 = -std::numeric_limits<double>::infinity();
+    } else if (str_value == "nan") {
+      data_.f64 = std::numeric_limits<double>::quiet_NaN();
+    } else {
+      data_.f64 = std::stod(str_value);
+    }
+  }
+
+  template <typename RT> inline RT to() const {
+    switch (dtype_) {
+    case FDDataType::FP32:
+      return static_cast<RT>(data_.f32);
+    case FDDataType::FP64:
+      return static_cast<RT>(data_.f64);
+    case FDDataType::FP16:
+      return static_cast<RT>(data_.f16);
+    case FDDataType::INT32:
+      return static_cast<RT>(data_.i32);
+    case FDDataType::INT64:
+      return static_cast<RT>(data_.i64);
+    case FDDataType::INT16:
+      return static_cast<RT>(data_.i16);
+    case FDDataType::INT8:
+      return static_cast<RT>(data_.i8);
+    case FDDataType::UINT8:
+      return static_cast<RT>(data_.ui8);
+    case FDDataType::BOOL:
+      return static_cast<RT>(data_.b);
+    default:
+      FDASSERT(false, "Invalid enum scalar data type `%s`.",
+               Str(dtype_).c_str());
+    }
+  }
+
+  FDDataType dtype() const { return dtype_; }
+
+ private:
+  FDDataType dtype_;
+  union data {
+    bool b;
+    int8_t i8;
+    int16_t i16;
+    int32_t i32;
+    int64_t i64;
+    uint8_t ui8;
+    float16 f16;
+    float f32;
+    double f64;
+  } data_;
+};
+
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/core/fd_tensor.h

@@ -19,6 +19,7 @@
 #include <vector>
 
 #include "fastdeploy/core/allocate.h"
+#include "fastdeploy/core/fd_scalar.h"
 #include "fastdeploy/core/fd_type.h"
 
 namespace fastdeploy {
@@ -76,7 +77,8 @@ struct FASTDEPLOY_DECL FDTensor {
   // So take care with the user buffer
   void SetExternalData(const std::vector<int64_t>& new_shape,
                        const FDDataType& data_type, void* data_buffer,
-                       const Device& new_device = Device::CPU);
+                       const Device& new_device = Device::CPU,
+                       int new_device_id = -1);
 
   // Expand the shape of a Tensor. Insert a new axis that will appear
   // at the `axis` position in the expanded Tensor shape.
@@ -126,6 +128,8 @@ struct FASTDEPLOY_DECL FDTensor {
 
   FDTensor() {}
   explicit FDTensor(const std::string& tensor_name);
+  explicit FDTensor(const char* tensor_name);
+
   // Deep copy
   FDTensor(const FDTensor& other);
   // Move constructor
@@ -136,6 +140,9 @@ struct FASTDEPLOY_DECL FDTensor {
   // Move assignment
   FDTensor& operator=(FDTensor&& other);
 
+  // Scalar to FDTensor
+  explicit FDTensor(const Scalar& scalar);
+
   ~FDTensor() { FreeFn(); }
 
   static void CopyBuffer(void* dst, const void* src, size_t nbytes,

3rdparty/include/fastdeploy/function/cast.h

@@ -0,0 +1,31 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "fastdeploy/core/fd_tensor.h"
+
+namespace fastdeploy {
+namespace function {
+
+/** Cast x to output data type element-wise. Only for float type FDTensor
+    @param x The input tensor.
+    @param out The output tensor which stores the result.
+    @param output_dtype The type of output tensor.
+*/
+FASTDEPLOY_DECL void Cast(const FDTensor& x, FDTensor* out,
+                          FDDataType output_dtype);
+
+}  // namespace function
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/function/clip.h

@@ -0,0 +1,32 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "fastdeploy/core/fd_tensor.h"
+
+namespace fastdeploy {
+namespace function {
+
+/** This operator clip all elements in input into the range [ min, max ]. Support float32, float64, int32, int64
+    @param x The input tensor.
+    @param min The lower bound
+    @param max The uppper bound
+    @param out The output tensor which stores the result.
+*/
+FASTDEPLOY_DECL void Clip(const FDTensor& x, double min, double max,
+                          FDTensor* out);
+
+}  // namespace function
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/function/concat.h

@@ -22,7 +22,7 @@ namespace function {
 /** Excute the concatenate operation for input FDTensor along given axis.
     @param x The input tensor.
     @param out The output tensor which stores the result.
-    @param axisi Axis which will be concatenated.
+    @param axis Axis which will be concatenated.
 */
 
 FASTDEPLOY_DECL void Concat(const std::vector<FDTensor>& x, FDTensor* out,

3rdparty/include/fastdeploy/function/cumprod.h

@@ -0,0 +1,31 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "fastdeploy/core/fd_tensor.h"
+
+namespace fastdeploy {
+namespace function {
+
+/** Excute the concatenate operation for input FDTensor along given axis.
+    @param x The input tensor.
+    @param out The output tensor which stores the result.
+    @param axisi Axis which will be concatenated.
+*/
+
+FASTDEPLOY_DECL void Cumprod(const FDTensor& x, FDTensor* out, int axis = 0);
+
+}  // namespace function
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/function/elementwise.h

@@ -0,0 +1,105 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "fastdeploy/core/fd_scalar.h"
+#include "fastdeploy/core/fd_tensor.h"
+
+namespace fastdeploy {
+
+namespace function {
+
+/** Excute the add operation for input FDTensors. *out = x + y.
+    @param x The input tensor.
+    @param y The input tensor.
+    @param out The output tensor which stores the result.
+*/
+FASTDEPLOY_DECL void Add(const FDTensor& x, const FDTensor& y, FDTensor* out);
+
+/** Excute the subtract operation for input FDTensors.  *out = x - y.
+    @param x The input tensor.
+    @param y The input tensor.
+    @param out The output tensor which stores the result.
+*/
+FASTDEPLOY_DECL void Subtract(const FDTensor& x, const FDTensor& y,
+                              FDTensor* out);
+
+/** Excute the multiply operation for input FDTensors.  *out = x * y.
+    @param x The input tensor.
+    @param y The input tensor.
+    @param out The output tensor which stores the result.
+*/
+FASTDEPLOY_DECL void Multiply(const FDTensor& x, const FDTensor& y,
+                              FDTensor* out);
+
+/** Excute the divide operation for input FDTensors.  *out = x / y.
+    @param x The input tensor.
+    @param y The input tensor.
+    @param out The output tensor which stores the result.
+*/
+FASTDEPLOY_DECL void Divide(const FDTensor& x, const FDTensor& y,
+                            FDTensor* out);
+
+/** Excute the maximum operation for input FDTensors.  *out = max(x, y).
+    @param x The input tensor.
+    @param y The input tensor.
+    @param out The output tensor which stores the result.
+*/
+FASTDEPLOY_DECL void Maximum(const FDTensor& x, const FDTensor& y,
+                             FDTensor* out);
+
+}  // namespace function
+
+FASTDEPLOY_DECL FDTensor operator+(const FDTensor& x, const FDTensor& y);
+
+template <typename T> FDTensor operator+(const FDTensor& x, T y) {
+  return x + FDTensor(Scalar(y));
+}
+
+template <typename T> FDTensor operator+(T x, const FDTensor& y) {
+  return FDTensor(Scalar(x)) + y;
+}
+
+FASTDEPLOY_DECL FDTensor operator-(const FDTensor& x, const FDTensor& y);
+
+template <typename T> FDTensor operator-(const FDTensor& x, T y) {
+  return x - FDTensor(Scalar(y));
+}
+
+template <typename T> FDTensor operator-(T x, const FDTensor& y) {
+  return FDTensor(Scalar(x)) - y;
+}
+
+FASTDEPLOY_DECL FDTensor operator*(const FDTensor& x, const FDTensor& y);
+
+template <typename T> FDTensor operator*(const FDTensor& x, T y) {
+  return x * FDTensor(Scalar(y));
+}
+
+template <typename T> FDTensor operator*(T x, const FDTensor& y) {
+  return FDTensor(Scalar(x)) * y;
+}
+
+FASTDEPLOY_DECL FDTensor operator/(const FDTensor& x, const FDTensor& y);
+
+template <typename T> FDTensor operator/(const FDTensor& x, T y) {
+  return x / FDTensor(Scalar(y));
+}
+
+template <typename T> FDTensor operator/(T x, const FDTensor& y) {
+  return FDTensor(Scalar(x)) / y;
+}
+
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/function/elementwise_base.h

@@ -0,0 +1,265 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <algorithm>
+
+#include "fastdeploy/core/fd_tensor.h"
+#include "fastdeploy/function/eigen.h"
+
+namespace fastdeploy {
+namespace function {
+
+#define DEFINE_ELEMENTWISE_OP(name)                                            \
+  template <typename T> struct name##RawKernel {                               \
+    void operator()(const FDTensor& x, const FDTensor& y, int axis,            \
+                    FDTensor* out) {                                           \
+      if (x.Shape() == y.Shape()) {                                            \
+        SameDimsElementwiseCompute<SameDims##name##Functor<T>>()(x, y, out);   \
+      } else {                                                                 \
+        auto x_dims = x.Shape();                                               \
+        auto y_dims = y.Shape();                                               \
+        if (x_dims.size() >= y_dims.size()) {                                  \
+          ElementwiseCompute<name##Functor<T>, T>(x, y, axis,                  \
+                                                  name##Functor<T>(), out);    \
+        } else {                                                               \
+          ElementwiseCompute<Inverse##name##Functor<T>, T>(                    \
+              x, y, axis, Inverse##name##Functor<T>(), out);                   \
+        }                                                                      \
+      }                                                                        \
+    }                                                                          \
+  }
+
+inline void GetMidDims(const std::vector<int64_t>& x_dims,
+                       const std::vector<int64_t>& y_dims, const int axis,
+                       int* pre, int* n, int* post,
+                       int* is_run_common_broadcast) {
+  *pre = 1;
+  *n = 1;
+  *post = 1;
+  *is_run_common_broadcast = 0;
+  for (int i = 0; i < axis; ++i) {
+    (*pre) *= x_dims[i];
+  }
+  for (int i = 0; i < y_dims.size(); ++i) {
+    if (x_dims[i + axis] != y_dims[i]) {
+      FDASSERT(y_dims[i] == 1 || x_dims[i + axis] == 1,
+               "Broadcast dimension mismatch. Operands "
+               "could not be broadcast together with the shape of "
+               "X = [%s] and the shape of Y = [%s]. Received [%d] "
+               "in X is not equal to [%d] in Y.",
+               Str(x_dims).c_str(), Str(y_dims).c_str(), x_dims[i + axis],
+               y_dims[i]);
+      *is_run_common_broadcast = 1;
+      return;
+    }
+    (*n) *= y_dims[i];
+  }
+  for (int i = axis + y_dims.size(); i < x_dims.size(); ++i) {
+    (*post) *= x_dims[i];
+  }
+}
+
+inline std::vector<int64_t>
+TrimTrailingSingularDims(const std::vector<int64_t>& dims) {
+  // Remove trailing dimensions of size 1 for y
+  auto actual_dims_size = dims.size();
+  for (; actual_dims_size != 0; --actual_dims_size) {
+    if (dims[actual_dims_size - 1] != 1)
+      break;
+  }
+  if (actual_dims_size == dims.size())
+    return dims;
+  std::vector<int64_t> trim_dims;
+  trim_dims.resize(actual_dims_size);
+  for (int i = 0; i < actual_dims_size; ++i) {
+    trim_dims[i] = dims[i];
+  }
+  return trim_dims;
+}
+
+inline int GetElementwiseIndex(const int64_t* x_dims_array, const int max_dim,
+                               const int64_t* index_array) {
+  int index_ = 0;
+  for (int i = 0; i < max_dim; i++) {
+    if (x_dims_array[i] > 1) {
+      index_ = index_ * x_dims_array[i] + index_array[i];
+    }
+  }
+  return index_;
+}
+
+inline void UpdateElementwiseIndexArray(const int64_t* out_dims_array,
+                                        const int max_dim,
+                                        int64_t* index_array) {
+  for (int i = max_dim - 1; i >= 0; --i) {
+    ++index_array[i];
+    if (index_array[i] >= out_dims_array[i]) {
+      index_array[i] -= out_dims_array[i];
+    } else {
+      break;
+    }
+  }
+}
+
+inline void GetBroadcastDimsArrays(const std::vector<int64_t>& x_dims,
+                                   const std::vector<int64_t>& y_dims,
+                                   int64_t* x_dims_array, int64_t* y_dims_array,
+                                   int64_t* out_dims_array, const int max_dim,
+                                   const int axis) {
+  FDASSERT(axis >= 0,
+           "Axis should be great than or equal to 0, but received axis is %d.",
+           axis);
+  FDASSERT(axis < max_dim,
+           "Axis should be less than %d, but received axis is %d.", max_dim,
+           axis);
+  if (x_dims.size() > y_dims.size()) {
+    std::fill(y_dims_array, y_dims_array + axis, 1);
+    if (axis + y_dims.size() < max_dim) {
+      std::fill(y_dims_array + axis + y_dims.size(), y_dims_array + max_dim, 1);
+    }
+    std::copy(x_dims.data(), x_dims.data() + x_dims.size(), x_dims_array);
+    std::copy(y_dims.data(), y_dims.data() + y_dims.size(),
+              y_dims_array + axis);
+  } else {
+    std::fill(x_dims_array, x_dims_array + axis, 1);
+    if (axis + x_dims.size() < max_dim) {
+      std::fill(x_dims_array + axis + x_dims.size(), x_dims_array + max_dim, 1);
+    }
+    std::copy(x_dims.data(), x_dims.data() + x_dims.size(),
+              x_dims_array + axis);
+    std::copy(y_dims.data(), y_dims.data() + y_dims.size(), y_dims_array);
+  }
+
+  for (int i = 0; i < max_dim; i++) {
+    FDASSERT(x_dims_array[i] == y_dims_array[i] || x_dims_array[i] <= 1 ||
+                 y_dims_array[i] <= 1,
+             "Broadcast dimension mismatch. Operands "
+             "could not be broadcast together with the shape of "
+             "X = [%s] and the shape of Y = [%s]. Received [%d] "
+             "in X is not equal to [%d] in Y.",
+             Str(x_dims).c_str(), Str(y_dims).c_str(), x_dims[i + axis],
+             y_dims[i]);
+    if ((x_dims_array[i] > 1 || y_dims_array[i] > 1) ||
+        (x_dims_array[i] == 1 && y_dims_array[i] == 1)) {
+      out_dims_array[i] = (std::max)(x_dims_array[i], y_dims_array[i]);
+    } else {
+      out_dims_array[i] = -1;
+    }
+  }
+}
+
+template <typename Functor, typename T, typename OutType = T>
+void CommonForwardBroadcastCPU(const FDTensor& x, const FDTensor& y,
+                               FDTensor* z, int64_t* x_dims_array,
+                               int64_t* y_dims_array, int64_t* out_dims_array,
+                               int max_dim, Functor func,
+                               const bool is_xsize_larger = true) {
+  std::vector<int64_t> index_array(max_dim, 0);
+  const T* x_data = reinterpret_cast<const T*>(x.Data());
+  const T* y_data = reinterpret_cast<const T*>(y.Data());
+  FDASSERT(x_data != nullptr, "The input X should not be empty.");
+  FDASSERT(y_data != nullptr, "The input X should not be empty.");
+  OutType* out_data = reinterpret_cast<OutType*>(z->Data());
+
+  const int out_size = std::accumulate(out_dims_array, out_dims_array + max_dim,
+                                       1, std::multiplies<int64_t>());
+  int x_index, y_index;
+  for (int out_index = 0; out_index < out_size; ++out_index) {
+    x_index = GetElementwiseIndex(x_dims_array, max_dim, index_array.data());
+    y_index = GetElementwiseIndex(y_dims_array, max_dim, index_array.data());
+    if (is_xsize_larger) {
+      out_data[out_index] = func(x_data[x_index], y_data[y_index]);
+    } else {
+      out_data[out_index] = func(y_data[y_index], x_data[x_index]);
+    }
+
+    UpdateElementwiseIndexArray(out_dims_array, max_dim, index_array.data());
+  }
+}
+
+template <typename Functor, typename T, typename OutType = T>
+void CommonElementwiseBroadcastForward(const FDTensor& x, const FDTensor& y,
+                                       FDTensor* z,
+                                       const std::vector<int64_t>& x_dims,
+                                       const std::vector<int64_t>& y_dims,
+                                       Functor func, int axis,
+                                       const bool is_xsize_larger = true) {
+  int x_dims_size = x_dims.size();
+  int y_dims_size = y_dims.size();
+  int max_dim = (std::max)(x_dims_size, y_dims_size);
+  axis = (axis == -1 ? std::abs(x_dims_size - y_dims_size) : axis);
+  FDASSERT(axis >= 0,
+           "Axis should be great than or equal to 0, but received axis is %d.",
+           axis);
+  FDASSERT(axis < max_dim,
+           "Axis should be less than %d, but received axis is %d.", max_dim,
+           axis);
+  std::vector<int64_t> x_dims_array(max_dim);
+  std::vector<int64_t> y_dims_array(max_dim);
+  std::vector<int64_t> out_dims_array(max_dim);
+  GetBroadcastDimsArrays(x_dims, y_dims, x_dims_array.data(),
+                         y_dims_array.data(), out_dims_array.data(), max_dim,
+                         axis);
+  FDTensor tmp;
+  tmp.Allocate(out_dims_array, TypeToDataType<OutType>::dtype);
+  CommonForwardBroadcastCPU<Functor, T, OutType>(
+      x, y, &tmp, x_dims_array.data(), y_dims_array.data(),
+      out_dims_array.data(), max_dim, func, is_xsize_larger);
+  *z = std::move(tmp);
+}
+
+template <typename Functor, typename T, typename OutType = T>
+void ElementwiseCompute(const FDTensor& x, const FDTensor& y, int axis,
+                        Functor func, FDTensor* z) {
+  auto x_dims = x.Shape();
+  auto y_dims = y.Shape();
+  bool is_xsize_larger = true;
+  int max_dim = x_dims.size();
+  if (x_dims.size() < y_dims.size()) {
+    is_xsize_larger = false;
+    max_dim = y_dims.size();
+  }
+
+  int diff_size = x_dims.size() - y_dims.size();
+  axis = (axis == -1 ? std::abs(diff_size) : axis);
+  FDASSERT(axis >= 0,
+           "Axis should be great than or equal to 0, but received axis is %d.",
+           axis);
+  FDASSERT(axis < max_dim,
+           "Axis should be less than %d, but received axis is %d.", max_dim,
+           axis);
+
+  int pre, n, post, is_run_common_broadcast, axis_trim = 0;
+  if (is_xsize_larger) {
+    auto y_dims_trimed = TrimTrailingSingularDims(y_dims);
+    axis_trim = (y_dims_trimed.size() == 0) ? x_dims.size() : axis;
+    GetMidDims(x_dims, y_dims_trimed, axis_trim, &pre, &n, &post,
+               &is_run_common_broadcast);
+  } else {
+    auto x_dims_trimed = TrimTrailingSingularDims(x_dims);
+    axis_trim = (x_dims_trimed.size() == 0) ? y_dims.size() : axis;
+    GetMidDims(y_dims, x_dims_trimed, axis_trim, &pre, &n, &post,
+               &is_run_common_broadcast);
+  }
+  // special case for common implementation.
+  // case 1: x=[2,3,1,5], y=[2,1,4,1]
+  // case 2: x=[2,3,4], y=[1,1,4]
+  CommonElementwiseBroadcastForward<Functor, T, OutType>(
+      x, y, z, x_dims, y_dims, func, axis, is_xsize_larger);
+}
+
+}  // namespace function
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/function/elementwise_functor.h

@@ -0,0 +1,131 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "fastdeploy/function/eigen.h"
+#include "fastdeploy/function/elementwise.h"
+#include "fastdeploy/function/elementwise_base.h"
+#include <algorithm>
+
+namespace fastdeploy {
+namespace function {
+
+template <typename Functor> struct SameDimsElementwiseCompute {
+  void operator()(const FDTensor& x, const FDTensor& y, FDTensor* z) {
+    z->Allocate(x.Shape(), x.Dtype());
+    Functor()(x, y, z);
+  }
+};
+
+template <typename T> struct SameDimsAddFunctor {
+  void operator()(const FDTensor& x, const FDTensor& y, FDTensor* z) {
+    const auto& dev = *EigenDeviceWrapper::GetInstance()->GetDevice();
+    auto eigen_x = EigenVector<T>::Flatten(x);
+    auto eigen_y = EigenVector<T>::Flatten(y);
+    auto eigen_z = EigenVector<T>::Flatten(*z);
+    eigen_z.device(dev) = eigen_x + eigen_y;
+  }
+};
+
+template <typename T> struct SameDimsSubtractFunctor {
+  void operator()(const FDTensor& x, const FDTensor& y, FDTensor* z) {
+    const auto& dev = *EigenDeviceWrapper::GetInstance()->GetDevice();
+    auto eigen_x = EigenVector<T>::Flatten(x);
+    auto eigen_y = EigenVector<T>::Flatten(y);
+    auto eigen_z = EigenVector<T>::Flatten(*z);
+    eigen_z.device(dev) = eigen_x - eigen_y;
+  }
+};
+
+template <typename T> struct SameDimsMultiplyFunctor {
+  void operator()(const FDTensor& x, const FDTensor& y, FDTensor* z) {
+    const auto& dev = *EigenDeviceWrapper::GetInstance()->GetDevice();
+    auto eigen_x = EigenVector<T>::Flatten(x);
+    auto eigen_y = EigenVector<T>::Flatten(y);
+    auto eigen_z = EigenVector<T>::Flatten(*z);
+    eigen_z.device(dev) = eigen_x * eigen_y;
+  }
+};
+
+template <typename T> struct SameDimsDivideFunctor {
+  void operator()(const FDTensor& x, const FDTensor& y, FDTensor* z) {
+    const auto& dev = *EigenDeviceWrapper::GetInstance()->GetDevice();
+    auto eigen_x = EigenVector<T>::Flatten(x);
+    auto eigen_y = EigenVector<T>::Flatten(y);
+    auto eigen_z = EigenVector<T>::Flatten(*z);
+    eigen_z.device(dev) = eigen_x / eigen_y;
+  }
+};
+
+// Add
+template <typename T> struct AddFunctor {
+  inline T operator()(const T a, const T b) const { return a + b; }
+};
+template <typename T> struct InverseAddFunctor {
+  inline T operator()(const T a, const T b) const { return b + a; }
+};
+
+// Subtract
+template <typename T> struct SubtractFunctor {
+  inline T operator()(const T a, const T b) const { return a - b; }
+};
+template <typename T> struct InverseSubtractFunctor {
+  inline T operator()(const T a, const T b) const { return b - a; }
+};
+
+// Multiply
+template <typename T> struct MultiplyFunctor {
+  inline T operator()(const T a, const T b) const { return a * b; }
+};
+template <> struct MultiplyFunctor<bool> {
+  inline bool operator()(const bool a, const bool b) const { return a && b; }
+};
+template <typename T> struct InverseMultiplyFunctor {
+  inline T operator()(const T a, const T b) const { return b * a; }
+};
+template <> struct InverseMultiplyFunctor<bool> {
+  inline bool operator()(const bool a, const bool b) const { return b && a; }
+};
+
+// Divide
+#define DIV_ERROR_INFO                                                         \
+  "InvalidArgumentError: Integer division by zero encountered in "             \
+  "(floor) divide. Please check the input value."
+
+template <typename T, typename Enable = void> struct DivideFunctor {
+  inline T operator()(const T a, const T b) const { return a / b; }
+};
+
+template <typename T>
+struct DivideFunctor<
+    T, typename std::enable_if<std::is_integral<T>::value>::type> {
+  inline T operator()(const T a, const T b) const {
+    // For int32/int64, need to check whether the divison is zero.
+    FDASSERT(b != 0, DIV_ERROR_INFO);
+    return a / b;
+  }
+};
+
+template <typename T, typename Enable = void> struct InverseDivideFunctor {
+  inline T operator()(const T a, const T b) const { return b / a; }
+};
+
+// Maximum
+template <typename T> struct MaximumFunctor {
+  inline T operator()(const T a, const T b) const { return a > b ? a : b; }
+};
+
+}  // namespace function
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/function/full.h

@@ -0,0 +1,44 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "fastdeploy/core/fd_scalar.h"
+#include "fastdeploy/core/fd_tensor.h"
+
+namespace fastdeploy {
+namespace function {
+
+/** Fill the value to tensor
+    @param value The value to be filled in tensor
+    @param shape The shape of output tensor.
+    @param out The output tensor which stores the result.
+    @param dtype The data type of output tensor. Default to float32
+*/
+FASTDEPLOY_DECL void Full(const Scalar& value,
+                          const std::vector<int64_t>& shape, FDTensor* out,
+                          FDDataType dtype = FDDataType::FP32);
+
+/** Fill the value to tensor
+    @param x The input tensor.
+    @param value The value to be filled in tensor
+    @param out The output tensor which stores the result.
+    @param dtype The data type of output tensor. Default to float32
+*/
+FASTDEPLOY_DECL void FullLike(const FDTensor& x, const Scalar& value,
+                              FDTensor* out,
+                              FDDataType dtype = FDDataType::FP32);
+
+}  // namespace function
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/function/functions.h

@@ -0,0 +1,35 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "fastdeploy/function/cast.h"
+#include "fastdeploy/function/clip.h"
+#include "fastdeploy/function/concat.h"
+#include "fastdeploy/function/cumprod.h"
+#include "fastdeploy/function/elementwise.h"
+#include "fastdeploy/function/full.h"
+#include "fastdeploy/function/gather_scatter_along_axis.h"
+#include "fastdeploy/function/isfinite.h"
+#include "fastdeploy/function/linspace.h"
+#include "fastdeploy/function/math.h"
+#include "fastdeploy/function/pad.h"
+#include "fastdeploy/function/quantile.h"
+#include "fastdeploy/function/reduce.h"
+#include "fastdeploy/function/slice.h"
+#include "fastdeploy/function/softmax.h"
+#include "fastdeploy/function/sort.h"
+#include "fastdeploy/function/split.h"
+#include "fastdeploy/function/tile.h"
+#include "fastdeploy/function/transpose.h"

3rdparty/include/fastdeploy/function/gather_scatter_along_axis.h

@@ -0,0 +1,33 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "fastdeploy/core/fd_tensor.h"
+
+namespace fastdeploy {
+namespace function {
+
+/** Output is obtained by gathering entries of axis of x indexed by index and
+ *  concatenate them together.
+    @param x The input tensor.
+    @param index The index of a tensor to gather.
+    @param out The output tensor which stores the result.
+    @param axis Axis which will be gathered.
+*/
+void GatherAlongAxis(const FDTensor& x, const FDTensor& index, FDTensor* result,
+                     int axis);
+
+}  // namespace function
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/function/isfinite.h

@@ -0,0 +1,47 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "fastdeploy/core/fd_tensor.h"
+
+namespace fastdeploy {
+namespace function {
+
+/** Return whether every element of input tensor is NaN or not.
+    @param x The input tensor.
+    @param out The output tensor which stores the result.
+    @param dtype The output data type
+*/
+FASTDEPLOY_DECL void IsNan(const FDTensor& x, FDTensor* out,
+                           FDDataType dtype = FDDataType::BOOL);
+
+/** Return whether every element of input tensor is Inf or not.
+    @param x The input tensor.
+    @param out The output tensor which stores the result.
+    @param dtype The output data type
+*/
+FASTDEPLOY_DECL void IsInf(const FDTensor& x, FDTensor* out,
+                           FDDataType dtype = FDDataType::BOOL);
+
+/** Return whether every element of input tensor is finite or not.
+    @param x The input tensor.
+    @param out The output tensor which stores the result.
+    @param dtype The output data type
+*/
+FASTDEPLOY_DECL void IsFinite(const FDTensor& x, FDTensor* out,
+                              FDDataType dtype = FDDataType::BOOL);
+
+}  // namespace function
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/function/linspace.h

@@ -0,0 +1,33 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "fastdeploy/core/fd_tensor.h"
+
+namespace fastdeploy {
+namespace function {
+
+/** Return fixed number of evenly spaced values within a given interval.
+    @param start The input start is start variable of range.
+    @param end The input stop is start variable of range.
+    @param num The input num is given num of the sequence.
+    @param out The output tensor which stores the result.
+    @param dtype The data type of output tensor, default to float32.
+*/
+FASTDEPLOY_DECL void Linspace(double start, double end, int num, FDTensor* out,
+                              FDDataType dtype = FDDataType::FP32);
+
+}  // namespace function
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/function/math.h

@@ -0,0 +1,65 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "fastdeploy/core/fd_tensor.h"
+
+namespace fastdeploy {
+namespace function {
+
+/** Calculates the sqrt of the given input Tensor, element-wise. Only for float type FDTensor
+    @param x The input tensor.
+    @param out The output tensor which stores the result.
+*/
+FASTDEPLOY_DECL void Sqrt(const FDTensor& x, FDTensor* out);
+
+/** Calculates the natural log of the given input Tensor, element-wise. Only for float type FDTensor
+    @param x The input tensor.
+    @param out The output tensor which stores the result.
+*/
+FASTDEPLOY_DECL void Log(const FDTensor& x, FDTensor* out);
+
+/** Rounds the values in the input to the nearest integer value, element-wise. Only for float type FDTensor
+    @param x The input tensor.
+    @param out The output tensor which stores the result.
+*/
+FASTDEPLOY_DECL void Round(const FDTensor& x, FDTensor* out);
+
+/** Computes exp of x element-wise with a natural number e as the base, element-wise. Only for float type FDTensor
+    @param x The input tensor.
+    @param out The output tensor which stores the result.
+*/
+FASTDEPLOY_DECL void Exp(const FDTensor& x, FDTensor* out);
+
+/** This operator is used to perform elementwise abs for input X. Only for float type FDTensor
+    @param x The input tensor.
+    @param out The output tensor which stores the result.
+*/
+FASTDEPLOY_DECL void Abs(const FDTensor& x, FDTensor* out);
+
+/** Computes ceil of x element-wise. Only for float type FDTensor
+    @param x The input tensor.
+    @param out The output tensor which stores the result.
+*/
+FASTDEPLOY_DECL void Ceil(const FDTensor& x, FDTensor* out);
+
+/** Computes floor of x element-wise. Only for float type FDTensor
+    @param x The input tensor.
+    @param out The output tensor which stores the result.
+*/
+FASTDEPLOY_DECL void Floor(const FDTensor& x, FDTensor* out);
+
+}  // namespace function
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/function/math_functor.h

@@ -0,0 +1,81 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "fastdeploy/function/eigen.h"
+
+namespace fastdeploy {
+namespace function {
+
+// log(x) = natural logarithm of x
+template <typename T> struct LogFunctor {
+  template <typename Device, typename X, typename Out>
+  void operator()(Device d, X x, Out out) const {
+    out.device(d) = x.log();
+  }
+};
+
+// exp functor
+// exp(x) = e^x
+template <typename T> struct ExpFunctor {
+  template <typename Device, typename X, typename Out>
+  void operator()(Device d, X x, Out out) const {
+    out.device(d) = x.exp();
+  }
+};
+
+// round(x) = [x]
+template <typename T> struct RoundFunctor {
+  template <typename Device, typename X, typename Out>
+  void operator()(Device d, X x, Out out) const {
+    out.device(d) = x.round();
+  }
+};
+
+// sqrt(x) = x^(1/2)
+template <typename T> struct SqrtFunctor {
+  template <typename Device, typename X, typename Out>
+  void operator()(Device d, X x, Out out) const {
+    out.device(d) = x.sqrt();
+  }
+};
+
+// abs(x) = x if x > 0 else -x
+template <typename T> struct AbsFunctor {
+  template <typename Device, typename X, typename Out>
+  void operator()(Device d, X x, Out out) const {
+    out.device(d) =
+        x.unaryExpr([](T v) { return v > static_cast<T>(0) ? v : -v; });
+  }
+};
+
+// ceil(x) = ceiling(x)
+template <typename T> struct CeilFunctor {
+  template <typename Device, typename X, typename Out>
+  void operator()(Device d, X x, Out out) const {
+    out.device(d) = x.ceil();
+  }
+};
+
+// floor(x) = flooring(x)
+template <typename T> struct FloorFunctor {
+  template <typename Device, typename X, typename Out>
+  void operator()(Device d, X x, Out out) const {
+    out.device(d) = x.floor();
+  }
+};
+
+}  // namespace function
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/function/quantile.h

@@ -0,0 +1,34 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "fastdeploy/core/fd_tensor.h"
+
+namespace fastdeploy {
+namespace function {
+
+/** Compute the quantile of the input along the specified axis. If any values
+ ** in a reduced row are NaN, then the quantiles for that reduction will be NaN.
+    @param x The input tensor.
+    @param q The q for calculate quantile, which should be in range [0, 1].
+    @param axis The axis along which to calculate quantile. axis should be int
+                or list of int.
+    @param out The output tensor which stores the result.
+*/
+FASTDEPLOY_DECL void Quantile(const FDTensor& x, const std::vector<double>& q,
+                              const std::vector<int>& axis, FDTensor* out);
+
+}  // namespace function
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/function/slice.h

@@ -0,0 +1,44 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "fastdeploy/core/fd_tensor.h"
+
+namespace fastdeploy {
+namespace function {
+
+/** This operator produces a slice of input along multiple axes.
+    @param x The input tensor.
+    @param axes Axes that starts and ends apply to.
+    @param starts If starts is a list or tuple, the elements of it should be
+      integers or Tensors with shape [1]. If starts is an Tensor, it should
+      be an 1-D Tensor. It represents starting indices of corresponding axis
+      in axes
+    @param ends If ends is a list or tuple, the elements of it should be
+      integers or Tensors with shape [1]. If ends is an Tensor, it should
+      be an 1-D Tensor . It represents ending indices of corresponding axis
+      in axes.
+    @param out The output tensor which stores the result.
+*/
+
+FASTDEPLOY_DECL void Slice(const FDTensor& x, const std::vector<int64_t>& axes,
+                           const std::vector<int64_t>& starts,
+                           const std::vector<int64_t>& ends, FDTensor* out);
+
+FASTDEPLOY_DECL void Slice(const FDTensor& x, const std::vector<int64_t>& axes,
+                           const std::vector<int64_t>& index, FDTensor* out);
+
+}  // namespace function
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/function/sort.h

@@ -0,0 +1,47 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "fastdeploy/core/fd_tensor.h"
+
+namespace fastdeploy {
+namespace function {
+
+/**
+ * @brief Performs sorting on the input tensor along the given axis and outputs
+ *        two tensors, Output(Out) and Output(Indices). They reserve the same
+ *        shape with Input(X), and Output(Out) represents the sorted tensor
+ *        while Output(Indices) gives the sorted order along the given axis
+ *        Attr(axis).
+ * @param  x            The input of sort
+ * @param  out          The sorted tensor of sort op, with the same shape as
+ *                      x
+ * @param  indices      The indices of a tensor giving the sorted order, with
+ *                      the same shape as x
+ * @param  axis         The axis along which to sort the tensor.
+ *                      When axis < 0, the actual axis will be the |axis|'th
+ *                      counting backwards
+ * @param  descending   The descending attribute is a flag to tell
+ *                      algorithm how to sort the input data.
+ *                      If descending is true, will sort by descending order,
+ *                      else if false, sort by ascending order
+ * @param  indices_type The data type of indices, default to int64
+ */
+FASTDEPLOY_DECL void Sort(const FDTensor& x, FDTensor* out, FDTensor* indices,
+                          int axis = 0, bool descending = false,
+                          FDDataType indices_type = FDDataType::INT64);
+
+}  // namespace function
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/function/split.h

@@ -0,0 +1,36 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "fastdeploy/core/fd_tensor.h"
+
+namespace fastdeploy {
+namespace function {
+
+/** Split the input tensor into multiple sub-Tensors.
+    @param x The input tensor.
+    @param num_or_sections f num_or_sections is an int, then num_or_sections
+           indicates the number of equal sized sub-Tensors that the x will
+           be divided into.
+    @param out The output vector tensor which stores the result.
+    @param axis Axis which will be splitted.
+*/
+
+FASTDEPLOY_DECL void Split(const FDTensor& x,
+                           const std::vector<int>& num_or_sections,
+                           std::vector<FDTensor>* out, int axis = 0);
+
+}  // namespace function
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/function/tile.h

@@ -0,0 +1,36 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "fastdeploy/core/fd_tensor.h"
+
+namespace fastdeploy {
+namespace function {
+
+/** Construct a new Tensor by repeating x the number of times given by
+ ** repeat_times. After tiling, the value of the i’th dimension of the
+ ** output is equal to x.shape[i]*repeat_times[i]. Both the number of
+ ** dimensions of x and the number of elements in repeat_times should
+ ** be less than or equal to 6.Support all data types.
+    @param x The input tensor.
+    @param repeat_times The lower bound
+    @param out The output tensor which stores the result.
+*/
+FASTDEPLOY_DECL void Tile(const FDTensor& x,
+                          const std::vector<int64_t>& repeat_times,
+                          FDTensor* out);
+
+}  // namespace function
+}  // namespace fastdeploy

3rdparty/include/fastdeploy/runtime.h

@@ -405,6 +405,12 @@ struct FASTDEPLOY_DECL Runtime {
   bool Infer(std::vector<FDTensor>& input_tensors,
              std::vector<FDTensor>* output_tensors);
 
+  /** \brief No params inference the model.
+   *
+   *  the input and output data need to pass through the BindInputTensor and GetOutputTensor interfaces.
+   */
+  bool Infer();
+
   /** \brief Compile TorchScript Module, only for Poros backend
    *
    * \param[in] prewarm_tensors Prewarm datas for compile
@@ -432,6 +438,12 @@ struct FASTDEPLOY_DECL Runtime {
   /** \brief Get all the output information
    */
   std::vector<TensorInfo> GetOutputInfos();
+  /** \brief Bind FDTensor by name, no copy and share input memory
+   */
+  void BindInputTensor(const std::string& name, FDTensor& input);
+  /** \brief Get output FDTensor by name, no copy and share backend output memory
+   */
+  FDTensor* GetOutputTensor(const std::string& name);
 
   /** \brief Clone new Runtime when multiple instances of the same model are created
    *
@@ -451,5 +463,7 @@ struct FASTDEPLOY_DECL Runtime {
   void CreateLiteBackend();
   void CreateRKNPU2Backend();
   std::unique_ptr<BaseBackend> backend_;
+  std::vector<FDTensor> input_tensors_;
+  std::vector<FDTensor> output_tensors_;
 };
 }  // namespace fastdeploy

3rdparty/include/fastdeploy/utils/utils.h

@@ -141,24 +141,26 @@ FASTDEPLOY_DECL bool ReadBinaryFromFile(const std::string& file,
     }                                                                          \
   }()
 
-#define FD_VISIT_INT_FLOAT_TYPES(TYPE, NAME, ...)                              \
-  [&] {                                                                        \
-    const auto& __dtype__ = TYPE;                                              \
-    switch (__dtype__) {                                                       \
-      FD_PRIVATE_CASE_TYPE(NAME, ::fastdeploy::FDDataType::INT32, int32_t,     \
-                           __VA_ARGS__)                                        \
-      FD_PRIVATE_CASE_TYPE(NAME, ::fastdeploy::FDDataType::INT64, int64_t,     \
-                           __VA_ARGS__)                                        \
-      FD_PRIVATE_CASE_TYPE(NAME, ::fastdeploy::FDDataType::FP32, float,        \
-                           __VA_ARGS__)                                        \
-      FD_PRIVATE_CASE_TYPE(NAME, ::fastdeploy::FDDataType::FP64, double,       \
-                           __VA_ARGS__)                                        \
-    default:                                                                   \
-      FDASSERT(false,                                                          \
-               "Invalid enum data type. Expect to accept data type INT32, "    \
-               "INT64, FP32, FP64, but receive type %s.",                      \
-               Str(__dtype__).c_str());                                        \
-    }                                                                          \
+#define FD_VISIT_INT_FLOAT_TYPES(TYPE, NAME, ...)                             \
+  [&] {                                                                       \
+    const auto& __dtype__ = TYPE;                                             \
+    switch (__dtype__) {                                                      \
+      FD_PRIVATE_CASE_TYPE(NAME, ::fastdeploy::FDDataType::INT32, int32_t,    \
+                           __VA_ARGS__)                                       \
+      FD_PRIVATE_CASE_TYPE(NAME, ::fastdeploy::FDDataType::INT64, int64_t,    \
+                           __VA_ARGS__)                                       \
+      FD_PRIVATE_CASE_TYPE(NAME, ::fastdeploy::FDDataType::FP32, float,       \
+                           __VA_ARGS__)                                       \
+      FD_PRIVATE_CASE_TYPE(NAME, ::fastdeploy::FDDataType::FP64, double,      \
+                           __VA_ARGS__)                                       \
+      FD_PRIVATE_CASE_TYPE(NAME, ::fastdeploy::FDDataType::UINT8, uint8_t,    \
+                           __VA_ARGS__)                                       \
+      default:                                                                \
+        FDASSERT(false,                                                       \
+                 "Invalid enum data type. Expect to accept data type INT32, " \
+                 "INT64, FP32, FP64, UINT8 but receive type %s.",             \
+                 Str(__dtype__).c_str());                                     \
+    }                                                                         \
   }()
 
 #define FD_VISIT_FLOAT_TYPES(TYPE, NAME, ...)                                  \
@@ -177,20 +179,22 @@ FASTDEPLOY_DECL bool ReadBinaryFromFile(const std::string& file,
     }                                                                          \
   }()
 
-#define FD_VISIT_INT_TYPES(TYPE, NAME, ...)                                    \
-  [&] {                                                                        \
-    const auto& __dtype__ = TYPE;                                              \
-    switch (__dtype__) {                                                       \
-      FD_PRIVATE_CASE_TYPE(NAME, ::fastdeploy::FDDataType::INT32, int32_t,     \
-                           __VA_ARGS__)                                        \
-      FD_PRIVATE_CASE_TYPE(NAME, ::fastdeploy::FDDataType::INT64, int64_t,     \
-                           __VA_ARGS__)                                        \
-    default:                                                                   \
-      FDASSERT(false,                                                          \
-               "Invalid enum data type. Expect to accept data type INT32, "    \
-               "INT64, but receive type %s.",                                  \
-               Str(__dtype__).c_str());                                        \
-    }                                                                          \
+#define FD_VISIT_INT_TYPES(TYPE, NAME, ...)                                   \
+  [&] {                                                                       \
+    const auto& __dtype__ = TYPE;                                             \
+    switch (__dtype__) {                                                      \
+      FD_PRIVATE_CASE_TYPE(NAME, ::fastdeploy::FDDataType::INT32, int32_t,    \
+                           __VA_ARGS__)                                       \
+      FD_PRIVATE_CASE_TYPE(NAME, ::fastdeploy::FDDataType::INT64, int64_t,    \
+                           __VA_ARGS__)                                       \
+      FD_PRIVATE_CASE_TYPE(NAME, ::fastdeploy::FDDataType::UINT8, uint8_t,    \
+                           __VA_ARGS__)                                       \
+      default:                                                                \
+        FDASSERT(false,                                                       \
+                 "Invalid enum data type. Expect to accept data type INT32, " \
+                 "INT64, UINT8 but receive type %s.",                         \
+                 Str(__dtype__).c_str());                                     \
+    }                                                                         \
   }()
 
 FASTDEPLOY_DECL std::vector<int64_t>

3rdparty/include/fastdeploy/vision/common/processors/mat.h

@@ -22,8 +22,8 @@ namespace vision {
 
 enum Layout { HWC, CHW };
 
-
 struct FASTDEPLOY_DECL Mat {
+  Mat() = default;
   explicit Mat(const cv::Mat& mat) {
     cpu_mat = mat;
     layout = Layout::HWC;
@@ -45,8 +45,12 @@ struct FASTDEPLOY_DECL Mat {
 #endif
 
   Mat(const Mat& mat) = default;
+  // Move assignment
   Mat& operator=(const Mat& mat) = default;
 
+  // Move constructor
+  Mat(Mat&& other) = default;
+
   // Careful if you use this interface
   // this only used if you don't want to write
   // the original data, and write to a new cv::Mat

3rdparty/include/fastdeploy/vision/common/result.h

@@ -247,6 +247,7 @@ struct FASTDEPLOY_DECL FaceAlignmentResult : public BaseResult {
 /*! @brief Segmentation result structure for all the segmentation models
  */
 struct FASTDEPLOY_DECL SegmentationResult : public BaseResult {
+  SegmentationResult() = default;
   /** \brief
    * `label_map` stores the pixel-level category labels for input image. the number of pixels is equal to label_map.size()
   */
@@ -257,12 +258,21 @@ struct FASTDEPLOY_DECL SegmentationResult : public BaseResult {
   std::vector<float> score_map;
   /// The output shape, means [H, W]
   std::vector<int64_t> shape;
+  /// SegmentationResult whether containing score_map
   bool contain_score_map = false;
 
+  /// Copy constructor
+  SegmentationResult(const SegmentationResult& other) = default;
+  /// Move assignment
+  SegmentationResult& operator=(SegmentationResult&& other);
+
   ResultType type = ResultType::SEGMENTATION;
-  /// Clear detection result
+  /// Clear Segmentation result
   void Clear();
 
+  /// Clear Segmentation result and free the memory
+  void Free();
+
   void Reserve(int size);
 
   void Resize(int size);

3rdparty/include/fastdeploy/vision/ocr/ppocr/classifier.h

@@ -43,7 +43,7 @@ class FASTDEPLOY_DECL Classifier : public FastDeployModel {
              const ModelFormat& model_format = ModelFormat::PADDLE);
   /// Get model's name
   std::string ModelName() const { return "ppocr/ocr_cls"; }
-
+  virtual bool Predict(const cv::Mat& img, int32_t* cls_label, float* cls_score);
   /** \brief BatchPredict the input image and get OCR classification model cls_result.
    *
    * \param[in] images The list of input image data, comes from cv::imread(), is a 3-D array with layout HWC, BGR format.
@@ -53,6 +53,10 @@ class FASTDEPLOY_DECL Classifier : public FastDeployModel {
   virtual bool BatchPredict(const std::vector<cv::Mat>& images,
                             std::vector<int32_t>* cls_labels,
                             std::vector<float>* cls_scores);
+  virtual bool BatchPredict(const std::vector<cv::Mat>& images,
+                            std::vector<int32_t>* cls_labels,
+                            std::vector<float>* cls_scores,
+                            size_t start_index, size_t end_index);
 
   ClassifierPreprocessor preprocessor_;
   ClassifierPostprocessor postprocessor_;

3rdparty/include/fastdeploy/vision/ocr/ppocr/cls_postprocessor.h

@@ -25,11 +25,6 @@ namespace ocr {
  */
 class FASTDEPLOY_DECL ClassifierPostprocessor {
  public:
-  /** \brief Create a postprocessor instance for Classifier serials model
-   *
-   */
-  ClassifierPostprocessor();
-
   /** \brief Process the result of runtime and fill to ClassifyResult structure
    *
    * \param[in] tensors The inference result from runtime
@@ -40,10 +35,11 @@ class FASTDEPLOY_DECL ClassifierPostprocessor {
   bool Run(const std::vector<FDTensor>& tensors,
           std::vector<int32_t>* cls_labels, std::vector<float>* cls_scores);
 
-  float cls_thresh_ = 0.9;
+  bool Run(const std::vector<FDTensor>& tensors,
+          std::vector<int32_t>* cls_labels, std::vector<float>* cls_scores,
+          size_t start_index, size_t total_size);
 
- private:
-  bool initialized_ = false;
+  float cls_thresh_ = 0.9;
 };
 
 }  // namespace ocr

3rdparty/include/fastdeploy/vision/ocr/ppocr/cls_preprocessor.h

@@ -24,11 +24,6 @@ namespace ocr {
  */
 class FASTDEPLOY_DECL ClassifierPreprocessor {
  public:
-  /** \brief Create a preprocessor instance for Classifier serials model
-   *
-   */
-  ClassifierPreprocessor();
-
   /** \brief Process the input image and prepare input tensors for runtime
    *
    * \param[in] images The input image data list, all the elements are returned by cv::imread()
@@ -36,14 +31,13 @@ class FASTDEPLOY_DECL ClassifierPreprocessor {
    * \return true if the preprocess successed, otherwise false
    */
   bool Run(std::vector<FDMat>* images, std::vector<FDTensor>* outputs);
+  bool Run(std::vector<FDMat>* images, std::vector<FDTensor>* outputs,
+           size_t start_index, size_t end_index);
 
   std::vector<float> mean_ = {0.5f, 0.5f, 0.5f};
   std::vector<float> scale_ = {0.5f, 0.5f, 0.5f};
   bool is_scale_ = true;
   std::vector<int> cls_image_shape_ = {3, 48, 192};
-
- private:
-  bool initialized_ = false;
 };
 
 }  // namespace ocr

3rdparty/include/fastdeploy/vision/ocr/ppocr/dbdetector.h

@@ -44,14 +44,6 @@ class FASTDEPLOY_DECL DBDetector : public FastDeployModel {
              const ModelFormat& model_format = ModelFormat::PADDLE);
   /// Get model's name
   std::string ModelName() const { return "ppocr/ocr_det"; }
-  /** \brief Predict the input image and get OCR detection model result.
-   *
-   * \param[in] img The input image data, comes from cv::imread(), is a 3-D array with layout HWC, BGR format.
-   * \param[in] boxes_result The output of OCR detection model result will be writen to this structure.
-   * \return true if the prediction is successed, otherwise false.
-   */
-  virtual bool Predict(cv::Mat* img,
-                       std::vector<std::array<int, 8>>* boxes_result);
    /** \brief Predict the input image and get OCR detection model result.
    *
    * \param[in] img The input image data, comes from cv::imread(), is a 3-D array with layout HWC, BGR format.

3rdparty/include/fastdeploy/vision/ocr/ppocr/det_postprocessor.h

@@ -25,11 +25,6 @@ namespace ocr {
  */
 class FASTDEPLOY_DECL DBDetectorPostprocessor {
  public:
-  /** \brief Create a postprocessor instance for DBDetector serials model
-   *
-   */
-  DBDetectorPostprocessor();
-
   /** \brief Process the result of runtime and fill to results structure
    *
    * \param[in] tensors The inference result from runtime
@@ -48,8 +43,7 @@ class FASTDEPLOY_DECL DBDetectorPostprocessor {
   bool use_dilation_ = false;
 
  private:
-  bool initialized_ = false;
-  PostProcessor post_processor_;
+  PostProcessor util_post_processor_;
   bool SingleBatchPostprocessor(const float* out_data,
                                 int n2,
                                 int n3,

3rdparty/include/fastdeploy/vision/ocr/ppocr/det_preprocessor.h

@@ -24,11 +24,6 @@ namespace ocr {
  */
 class FASTDEPLOY_DECL DBDetectorPreprocessor {
  public:
-  /** \brief Create a preprocessor instance for DBDetector serials model
-   *
-   */
-  DBDetectorPreprocessor();
-
   /** \brief Process the input image and prepare input tensors for runtime
    *
    * \param[in] images The input image data list, all the elements are returned by cv::imread()
@@ -44,9 +39,6 @@ class FASTDEPLOY_DECL DBDetectorPreprocessor {
   std::vector<float> mean_ = {0.485f, 0.456f, 0.406f};
   std::vector<float> scale_ = {0.229f, 0.224f, 0.225f};
   bool is_scale_ = true;
-
- private:
-  bool initialized_ = false;
 };
 
 }  // namespace ocr

3rdparty/include/fastdeploy/vision/ocr/ppocr/ppocr_v2.h

@@ -59,6 +59,7 @@ class FASTDEPLOY_DECL PPOCRv2 : public FastDeployModel {
    * \return true if the prediction successed, otherwise false.
    */
   virtual bool Predict(cv::Mat* img, fastdeploy::vision::OCRResult* result);
+  virtual bool Predict(const cv::Mat& img, fastdeploy::vision::OCRResult* result);
   /** \brief BatchPredict the input image and get OCR result.
    *
    * \param[in] images The list of input image data, comes from cv::imread(), is a 3-D array with layout HWC, BGR format.
@@ -68,11 +69,19 @@ class FASTDEPLOY_DECL PPOCRv2 : public FastDeployModel {
   virtual bool BatchPredict(const std::vector<cv::Mat>& images,
                std::vector<fastdeploy::vision::OCRResult>* batch_result);
   bool Initialized() const override;
+  bool SetClsBatchSize(int cls_batch_size);
+  int GetClsBatchSize();
+  bool SetRecBatchSize(int rec_batch_size);
+  int GetRecBatchSize();
 
  protected:
   fastdeploy::vision::ocr::DBDetector* detector_ = nullptr;
   fastdeploy::vision::ocr::Classifier* classifier_ = nullptr;
   fastdeploy::vision::ocr::Recognizer* recognizer_ = nullptr;
+
+ private:
+  int cls_batch_size_ = 1;
+  int rec_batch_size_ = 6;
   /// Launch the detection process in OCR.
 };
 

3rdparty/include/fastdeploy/vision/ocr/ppocr/rec_postprocessor.h

@@ -32,7 +32,7 @@ class FASTDEPLOY_DECL RecognizerPostprocessor {
    */
   explicit RecognizerPostprocessor(const std::string& label_path);
 
-  /** \brief Process the result of runtime and fill to ClassifyResult structure
+  /** \brief Process the result of runtime and fill to RecognizerResult
    *
    * \param[in] tensors The inference result from runtime
    * \param[in] texts The output result of recognizer
@@ -42,6 +42,11 @@ class FASTDEPLOY_DECL RecognizerPostprocessor {
   bool Run(const std::vector<FDTensor>& tensors,
            std::vector<std::string>* texts, std::vector<float>* rec_scores);
 
+  bool Run(const std::vector<FDTensor>& tensors,
+           std::vector<std::string>* texts, std::vector<float>* rec_scores,
+           size_t start_index, size_t total_size,
+           const std::vector<int>& indices);
+
  private:
   bool SingleBatchPostprocessor(const float* out_data,
                               const std::vector<int64_t>& output_shape,

3rdparty/include/fastdeploy/vision/ocr/ppocr/rec_preprocessor.h

@@ -24,12 +24,6 @@ namespace ocr {
  */
 class FASTDEPLOY_DECL RecognizerPreprocessor {
  public:
-  /** \brief Create a preprocessor instance for PaddleClas serials model
-   *
-   * \param[in] config_file Path of configuration file for deployment, e.g resnet/infer_cfg.yml
-   */
-  RecognizerPreprocessor();
-
   /** \brief Process the input image and prepare input tensors for runtime
    *
    * \param[in] images The input image data list, all the elements are returned by cv::imread()
@@ -37,14 +31,14 @@ class FASTDEPLOY_DECL RecognizerPreprocessor {
    * \return true if the preprocess successed, otherwise false
    */
   bool Run(std::vector<FDMat>* images, std::vector<FDTensor>* outputs);
+  bool Run(std::vector<FDMat>* images, std::vector<FDTensor>* outputs,
+           size_t start_index, size_t end_index,
+           const std::vector<int>& indices);
 
   std::vector<int> rec_image_shape_ = {3, 48, 320};
   std::vector<float> mean_ = {0.5f, 0.5f, 0.5f};
   std::vector<float> scale_ = {0.5f, 0.5f, 0.5f};
   bool is_scale_ = true;
-
- private:
-  bool initialized_ = false;
 };
 
 }  // namespace ocr

3rdparty/include/fastdeploy/vision/ocr/ppocr/recognizer.h

@@ -45,6 +45,7 @@ class FASTDEPLOY_DECL Recognizer : public FastDeployModel {
              const ModelFormat& model_format = ModelFormat::PADDLE);
   /// Get model's name
   std::string ModelName() const { return "ppocr/ocr_rec"; }
+  virtual bool Predict(const cv::Mat& img, std::string* text, float* rec_score);
   /** \brief BatchPredict the input image and get OCR recognition model result.
    *
    * \param[in] images The list of input image data, comes from cv::imread(), is a 3-D array with layout HWC, BGR format.
@@ -53,6 +54,10 @@ class FASTDEPLOY_DECL Recognizer : public FastDeployModel {
    */
   virtual bool BatchPredict(const std::vector<cv::Mat>& images,
                std::vector<std::string>* texts, std::vector<float>* rec_scores);
+  virtual bool BatchPredict(const std::vector<cv::Mat>& images,
+               std::vector<std::string>* texts, std::vector<float>* rec_scores,
+               size_t start_index, size_t end_index,
+               const std::vector<int>& indices);
 
   RecognizerPreprocessor preprocessor_;
   RecognizerPostprocessor postprocessor_;

3rdparty/include/fastdeploy/vision/ocr/ppocr/utils/ocr_utils.h

@@ -33,6 +33,8 @@ FASTDEPLOY_DECL cv::Mat GetRotateCropImage(const cv::Mat& srcimage,
 
 FASTDEPLOY_DECL void SortBoxes(std::vector<std::array<int, 8>>* boxes);
 
+FASTDEPLOY_DECL std::vector<int> ArgSort(const std::vector<float> &array);
+
 }  // namespace ocr
 }  // namespace vision
 }  // namespace fastdeploy

src/MaaCore/Common/AsstConf.h

@@ -19,7 +19,7 @@
 #ifdef _MSC_VER
 #define ASST_SUPPRESS_CV_WARNINGS_START \
     ASST_DO_PRAGMA(warning(push))       \
-    ASST_DO_PRAGMA(warning(disable : 5054 4251 4305 4275 4100))
+    ASST_DO_PRAGMA(warning(disable : 5054 4251 4305 4275 4100 4244))
 #define ASST_SUPPRESS_CV_WARNINGS_END ASST_DO_PRAGMA(warning(pop))
 #elif defined(__clang__)
 #define ASST_SUPPRESS_CV_WARNINGS_START                                               \

src/MaaCore/Config/Miscellaneous/OcrPack.cpp

@@ -107,16 +107,15 @@ std::vector<asst::TextRect> asst::OcrPack::recognize(const cv::Mat& image, const
     fastdeploy::vision::OCRResult ocr_result;
     if (!without_det) {
         LogTraceScope("Ocr Pipeline with " + class_type);
-        cv::Mat copied = image;
-        m_ocr->Predict(&copied, &ocr_result);
+        m_ocr->Predict(image, &ocr_result);
     }
     else {
         LogTraceScope("Ocr Rec with " + class_type);
-        std::vector<std::string> rec_texts;
-        std::vector<float> rec_scores;
-        m_rec->BatchPredict({ image }, &rec_texts, &rec_scores);
-        ocr_result.text = std::move(rec_texts);
-        ocr_result.rec_scores = std::move(rec_scores);
+        std::string rec_text;
+        float rec_score = 0;
+        m_rec->Predict(image, &rec_text, &rec_score);
+        ocr_result.text.emplace_back(std::move(rec_text));
+        ocr_result.rec_scores.emplace_back(rec_score);
     }
 
 #ifdef ASST_DEBUG