[Feature](bangc-ops): add Sybn op.

bangc-ops/kernels/kernel_wrapper/wrapper.h

@@ -178,6 +178,73 @@
     const mluOpTensorDescriptor_t,                                            \
     void *
 
+#define SYNCBATCHNORMSTATS_PARAM_TYPE                                       \
+  mluOpHandle_t, const mluOpTensorDescriptor_t, const void *, const float,  \
+      const mluOpTensorDescriptor_t, void *, const mluOpTensorDescriptor_t, \
+      void *
+
+#define SYNCBATCHNORMSTATS_V2_PARAM_TYPE                                      \
+  mluOpHandle_t, const mluOpTensorDescriptor_t, const void *, void *, size_t, \
+      const float, const mluOpTensorDescriptor_t, void *,                     \
+      const mluOpTensorDescriptor_t, void *
+
+#define SYNCBATCHNORMGATHERSTATSWITHCOUNTS_PARAM_TYPE                       \
+  mluOpHandle_t, const mluOpTensorDescriptor_t, const void *,               \
+      const mluOpTensorDescriptor_t, const void *,                          \
+      const mluOpTensorDescriptor_t, void *, const mluOpTensorDescriptor_t, \
+      void *, float, float, const mluOpTensorDescriptor_t, const void *,    \
+      const mluOpTensorDescriptor_t, void *, const mluOpTensorDescriptor_t, \
+      void *
+
+#define SYNCBATCHNORMELEMT_PARAM_TYPE                         \
+  mluOpHandle_t, const mluOpTensorDescriptor_t, const void *, \
+      const mluOpTensorDescriptor_t, const void *,            \
+      const mluOpTensorDescriptor_t, const void *,            \
+      const mluOpTensorDescriptor_t, const void *,            \
+      const mluOpTensorDescriptor_t, const void *,            \
+      const mluOpTensorDescriptor_t, void *
+
+#define SYNCBATCHNORMBACKWADREDUCE_PARAM_TYPE                               \
+  mluOpHandle_t, const mluOpTensorDescriptor_t, const void *,               \
+      const mluOpTensorDescriptor_t, const void *,                          \
+      const mluOpTensorDescriptor_t, const void *,                          \
+      const mluOpTensorDescriptor_t, const void *,                          \
+      const mluOpTensorDescriptor_t, void *, const mluOpTensorDescriptor_t, \
+      void *, const mluOpTensorDescriptor_t, void *,                        \
+      const mluOpTensorDescriptor_t, void *, const bool, const bool,        \
+      const bool
+
+#define SYNCBATCHNORMBACKWADREDUCE_V2_PARAM_TYPE                            \
+  mluOpHandle_t, const mluOpTensorDescriptor_t, const void *,               \
+      const mluOpTensorDescriptor_t, const void *,                          \
+      const mluOpTensorDescriptor_t, const void *,                          \
+      const mluOpTensorDescriptor_t, const void *, void *, size_t,          \
+      const mluOpTensorDescriptor_t, void *, const mluOpTensorDescriptor_t, \
+      void *, const mluOpTensorDescriptor_t, void *,                        \
+      const mluOpTensorDescriptor_t, void *, const bool, const bool,        \
+      const bool
+
+#define SYNCBATCHNORMBACKWARDELEMT_PARAM_TYPE                 \
+  mluOpHandle_t, const mluOpTensorDescriptor_t, const void *, \
+      const mluOpTensorDescriptor_t, const void *,            \
+      const mluOpTensorDescriptor_t, const void *,            \
+      const mluOpTensorDescriptor_t, const void *,            \
+      const mluOpTensorDescriptor_t, const void *,            \
+      const mluOpTensorDescriptor_t, const void *,            \
+      const mluOpTensorDescriptor_t, const void *,            \
+      const mluOpTensorDescriptor_t, void *
+
+#define SYNCBATCHNORMBACKWARDELEMT_V2_PARAM_TYPE                 \
+  mluOpHandle_t, const mluOpTensorDescriptor_t, const void *,    \
+    const mluOpTensorDescriptor_t,                               \
+      const void *, const mluOpTensorDescriptor_t, const void *, \
+      const mluOpTensorDescriptor_t, const void *,               \
+      const mluOpTensorDescriptor_t, const void *,               \
+      const mluOpTensorDescriptor_t, const void *,               \
+      const mluOpTensorDescriptor_t, const void *,               \
+      const mluOpTensorDescriptor_t, const void *,               \
+      const mluOpTensorDescriptor_t, void *diff_x
+
 /* Kernel register */
 KERNEL_REGISTER(addN, ADDN_PARAM_TYPE);
 KERNEL_REGISTER(addNV2, ADDNV2_PARAM_TYPE);
@@ -203,4 +270,17 @@ KERNEL_REGISTER(RoiAlignBackward, ROIALIGNBACKWARD_PARAM_TYPE);
 KERNEL_REGISTER(RoiAlignBackwardV2, ROIALIGNBACKWARD_V2_PARAM_TYPE);
 KERNEL_REGISTER(RoiPoolingForward, ROIPOOLINGFORWARD_PARAM_TYPE);
 KERNEL_REGISTER(RoiPoolingBackward, ROIPOOLINGBACKWARD_PARAM_TYPE);
+KERNEL_REGISTER(SyncBatchNormStats, SYNCBATCHNORMSTATS_PARAM_TYPE);
+KERNEL_REGISTER(SyncBatchNormStatsV2, SYNCBATCHNORMSTATS_V2_PARAM_TYPE);
+KERNEL_REGISTER(SyncBatchNormGatherStatsWithCounts,
+                SYNCBATCHNORMGATHERSTATSWITHCOUNTS_PARAM_TYPE);
+KERNEL_REGISTER(SyncBatchNormElemt, SYNCBATCHNORMELEMT_PARAM_TYPE);
+KERNEL_REGISTER(SyncBatchnormBackwardReduce,
+        SYNCBATCHNORMBACKWADREDUCE_PARAM_TYPE);
+KERNEL_REGISTER(SyncBatchnormBackwardReduceV2,
+        SYNCBATCHNORMBACKWADREDUCE_V2_PARAM_TYPE);
+KERNEL_REGISTER(SyncBatchNormBackwardElemt,
+        SYNCBATCHNORMBACKWARDELEMT_PARAM_TYPE);
+KERNEL_REGISTER(SyncBatchNormBackwardElemtV2,
+        SYNCBATCHNORMBACKWARDELEMT_V2_PARAM_TYPE);
 #endif  // KERNELS_KERNEL_WRAPPER_WRAPPER_H

bangc-ops/kernels/sync_batch_norm_backward_elemt/sync_batch_norm_backward_elemt.cpp

@@ -0,0 +1,49 @@
+/*************************************************************************
+ * Copyright (C) [2023] by Cambricon, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *************************************************************************/
+#include "kernels/kernel_wrapper/wrapper.h"
+
+mluOpStatus_t MLUOP_WIN_API mluOpSyncBatchNormBackwardElemt(
+    mluOpHandle_t handle,
+    const mluOpTensorDescriptor_t diff_y_desc,
+    const void *diff_y,
+    const mluOpTensorDescriptor_t x_desc,
+    const void *x,
+    const mluOpTensorDescriptor_t mean_desc,
+    const void *mean,
+    const mluOpTensorDescriptor_t invstd_desc,
+    const void *invstd,
+    const mluOpTensorDescriptor_t filter_desc,
+    const void *filter,
+    const mluOpTensorDescriptor_t mean_dy_desc,
+    const void *mean_dy,
+    const mluOpTensorDescriptor_t mean_dy_xmu_desc,
+    const void *mean_dy_xmu,
+    const mluOpTensorDescriptor_t diff_x_desc,
+    void *diff_x) {
+  SyncBatchNormBackwardElemtWrapper wrapper;
+  mluOpStatus_t ret = wrapper.invoke(handle, diff_y_desc, diff_y, x_desc,
+        x, mean_desc, mean, invstd_desc, invstd, filter_desc, filter,
+        mean_dy_desc, mean_dy, mean_dy_xmu_desc, mean_dy_xmu, diff_x_desc,
+        diff_x);
+  return ret;
+}

bangc-ops/kernels/sync_batchnorm_backward_elemt_v2/sync_batchnorm_backward_elemt_v2.cpp

@@ -0,0 +1,42 @@
+/*************************************************************************
+ * Copyright (C) [2023] by Cambricon, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *************************************************************************/
+#include "kernels/kernel_wrapper/wrapper.h"
+
+mluOpStatus_t MLUOP_WIN_API mluOpSyncBatchNormBackwardElemtV2(
+    mluOpHandle_t handle, const mluOpTensorDescriptor_t diff_y_desc,
+    const void *diff_y, const mluOpTensorDescriptor_t x_desc, const void *x,
+    const mluOpTensorDescriptor_t mean_desc, const void *mean,
+    const mluOpTensorDescriptor_t invstd_desc, const void *invstd,
+    const mluOpTensorDescriptor_t filter_desc, const void *filter,
+    const mluOpTensorDescriptor_t sum_dy_desc, const void *sum_dy,
+    const mluOpTensorDescriptor_t sum_dy_xmu_desc, const void *sum_dy_xmu,
+    const mluOpTensorDescriptor_t count_desc, const void *count,
+    const mluOpTensorDescriptor_t diff_x_desc, void *diff_x) {
+  SyncBatchNormBackwardElemtV2Wrapper wrapper;
+  mluOpStatus_t ret = wrapper.invoke(
+      handle, diff_y_desc, diff_y, x_desc, x, mean_desc, mean, invstd_desc,
+      invstd, filter_desc, filter, sum_dy_desc, sum_dy, sum_dy_xmu_desc,
+      sum_dy_xmu, count_desc, count, diff_x_desc, diff_x);
+  return ret;
+}
+

bangc-ops/kernels/sync_batchnorm_backward_reduce/sync_batchnorm_backward_reduce.cpp

@@ -0,0 +1,65 @@
+/*************************************************************************
+ * Copyright (C) [2023] by Cambricon, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *************************************************************************/
+#include "kernels/kernel_wrapper/wrapper.h"
+
+mluOpStatus_t MLUOP_WIN_API mluOpSyncBatchnormBackwardReduce(
+    mluOpHandle_t handle, const mluOpTensorDescriptor_t desc_dz, const void *dz,
+    const mluOpTensorDescriptor_t desc_x, const void *x,
+    const mluOpTensorDescriptor_t desc_mean, const void *mean,
+    const mluOpTensorDescriptor_t desc_invstd, const void *invstd,
+    const mluOpTensorDescriptor_t desc_dfilter, void *dfilter,
+    const mluOpTensorDescriptor_t desc_dbias, void *dbias,
+    const mluOpTensorDescriptor_t desc_sum_dy, void *sum_dy,
+    const mluOpTensorDescriptor_t desc_sum_dy_xmu, void *sum_dy_xmu,
+    const bool needs_input_grad0, const bool needs_input_grad1,
+    const bool needs_input_grad2) {
+  SyncBatchnormBackwardReduceWrapper wrapper;
+  mluOpStatus_t ret =
+      wrapper.invoke(handle, desc_dz, dz, desc_x, x, desc_mean, mean,
+                     desc_invstd, invstd, desc_dfilter, dfilter, desc_dbias,
+                     dbias, desc_sum_dy, sum_dy, desc_sum_dy_xmu, sum_dy_xmu,
+                     needs_input_grad0, needs_input_grad1, needs_input_grad2);
+  return ret;
+}
+
+mluOpStatus_t MLUOP_WIN_API mluOpSyncBatchnormBackwardReduce_v2(
+    mluOpHandle_t handle, const mluOpTensorDescriptor_t desc_dz, const void *dz,
+    const mluOpTensorDescriptor_t desc_x, const void *x,
+    const mluOpTensorDescriptor_t desc_mean, const void *mean,
+    const mluOpTensorDescriptor_t desc_invstd, const void *invstd,
+    void *workspace, size_t workspace_size,
+    const mluOpTensorDescriptor_t desc_dfilter, void *dfilter,
+    const mluOpTensorDescriptor_t desc_dbias, void *dbias,
+    const mluOpTensorDescriptor_t desc_sum_dy, void *sum_dy,
+    const mluOpTensorDescriptor_t desc_sum_dy_xmu, void *sum_dy_xmu,
+    const bool needs_input_grad0, const bool needs_input_grad1,
+    const bool needs_input_grad2) {
+  SyncBatchnormBackwardReduceV2Wrapper wrapper;
+  mluOpStatus_t ret = wrapper.invoke(
+      handle, desc_dz, dz, desc_x, x, desc_mean, mean, desc_invstd, invstd,
+      workspace, workspace_size, desc_dfilter, dfilter, desc_dbias, dbias,
+      desc_sum_dy, sum_dy, desc_sum_dy_xmu, sum_dy_xmu, needs_input_grad0,
+      needs_input_grad1, needs_input_grad2);
+  return ret;
+}
+

bangc-ops/kernels/sync_batchnorm_elemt/sync_batchnorm_elemt.cpp

@@ -0,0 +1,38 @@
+/*************************************************************************
+ * Copyright (C) [2023] by Cambricon, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *************************************************************************/
+#include "kernels/kernel_wrapper/wrapper.h"
+
+mluOpStatus_t MLUOP_WIN_API mluOpSyncBatchNormElemt(
+    mluOpHandle_t handle, const mluOpTensorDescriptor_t x_desc, const void *x,
+    const mluOpTensorDescriptor_t mean_desc, const void *mean,
+    const mluOpTensorDescriptor_t invstd_desc, const void *invstd,
+    const mluOpTensorDescriptor_t filter_desc, const void *filter,
+    const mluOpTensorDescriptor_t bias_desc, const void *bias,
+    const mluOpTensorDescriptor_t y_desc, void *y) {
+  SyncBatchNormElemtWrapper wrapper;
+  mluOpStatus_t ret =
+      wrapper.invoke(handle, x_desc, x, mean_desc, mean, invstd_desc, invstd,
+                     filter_desc, filter, bias_desc, bias, y_desc, y);
+  return ret;
+}
+

bangc-ops/kernels/sync_batchnorm_gather_stats_with_counts/sync_batchnorm_gather_stats_with_counts.cpp

@@ -0,0 +1,50 @@
+/*************************************************************************
+ * Copyright (C) [2023] by Cambricon, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *************************************************************************/
+#include "kernels/kernel_wrapper/wrapper.h"
+
+mluOpStatus_t MLUOP_WIN_API mluOpSyncBatchNormGatherStatsWithCounts(
+    mluOpHandle_t handle,
+    const mluOpTensorDescriptor_t mean_all_desc,
+    const void *mean_all,
+    const mluOpTensorDescriptor_t invstd_all_desc,
+    const void *invstd_all,
+    const mluOpTensorDescriptor_t moving_mean_desc,
+    void *moving_mean,
+    const mluOpTensorDescriptor_t moving_var_desc,
+    void *moving_var,
+    float momentum,
+    float eps,
+    const mluOpTensorDescriptor_t count_all_desc,
+    const void *count_all,
+    const mluOpTensorDescriptor_t mean_desc,
+    void *mean,
+    const mluOpTensorDescriptor_t invstd_desc,
+    void *invstd) {
+  SyncBatchNormGatherStatsWithCountsWrapper wrapper;
+  mluOpStatus_t ret = wrapper.invoke(handle, mean_all_desc, mean_all,
+        invstd_all_desc, invstd_all, moving_mean_desc, moving_mean,
+        moving_var_desc, moving_var, momentum, eps, count_all_desc,
+        count_all, mean_desc, mean, invstd_desc, invstd);
+  return ret;
+}
+

bangc-ops/kernels/sync_batchnorm_stats/sync_batchnorm_stats.cpp

@@ -0,0 +1,45 @@
+/*************************************************************************
+ * Copyright (C) [2023] by Cambricon, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *************************************************************************/
+#include "kernels/kernel_wrapper/wrapper.h"
+
+mluOpStatus_t MLUOP_WIN_API mluOpSyncBatchNormStats(
+    mluOpHandle_t handle, const mluOpTensorDescriptor_t x_desc, const void *x,
+    const float eps, const mluOpTensorDescriptor_t mean_desc, void *mean,
+    const mluOpTensorDescriptor_t invstd_desc, void *invstd) {
+  SyncBatchNormStatsWrapper wrapper;
+  mluOpStatus_t ret = wrapper.invoke(handle, x_desc, x, eps, mean_desc, mean,
+                                     invstd_desc, invstd);
+  return ret;
+}
+
+mluOpStatus_t MLUOP_WIN_API mluOpSyncBatchNormStats_v2(
+    mluOpHandle_t handle, const mluOpTensorDescriptor_t x_desc, const void *x,
+    void *workspace, size_t workspace_size, const float eps,
+    const mluOpTensorDescriptor_t mean_desc, void *mean,
+    const mluOpTensorDescriptor_t invstd_desc, void *invstd) {
+  SyncBatchNormStatsV2Wrapper wrapper;
+  mluOpStatus_t ret =
+      wrapper.invoke(handle, x_desc, x, workspace, workspace_size, eps,
+                     mean_desc, mean, invstd_desc, invstd);
+  return ret;
+}