[DAPHNE-#711] Data transfer with numpy for arrays of more than 2 dime…

…nsions

- Previously only numpy arrays with 1 or 2 dimensions were supported.
- Added support for numpy arrays with more than 2 dimensions.
- Implementation is analog to tensorflow and pytorch with a simple reshape to a 2d numpy array.
- Added tests for ndim numpy arrays transfers.

doc/DaphneLib/APIRef.md

@@ -31,7 +31,7 @@ However, as the methods largely map to DaphneDSL built-in functions, you can fin
 
 **Importing data from other Python libraries:**
 
-- **`from_numpy`**`(mat: np.array, shared_memory=True, verbose=False) -> Matrix`
+- **`from_numpy`**`(mat: np.array, shared_memory=True, verbose=False, return_shape=False) -> Matrix`
 - **`from_pandas`**`(df: pd.DataFrame, shared_memory=True, verbose=False, keepIndex=False) -> Frame`
 - **`from_tensorflow`**`(tensor: tf.Tensor, shared_memory=True, verbose=False, return_shape=False) -> Matrix`
 - **`from_pytorch`**`(tensor: torch.Tensor, shared_memory=True, verbose=False, return_shape=False) -> Matrix`

src/api/python/daphne/context/daphne_context.py

@@ -70,25 +70,28 @@ def readFrame(self, file: str) -> Frame:
         unnamed_params = ['\"'+file+'\"']
         return Frame(self, 'readFrame', unnamed_params)
 
-    def from_numpy(self, mat: np.array, shared_memory=True, verbose=False) -> Matrix:
+    def from_numpy(self, mat: np.array, shared_memory=True, verbose=False, return_shape=False):
         """Generates a `DAGNode` representing a matrix with data given by a numpy `array`.
         :param mat: The numpy array.
         :param shared_memory: Whether to use shared memory data transfer (True) or not (False).
         :param verbose: Whether to print timing information (True) or not (False).
+        :param return_shape: Whether to return the original shape of the input array.
         :return: The data from numpy as a Matrix.
         """
 
+        original_shape = mat.shape
+
         if verbose:
             start_time = time.time()
 
         # Handle the dimensionality of the matrix.
         if mat.ndim == 1:
             rows = mat.shape[0]
             cols = 1
-        elif mat.ndim == 2:
+        elif mat.ndim >= 2:
+            if mat.ndim > 2:
+                mat = mat.reshape((original_shape[0], -1))
             rows, cols = mat.shape
-        else:
-            raise ValueError("input numpy array should be 1d or 2d")
 
         if shared_memory:
             # Data transfer via shared memory.
@@ -136,7 +139,7 @@ def from_numpy(self, mat: np.array, shared_memory=True, verbose=False) -> Matrix
         if verbose:
             print(f"from_numpy(): total Python-side execution time: {(time.time() - start_time):.10f} seconds")
 
-        return res
+        return (res, original_shape) if return_shape else res
 
     def from_pandas(self, df: pd.DataFrame, shared_memory=True, verbose=False, keepIndex=False) -> Frame:
         """Generates a `DAGNode` representing a frame with data given by a pandas `DataFrame`.

test/api/python/DaphneLibTest.cpp

@@ -68,6 +68,7 @@ const std::string dirPath = "test/api/python/";
 
 MAKE_TEST_CASE("data_transfer_numpy_1")
 MAKE_TEST_CASE("data_transfer_numpy_2")
+MAKE_TEST_CASE("data_transfer_numpy_3")
 MAKE_TEST_CASE("data_transfer_pandas_1")
 MAKE_TEST_CASE("data_transfer_pandas_2")
 MAKE_TEST_CASE("data_transfer_pandas_3_series")

test/api/python/data_transfer_numpy_3.daphne

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2022 The DAPHNE Consortium
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+m1 = reshape(seq(0, 7), 2, 4);
+m2 = reshape(seq(0, 26), 3, 9);
+m3 = reshape(seq(0, 31), 2, 16);
+print(m1);
+print(m2);
+print(m3);
+
+# verify that the original_shapes of DaphneLib, match the returned output
+print("(2, 2, 2)");
+print("(3, 3, 3)");
+print("(2, 2, 2, 2, 2)");

test/api/python/data_transfer_numpy_3.py

@@ -0,0 +1,37 @@
+#!/usr/bin/python
+
+# Copyright 2022 The DAPHNE Consortium
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Data transfer from numpy to DAPHNE and back, via shared memory.
+
+import numpy as np
+from daphne.context.daphne_context import DaphneContext
+
+m1 = np.arange(8).reshape((2,2,2))
+m2 = np.arange(27).reshape((3,3,3))
+m3 = np.arange(32).reshape((2,2,2,2,2))
+
+dctx = DaphneContext()
+
+X, m1_og_shape = dctx.from_numpy(m1, shared_memory=True, return_shape=True)
+Y, m2_og_shape = dctx.from_numpy(m2, shared_memory=True, return_shape=True)
+Z, m3_og_shape = dctx.from_numpy(m3, shared_memory=True, return_shape=True)
+
+X.print().compute()
+Y.print().compute()
+Z.print().compute()
+print(m1_og_shape)
+print(m2_og_shape)
+print(m3_og_shape)