Merge remote-tracking branch 'origin/master' into rblaczkowski/change…

…-code-base-to-OpenVino

.ci/jenkins/ci.groovy

@@ -21,9 +21,9 @@ if(DOCKER_REGISTRY.trim() == "") {throw new Exception("Missing Docker registry u
 try {if(BACKEND_SKU_CONFIGURATION.trim() == "") {throw new Exception()}}
 catch (Exception e) {
     BACKEND_SKU_CONFIGURATIONS = [
-        [ sku : "skx", backends : ["cpu", "interpreter"] ],
-        [ sku : "clx", backends : ["cpu", "interpreter"] ],
-        [ sku : "bdw", backends : ["cpu", "interpreter"] ]
+        // [ sku : "skx", backends : ["cpu", "interpreter"] ],
+        [ sku : "clx", backends : ["cpu", "interpreter"] ]
+        // [ sku : "bdw", backends : ["cpu", "interpreter"] ]
         // [ sku: "iris", backend : "igpu" ]
     ]
 }
@@ -32,11 +32,11 @@ echo "BACKEND_SKU_CONFIGURATIONS=${BACKEND_SKU_CONFIGURATIONS}"
 // --- CI constants ---
 NGRAPH_ONNX_REPO_ADDRESS="[email protected]:NervanaSystems/ngraph-onnx.git"
 NGRAPH_REPO_ADDRESS="[email protected]:NervanaSystems/ngraph.git"
-DLDT_REPO_ADDRESS = "git@gitlab-icv.inn.intel.com:inference-engine/dldt.git"
+OPENVINO_REPO_ADDRESS = "git@github.com:openvinotoolkit/openvino.git"
 
 CI_LABELS = "ngraph_onnx && ci"
 CI_DIR = "ngraph-onnx/.ci/jenkins"
-DOCKER_CONTAINER_NAME = "jenkins_ngraph-onnx_ci"
+DOCKER_CONTAINER_PREFIX = "jenkins_ngraph-onnx_ci"
 
 JENKINS_GITHUB_CREDENTIAL_ID = "7157091e-bc04-42f0-99fd-dc4da2922a55"
 JENKINS_GITLAB_CREDENTIAL_ID = "1caab8d7-1d0c-4b8a-9438-b65336862ead"
@@ -51,6 +51,7 @@ CONFIGURATION_WORKFLOW = { configuration ->
         timeout(activity: true, time: 60) {
             WORKDIR = "${WORKSPACE}/${BUILD_NUMBER}"
             DOCKER_HOME = "/home/${USER}"
+            DOCKER_CONTAINER_NAME="${DOCKER_CONTAINER_PREFIX}_${EXECUTOR_NUMBER}"
             try {
                 stage("Clone repositories") {
                     dir (WORKDIR) {
@@ -60,11 +61,11 @@ CONFIGURATION_WORKFLOW = { configuration ->
                         gitClone("Clone ngraph", NGRAPH_REPO_ADDRESS, configuration.ngraphBranch)
                     }
                     dir (WORKDIR) {
-                        gitClone("Clone dldt", DLDT_REPO_ADDRESS, configuration.dldtBranch)
+                        gitClone("Clone openvino", OPENVINO_REPO_ADDRESS, configuration.openvinoBranch)
                     }
-                    gitSubmoduleUpdate("dldt")
+                    gitSubmoduleUpdate("openvino")
                 }
-                String imageName = "${DOCKER_REGISTRY}/aibt/aibt/ngraph_cpp/${configuration.os}/ubuntu_18_04"
+                String imageName = "${DOCKER_REGISTRY}/aibt/aibt/ngraph_cpp/${configuration.os}/ubuntu_18_04_test"
                 stage("Prepare Docker image") {
                     pullDockerImage(imageName)
                     appendUserToDockerImage(imageName)
@@ -172,7 +173,7 @@ def runDockerContainer(String imageName) {
         mkdir -p ${HOME}/ONNX_CI
         docker run -id --privileged \
                 --user ${USER} \
-                --name ${DOCKER_CONTAINER_NAME}  \
+                --name ${DOCKER_CONTAINER_NAME} \
                 --volume ${WORKDIR}:/logs \
                 --volume ${HOME}/ONNX_CI/onnx_models/.onnx:${dockerOnnxModels} \
                 --volume ${HOME}/ONNX_CI/cache:${dockerCache} \
@@ -194,7 +195,7 @@ def prepareEnvironment(List<String> backends, String ngraph_branch) {
 def runToxTests(String backend) {
     String toxEnvVar = "TOX_INSTALL_NGRAPH_FROM=\${NGRAPH_WHL}"
     String backendEnvVar = "NGRAPH_BACKEND=${backend.toUpperCase()}"
-    String libraryVar = (backend == "ie") ? "LD_LIBRARY_PATH=${DOCKER_HOME}/dldt_dist/deployment_tools/inference_engine/external/tbb/lib:${DOCKER_HOME}/dldt_dist/deployment_tools/inference_engine/lib/intel64:${DOCKER_HOME}/dldt_dist/deployment_tools/inference_engine/external/mkltiny_lnx/lib:${DOCKER_HOME}/dldt_dist/deployment_tools/ngraph/lib" : "LD_LIBRARY_PATH="
+    String libraryVar = (backend == "ie") ? "LD_LIBRARY_PATH=${DOCKER_HOME}/openvino_dist/deployment_tools/inference_engine/external/tbb/lib:${DOCKER_HOME}/openvino_dist/deployment_tools/inference_engine/lib/intel64:${DOCKER_HOME}/openvino_dist/deployment_tools/inference_engine/external/mkltiny_lnx/lib:${DOCKER_HOME}/openvino_dist/deployment_tools/ngraph/lib" : "LD_LIBRARY_PATH="
 
 
     if (backend == "ie") {
@@ -216,10 +217,10 @@ def cleanup() {
         docker rm -f \$(docker ps -a -q) || true
         printf 'y' | docker system prune
     """
-    deleteDir()
+    // deleteDir()
 }
 
-def getConfigurationsMap(String dockerfilesPath, String ngraphOnnxBranch, String ngraphBranch, String dldtBranch) {
+def getConfigurationsMap(String dockerfilesPath, String ngraphOnnxBranch, String ngraphBranch, String openvinoBranch) {
     def configurationsMap = [:]
     def osImages = sh (script: "find ${dockerfilesPath} -maxdepth 1 -name '*.dockerfile' -printf '%f\n'",
                     returnStdout: true).trim().replaceAll(".dockerfile","").split("\n") as List
@@ -230,7 +231,7 @@ def getConfigurationsMap(String dockerfilesPath, String ngraphOnnxBranch, String
             configuration.os = os
             configuration.ngraphOnnxBranch = ngraphOnnxBranch
             configuration.ngraphBranch = ngraphBranch
-            configuration.dldtBranch = dldtBranch
+            configuration.openvinoBranch = openvinoBranch
             String backendLabels = configuration.backends.join(" && ")
             configuration.label = "${backendLabels} && ${configuration.sku} && ${CI_LABELS}"
             configuration.name = "${configuration.sku}-${configuration.os}"

.ci/jenkins/dockerfiles/ubuntu_18_04.dockerfile

@@ -55,7 +55,8 @@ RUN curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.d
 
 RUN pip3 install --upgrade pip==19.0.3 \
         setuptools==41.0.0 \
-        wheel==0.33.1
+        wheel==0.33.1 \
+        cython
 
 # ONNX dependencies
 RUN apt-get -y --no-install-recommends install \

.ci/jenkins/prepare_environment.sh

@@ -57,31 +57,55 @@ function build_ngraph() {
     return 0
 }
 
-function build_dldt() {
+function build_open_vino() {
     set -x
     local directory="$1"
-    local ngraph_branch="$2"
-    CMAKE_ARGS="-DNGRAPH_LIBRARY_OUTPUT_DIRECTORY=${directory}/dldt_dist \
-                -DNGRAPH_COMPONENT_PREFIX=deployment_tools/ngraph/ -DNGRAPH_USE_PREBUILT_LLVM=TRUE \
-                -DNGRAPH_TOOLS_ENABLE=TRUE -DNGRAPH_WARNINGS_AS_ERRORS=TRUE -DNGRAPH_UNIT_TEST_ENABLE=FALSE \
-                -DCMAKE_BUILD_TYPE=Release -DENABLE_PYTHON=OFF -DENABLE_RPATH=ON -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
-                -DENABLE_PERFORMANCE_TESTS=ON -DENABLE_TESTS=ON -DNGRAPH_DEBUG_ENABLE=OFF \
-                -DENABLE_SAMPLES=OFF -DENABLE_FUNCTIONAL_TESTS=ON -DENABLE_MODELS=OFF -DENABLE_PRIVATE_MODELS=OFF \
-                -DENABLE_GNA=OFF -DENABLE_VPU=OFF -DENABLE_SANITIZER=OFF -DENABLE_MYRIAD=OFF -DENABLE_MKL_DNN=ON \
-                -DENABLE_CLDNN=OFF -DENABLE_VALIDATION_SET=OFF -DPYTHON_EXECUTABLE=`which python` \
-                -DNGRAPH_ONNX_IMPORT_ENABLE=ON -DNGRAPH_UNIT_TEST_OPENVINO_ENABLE=TRUE -DNGRAPH_IE_ENABLE=ON \
-                -DCMAKE_INSTALL_PREFIX=${directory}/dldt_dist -DNGRAPH_DYNAMIC_COMPONENTS_ENABLE=ON"
-    cd "${directory}/dldt/ngraph"
-    git checkout "${ngraph_branch}"
+    CMAKE_ARGS="-DCMAKE_BUILD_TYPE=Debug \
+                -DENABLE_VALIDATION_SET=OFF \
+                -DENABLE_VPU=OFF \
+                -DENABLE_DLIA=OFF \
+                -DENABLE_GNA=OFF \
+                -DENABLE_CPPLINT=OFF \
+                -DENABLE_TESTS=OFF \
+                -DENABLE_BEH_TESTS=OFF \
+                -DENABLE_FUNCTIONAL_TESTS=OFF \
+                -DENABLE_MKL_DNN=ON \
+                -DENABLE_CLDNN=OFF \
+                -DENABLE_PROFILING_ITT=OFF \
+                -DENABLE_SAMPLES=OFF \
+                -DENABLE_SPEECH_DEMO=OFF \
+                -DENABLE_PYTHON=ON \
+                -DPYTHON_EXECUTABLE=`which python3` \
+                -DNGRAPH_ONNX_IMPORT_ENABLE=ON \
+                -DNGRAPH_IE_ENABLE=ON \
+                -DNGRAPH_INTERPRETER_ENABLE=ON \
+                -DNGRAPH_DEBUG_ENABLE=OFF \
+                -DNGRAPH_DYNAMIC_COMPONENTS_ENABLE=ON \
+                -DCMAKE_INSTALL_PREFIX=${directory}/openvino_dist"
 
-    cd "${directory}/dldt"
+    cd "${directory}/openvino"
 
     mkdir -p ./build
     cd ./build
-    git lfs install
     cmake ${CMAKE_ARGS} ..  || return 1
     make -j $(lscpu --parse=CORE | grep -v '#' | sort | uniq | wc -l) || return 1
     make install || return 1
+
+    cd "${directory}/openvino/ngraph/python"
+    if [ ! -d ./pybind11 ]; then
+        git clone --recursive https://github.com/pybind/pybind11.git
+    fi
+    virtualenv -p `which python3` venv
+    . venv/bin/activate
+    rm -f "${directory}/openvino/ngraph/python/dist/ngraph*.whl"
+    rm -rf "${directory}/openvino/ngraph/python/*.so ${directory}/openvino/ngraph/python/build"
+    export PYBIND_HEADERS_PATH="${directory}/openvino/ngraph/python/pybind11"
+    export NGRAPH_CPP_BUILD_PATH="${directory}/openvino_dist"
+    export NGRAPH_ONNX_IMPORT_ENABLE="TRUE"
+    mv "${directory}/ngraph-onnx/.ci/jenkins/setup.py" .
+    python3 setup.py develop || return 1
+    # Clean build artifacts
+    # rm -rf "${directory}/openvino_dist"
     return 0
 }
 
@@ -115,11 +139,9 @@ function main() {
         esac
     done
 
-    BUILD_NGRAPH_CALL="build_ngraph \"${BUILD_DIR}\" \"${BACKENDS}\""
-    BUILD_DLDT_CALL="build_dldt \"${BUILD_DIR}\" \"${NGRAPH_BRANCH}\""
+    BUILD_OV_CALL="build_open_vino \"${BUILD_DIR}\" \"${NGRAPH_BRANCH}\""
 
-    eval "${BUILD_NGRAPH_CALL}"
-    # eval "${BUILD_DLDT_CALL}"
+    eval "${BUILD_OV_CALL}"
 
 }