Merge pull request #315 from vespa-engine/kkraune/import-export

Kkraune/import export

screwdriver.yaml

@@ -30,6 +30,7 @@ jobs:
           pytest --doctest-modules --ignore-glob=vespa/test_*.py
       - run-unit-tests: |
           pytest --ignore-glob=vespa/test_integration*.py
+
   integration-except-cloud:
     requires: [~commit,~pr]
     annotations:
@@ -59,6 +60,7 @@ jobs:
           pytest vespa/test_integration_running_instance.py -s -v
       - run-integration-docker: |
           pytest vespa/test_integration_docker.py -s -v
+
   notebooks-except-cloud:
     requires: [~commit,~pr]
     annotations:
@@ -101,6 +103,7 @@ jobs:
             do echo -e "**********\n*** Running $notebook ***\n**********"; \
             runnb --allow-not-trusted $notebook || exit 1;\
           done
+
   integration-cloud:
     requires: [~commit]
     annotations:
@@ -116,14 +119,14 @@ jobs:
       - setup: |
           export WORK_DIR=$SD_DIND_SHARE_PATH
           export RESOURCES_DIR=$(pwd)/vespa/resources
-          export PYVESPA_DEBUG=true
       - install-python: |
           dnf install -y python38-pip
           python3 -m pip install --upgrade pip
           python3 -m pip install pytest
           python3 -m pip install -e .[full]
       - run-integration-cloud: |
           pytest vespa/test_integration_vespa_cloud.py -s -v
+
   notebooks-cloud:
     requires: [integration-cloud]
     annotations:
@@ -148,6 +151,7 @@ jobs:
       - run-notebooks-cloud-related: |
           find docs -name '*cloud*.ipynb'
           find docs -name '*cloud*.ipynb' -exec ls {} \; -name '*cloud*.ipynb' -exec runnb --allow-not-trusted {} \;
+
   deploy-test-server:
     requires: [unit-tests, integration-except-cloud, notebooks-except-cloud, notebooks-cloud]
     annotations:
@@ -167,6 +171,7 @@ jobs:
           python3 -m twine upload --repository testpypi dist/* -u __token__ -p $TEST_PYPI_TOKEN
       - sleep: |
           sleep 300 # sleep to give time for latest version to show on test pypi index
+
   retrieve-install-test:
     requires: [deploy-test-server]
     annotations:
@@ -182,6 +187,8 @@ jobs:
           python3 -m pip install pytest
       - install-package: |
           python3 -m pip cache purge
-          python3 -m pip install --no-cache-dir --no-cache --quiet --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -Iv pyvespa[full]==$PYVESPA_VERSION.$SD_EVENT_ID 2>&1 | tee $SD_ARTIFACTS_DIR/mylogfile.txt
+          python3 -m pip install --no-cache-dir --no-cache --quiet \
+            --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple \
+            -Iv pyvespa[full]==$PYVESPA_VERSION.$SD_EVENT_ID 2>&1 | tee $SD_ARTIFACTS_DIR/mylogfile.txt
       - test-package: |
           pytest --ignore-glob=vespa/test_integration*.py -s -v

vespa/deployment.py

@@ -186,6 +186,7 @@ def export_application_package(
         :param application_package: Application package to export.
         :return: None. Application package file will be stored on `disk_folder`.
         """
+        # ToDo: remove this method, not needed with ApplicationPackage::tofiles
         if not self.disk_folder:
             self.disk_folder = os.path.join(os.getcwd(), application_package.name)
 
@@ -260,6 +261,7 @@ def _execute_deployment(
         application_folder: Optional[str] = None,
         application_package: Optional[ApplicationPackage] = None,
     ):
+        # ToDo: Remove this method
 
         self._run_vespa_engine_container(
             application_name=application_name,
@@ -299,34 +301,6 @@ def _execute_deployment(
 
         return app
 
-    def _execute_deployment_zip(
-            self,
-            app_package: ApplicationPackage
-    ) -> Vespa:
-
-        self._run_vespa_engine_container(
-            application_name=app_package.name,
-            container_memory=self.container_memory,
-        )
-        self.wait_for_config_server_start(max_wait=CFG_SERVER_START_TIMEOUT)
-
-        r = requests.post("http://localhost:{}/application/v2/tenant/default/prepareandactivate".format(self.cfgsrv_port),
-                          headers={"Content-Type": "application/zip"},
-                          data=app_package.to_zip(),
-                          verify=False)
-        logging.debug("Deploy status code: {}".format(r.status_code))
-        if r.status_code != 200:
-            raise RuntimeError("Deployment failed, code: {}".format(r.status_code))
-
-        app = Vespa(
-            url=self.url,
-            port=self.local_port,
-        )
-        app.wait_for_application_up(max_wait=APP_INIT_TIMEOUT)
-
-        print("Finished deployment.", file=self.output)
-        return app
-
     def wait_for_config_server_start(self, max_wait):
         """
         Waits for Config Server to start inside the Docker image
@@ -359,7 +333,7 @@ def deploy(
         :return: a Vespa connection instance.
         """
         if not self.disk_folder:
-            return self._execute_deployment_zip(app_package=application_package)
+            return self._deploy_data(application_package.name, application_package.to_zip())
 
         self.export_application_package(application_package=application_package)
         return self._execute_deployment(
@@ -383,6 +357,7 @@ def deploy_from_disk(
             If None, we assume `disk_folder` to be the application folder.
         :return: a Vespa connection instance.
         """
+        # ToDo: Remove this method in a later release - deploy_zipped_from_disk will replace this / take its place
         if not self.disk_folder:
             self.disk_folder = os.path.join(os.getcwd(), application_name)
 
@@ -393,6 +368,63 @@ def deploy_from_disk(
             application_folder=application_folder,
         )
 
+    def deploy_zipped_from_disk(self, app_name: str, app_root: Path) -> Vespa:
+        """
+        Deploy from a directory tree.
+        Used when making changes to application package files not supported by pyvespa -
+        This is why this method is not found in the ApplicationPackage class.
+
+        :param app_name: Application package name
+        :param app_root: Application package directory root
+        :return: A Vespa connection instance
+        """
+        TMP_ZIP="tmp_app_package.zip"
+        orig_dir = os.getcwd()
+        zipf = zipfile.ZipFile(TMP_ZIP, "w", zipfile.ZIP_DEFLATED)
+        os.chdir(app_root)  # Workaround to avoid the top-level directory
+        for root, dirs, files in os.walk("."):
+            for file in files:
+                zipf.write(os.path.join(root, file))
+        zipf.close()
+        os.chdir(orig_dir)
+        with open(TMP_ZIP, "rb") as f:
+            data = f.read()
+        os.remove(TMP_ZIP)
+
+        return self._deploy_data(app_name, data)
+
+
+    def _deploy_data(self, app_name: str, data) -> Vespa:
+        """
+        Deploys an Application Package as zipped data
+
+        :param app_name: Application package name
+        :param app_root: Application package directory root
+        :return: A Vespa connection instance
+        """
+        self._run_vespa_engine_container(
+            application_name=app_name,
+            container_memory=self.container_memory,
+        )
+        self.wait_for_config_server_start(max_wait=CFG_SERVER_START_TIMEOUT)
+
+        r = requests.post("http://localhost:{}/application/v2/tenant/default/prepareandactivate".format(self.cfgsrv_port),
+                          headers={"Content-Type": "application/zip"},
+                          data=data,
+                          verify=False)
+        logging.debug("Deploy status code: {}".format(r.status_code))
+        if r.status_code != 200:
+            raise RuntimeError("Deployment failed, code: {}".format(r.status_code))
+
+        app = Vespa(
+            url=self.url,
+            port=self.local_port,
+        )
+        app.wait_for_application_up(max_wait=APP_INIT_TIMEOUT)
+
+        print("Finished deployment.", file=self.output)
+        return app
+
     def stop_services(self):
         """
         Stop Vespa services.

vespa/package.py

@@ -1174,11 +1174,7 @@ def __init__(
         default_query_model: Optional[QueryModel] = None
     ) -> None:
         """
-        Create a Vespa Application Package.
-
-        Check the `Vespa documentation <https://docs.vespa.ai/en/cloudconfig/application-packages.html>`__
-        for more detailed information about application packages.
-
+        Create an `Application Package <https://docs.vespa.ai/en/cloudconfig/application-packages.html>`__.
         An :class:`ApplicationPackage` instance comes with a default :class:`Schema`
         that contains a default :class:`Document`
 
@@ -1356,6 +1352,7 @@ def query_profile_type_to_text(self):
 
     @property
     def hosts_to_text(self):
+        # ToDo: Remove, not needed anymore
         env = Environment(
             loader=PackageLoader("vespa", "templates"),
             autoescape=select_autoescape(
@@ -1419,6 +1416,11 @@ def load(disk_folder: str) -> "ApplicationPackage":
             return ApplicationPackage.from_json(f.read())
 
     def to_zip(self) -> BytesIO:
+        """
+        Return the application package as zipped bytes,
+        to be used in a subsequent deploy
+        :return: BytesIO buffer
+        """
         buffer = BytesIO()
         with zipfile.ZipFile(buffer, "a") as zip_archive:
             zip_archive.writestr(
@@ -1465,11 +1467,25 @@ def to_zip(self) -> BytesIO:
         #    app.public_bytes(serialization.Encoding.PEM),
         #)
 
-    def to_zipfile(self, path):
-        with open(path, "wb") as f:
+    def to_zipfile(self, zipfile: Path) -> None:
+        """
+        Export the application package as a deployable zipfile.
+        See `application packages <https://docs.vespa.ai/en/cloudconfig/application-packages.html>`__
+        for deployment options.
+
+        :param zipfile: Filename to export to
+        :return:
+        """
+        with open(zipfile, "wb") as f:
             f.write(self.to_zip().getbuffer().tobytes())
 
     def to_files(self, root: Path) -> None:
+        """
+        Export the application package as a directory tree.
+
+        :param root: Directory to export files to
+        :return:
+        """
         if not os.path.exists(root):
             raise ValueError("Invalid path for export: {}".format(root))
 
@@ -1493,8 +1509,6 @@ def to_files(self, root: Path) -> None:
             with open(os.path.join(root, "search/query-profiles/types/root.xml"), "w") as f:
                 f.write(self.query_profile_type_to_text)
 
-        # with open(os.path.join(root, "hosts.xml"), "w") as f:
-        #     f.write(self.hosts_to_text)
         with open(os.path.join(root, "services.xml"), "w") as f:
             f.write(self.services_to_text)