From 68e2bafe815e07d005edd909ad4eb5842cb6f799 Mon Sep 17 00:00:00 2001 From: Kristian Aune Date: Wed, 22 Feb 2023 09:09:26 +0100 Subject: [PATCH 1/4] First draft app package guide --- docs/sphinx/source/application-packages.ipynb | 300 ++++++++++++++++++ .../examples/text-search-quick-start.ipynb | 4 +- 2 files changed, 302 insertions(+), 2 deletions(-) create mode 100644 docs/sphinx/source/application-packages.ipynb diff --git a/docs/sphinx/source/application-packages.ipynb b/docs/sphinx/source/application-packages.ipynb new file mode 100644 index 00000000..4f3bccd5 --- /dev/null +++ b/docs/sphinx/source/application-packages.ipynb @@ -0,0 +1,300 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "e05d0811", + "metadata": {}, + "source": [ + "![Vespa logo](https://vespa.ai/assets/vespa-logo-color.png)\n", + "\n", + "# Application packages\n", + "\n", + "Vespa is configured using an [application package](https://docs.vespa.ai/en/application-packages.html).\n", + "Pyvespa provides an API to generate a deployable application package.\n", + "\n", + "An application package has at a minimum a [schema](https://docs.vespa.ai/en/schemas.html)\n", + "and [services.xml](https://docs.vespa.ai/en/reference/services.html).\n", + "\n", + "Example - create an empty application package:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7e3477a6", + "metadata": {}, + "outputs": [], + "source": [ + "from vespa.package import ApplicationPackage\n", + "\n", + "app_package = ApplicationPackage(name=\"myschema\")" + ] + }, + { + "cell_type": "markdown", + "id": "e3f1e7d5", + "metadata": {}, + "source": [ + "To inspect an application package, dump it to disk using\n", + "[to_files](https://pyvespa.readthedocs.io/en/latest/reference-api.html#vespa.package.ApplicationPackage.to_files):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d05523a8", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/var/folders/9_/z105jyln7jz8h2vwsrjb7kxh0000gp/T/tmpan_c5inu\n" + ] + } + ], + "source": [ + "import tempfile, os\n", + "\n", + "temp_dir = tempfile.TemporaryDirectory()\n", + "print(temp_dir.name)\n", + "os.environ[\"TMP_APP_DIR\"] = temp_dir.name\n", + "\n", + "app_package.to_files(temp_dir.name)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e3a4dc05", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "./services.xml\r\n", + "./schemas/myschema.sd\r\n", + "./search/query-profiles/types/root.xml\r\n", + "./search/query-profiles/default.xml\r\n" + ] + } + ], + "source": [ + "!cd $TMP_APP_DIR && find . -type f" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4abae84e", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r\n", + "\r\n", + " \r\n", + " \r\n", + " \r\n", + " \r\n", + " \r\n", + " 1\r\n", + " \r\n", + " \r\n", + " \r\n", + " \r\n", + " \r\n", + " \r\n", + " \r\n", + "" + ] + } + ], + "source": [ + "!cat $TMP_APP_DIR/services.xml" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "923edec8", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "schema myschema {\r\n", + " document myschema {\r\n", + " }\r\n", + "}" + ] + } + ], + "source": [ + "!cat $TMP_APP_DIR/schemas/myschema.sd" + ] + }, + { + "cell_type": "markdown", + "id": "7f92ab60", + "metadata": {}, + "source": [ + "Observe:\n", + "* An empty schema is created, with the same name as the application package\n", + "* A content cluster (this is where the index is stored) called `myschema_content` is created.\n", + " This is information not normally needed, unless using\n", + " [delete_all_docs](https://pyvespa.readthedocs.io/en/latest/reference-api.html#vespa.application.Vespa.delete_all_docs)\n", + " to quickly remove all documents from a schema" + ] + }, + { + "cell_type": "markdown", + "id": "c038d33a", + "metadata": {}, + "source": [ + "Ignore these files for now:\n", + "\n", + " ./search/query-profiles/types/root.xml\n", + " ./search/query-profiles/default.xml" + ] + }, + { + "cell_type": "markdown", + "id": "2d558849", + "metadata": {}, + "source": [ + "## Schema\n", + "\n", + "Create fields, fieldsets and a ranking function:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c83c1945", + "metadata": {}, + "outputs": [], + "source": [ + "from vespa.package import Field, FieldSet, RankProfile\n", + "\n", + "app_package.schema.add_fields(\n", + " Field(name = \"id\", type = \"string\", indexing = [\"attribute\", \"summary\"]),\n", + " Field(name = \"title\", type = \"string\", indexing = [\"index\", \"summary\"], index = \"enable-bm25\"),\n", + " Field(name = \"body\", type = \"string\", indexing = [\"index\", \"summary\"], index = \"enable-bm25\")\n", + ")\n", + "\n", + "app_package.schema.add_field_set(\n", + " FieldSet(name = \"default\", fields = [\"title\", \"body\"])\n", + ")\n", + "\n", + "app_package.schema.add_rank_profile(\n", + " RankProfile(name = \"default\", first_phase = \"bm25(title) + bm25(body)\")\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "f721bdfd", + "metadata": {}, + "source": [ + "Dump application package again, show schema:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4fcd3de2", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "schema myschema {\r\n", + " document myschema {\r\n", + " field id type string {\r\n", + " indexing: attribute | summary\r\n", + " }\r\n", + " field title type string {\r\n", + " indexing: index | summary\r\n", + " index: enable-bm25\r\n", + " }\r\n", + " field body type string {\r\n", + " indexing: index | summary\r\n", + " index: enable-bm25\r\n", + " }\r\n", + " }\r\n", + " fieldset default {\r\n", + " fields: title, body\r\n", + " }\r\n", + " rank-profile default {\r\n", + " first-phase {\r\n", + " expression: bm25(title) + bm25(body)\r\n", + " }\r\n", + " }\r\n", + "}" + ] + } + ], + "source": [ + "app_package.to_files(temp_dir.name)\n", + "!cat $TMP_APP_DIR/schemas/myschema.sd" + ] + }, + { + "cell_type": "markdown", + "id": "7cc78157", + "metadata": {}, + "source": [ + "Note how the indexing settings are written to the schema.\n", + "\n", + "> **_It is important to know that pyvespa generally does not support all indexing options in Vespa.\n", + " Pyvespa is made for easy experimentation.\n", + " To configure setting an unsupported indexing option, dump the application package, modify the schema file\n", + " and deploy the application package from the directory or as a zipped file.\n", + " [Read more](https://pyvespa.readthedocs.io/en/latest/deploy-docker.html)_**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "84ce16e8", + "metadata": {}, + "outputs": [], + "source": [ + "temp_dir.cleanup()" + ] + }, + { + "cell_type": "markdown", + "id": "e242ac80", + "metadata": {}, + "source": [ + "## Next steps\n", + "\n", + "Link to deployment options here" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aa0d6b43", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "python3", + "language": "python", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/sphinx/source/examples/text-search-quick-start.ipynb b/docs/sphinx/source/examples/text-search-quick-start.ipynb index f635aeef..6be4825a 100644 --- a/docs/sphinx/source/examples/text-search-quick-start.ipynb +++ b/docs/sphinx/source/examples/text-search-quick-start.ipynb @@ -439,11 +439,11 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "python3", "language": "python", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 5 -} \ No newline at end of file +} From 3c5458f96b8e6cf963619fa532d5e2ebe669ce46 Mon Sep 17 00:00:00 2001 From: Kristian Aune Date: Mon, 27 Feb 2023 15:57:39 +0100 Subject: [PATCH 2/4] Schema and Services --- docs/sphinx/source/application-packages.ipynb | 179 ++++++++++-------- ...-for-question-answering-applications.ipynb | 4 +- 2 files changed, 101 insertions(+), 82 deletions(-) diff --git a/docs/sphinx/source/application-packages.ipynb b/docs/sphinx/source/application-packages.ipynb index 4f3bccd5..c25d49f4 100644 --- a/docs/sphinx/source/application-packages.ipynb +++ b/docs/sphinx/source/application-packages.ipynb @@ -44,23 +44,14 @@ "execution_count": null, "id": "d05523a8", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "/var/folders/9_/z105jyln7jz8h2vwsrjb7kxh0000gp/T/tmpan_c5inu\n" - ] - } - ], + "outputs": [], "source": [ "import tempfile, os\n", "\n", "temp_dir = tempfile.TemporaryDirectory()\n", - "print(temp_dir.name)\n", "os.environ[\"TMP_APP_DIR\"] = temp_dir.name\n", - "\n", - "app_package.to_files(temp_dir.name)" + "app_package.to_files(temp_dir.name)\n", + "print(temp_dir.name)" ] }, { @@ -85,36 +76,24 @@ ] }, { - "cell_type": "code", - "execution_count": null, - "id": "4abae84e", + "cell_type": "markdown", + "id": "c038d33a", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r\n", - "\r\n", - " \r\n", - " \r\n", - " \r\n", - " \r\n", - " \r\n", - " 1\r\n", - " \r\n", - " \r\n", - " \r\n", - " \r\n", - " \r\n", - " \r\n", - " \r\n", - "" - ] - } - ], "source": [ - "!cat $TMP_APP_DIR/services.xml" + "Ignore these files for now:\n", + "\n", + " ./search/query-profiles/types/root.xml\n", + " ./search/query-profiles/default.xml" + ] + }, + { + "cell_type": "markdown", + "id": "7b01cd09", + "metadata": {}, + "source": [ + "## Schema\n", + "\n", + "Use a schema to Create fields, fieldsets and a ranking function - dump the empty schema (An empty schema is created, with the same name as the application package):" ] }, { @@ -140,36 +119,10 @@ }, { "cell_type": "markdown", - "id": "7f92ab60", - "metadata": {}, - "source": [ - "Observe:\n", - "* An empty schema is created, with the same name as the application package\n", - "* A content cluster (this is where the index is stored) called `myschema_content` is created.\n", - " This is information not normally needed, unless using\n", - " [delete_all_docs](https://pyvespa.readthedocs.io/en/latest/reference-api.html#vespa.application.Vespa.delete_all_docs)\n", - " to quickly remove all documents from a schema" - ] - }, - { - "cell_type": "markdown", - "id": "c038d33a", - "metadata": {}, - "source": [ - "Ignore these files for now:\n", - "\n", - " ./search/query-profiles/types/root.xml\n", - " ./search/query-profiles/default.xml" - ] - }, - { - "cell_type": "markdown", - "id": "2d558849", + "id": "5a1cbaf2", "metadata": {}, "source": [ - "## Schema\n", - "\n", - "Create fields, fieldsets and a ranking function:" + "Add fields, a fieldset and a ranking function:" ] }, { @@ -252,40 +205,106 @@ "source": [ "Note how the indexing settings are written to the schema.\n", "\n", - "> **_It is important to know that pyvespa generally does not support all indexing options in Vespa.\n", - " Pyvespa is made for easy experimentation.\n", - " To configure setting an unsupported indexing option, dump the application package, modify the schema file\n", - " and deploy the application package from the directory or as a zipped file.\n", + "> **_Pyvespa generally does not support all indexing options in Vespa - it is made for easy experimentation.\n", + " To configure setting an unsupported indexing option (or any other unsupported option),\n", + " dump the application package, modify the schema file\n", + " and deploy the application package from the directory, or as a zipped file.\n", " [Read more](https://pyvespa.readthedocs.io/en/latest/deploy-docker.html)_**" ] }, + { + "cell_type": "markdown", + "id": "cfd73872", + "metadata": {}, + "source": [ + "At this point, review the Vespa documentation:\n", + "* [field](https://docs.vespa.ai/en/schemas.html#field)\n", + "* [fieldset](https://docs.vespa.ai/en/schemas.html#fieldset)\n", + "* [rank-profile](https://docs.vespa.ai/en/ranking.html#rank-profiles)" + ] + }, + { + "cell_type": "markdown", + "id": "a51353a4", + "metadata": {}, + "source": [ + "## Services\n", + "\n", + "In `services.xml` you will find a container and content cluster -\n", + "see the [Vespa Overview](https://docs.vespa.ai/en/overview.html).\n", + "This is a file you will normally not change or need to know much about - dump the default file:" + ] + }, { "cell_type": "code", "execution_count": null, - "id": "84ce16e8", + "id": "4abae84e", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r\n", + "\r\n", + " \r\n", + " \r\n", + " \r\n", + " \r\n", + " \r\n", + " 1\r\n", + " \r\n", + " \r\n", + " \r\n", + " \r\n", + " \r\n", + " \r\n", + " \r\n", + "" + ] + } + ], "source": [ - "temp_dir.cleanup()" + "!cat $TMP_APP_DIR/services.xml" ] }, { "cell_type": "markdown", - "id": "e242ac80", + "id": "d6477c44", "metadata": {}, "source": [ - "## Next steps\n", + "Observe:\n", + "* A content cluster (this is where the index is stored) called `myschema_content` is created.\n", + " This is information not normally needed, unless using\n", + " [delete_all_docs](https://pyvespa.readthedocs.io/en/latest/reference-api.html#vespa.application.Vespa.delete_all_docs)\n", + " to quickly remove all documents from a schema\n", "\n", - "Link to deployment options here" + "Remove the temporary application package file dump:" ] }, { "cell_type": "code", "execution_count": null, - "id": "aa0d6b43", + "id": "84ce16e8", "metadata": {}, "outputs": [], - "source": [] + "source": [ + "temp_dir.cleanup()" + ] + }, + { + "cell_type": "markdown", + "id": "e242ac80", + "metadata": {}, + "source": [ + "## Next step: Deploy, feed and query\n", + "\n", + "Once the schema is ready for deployment, decide deployment option and deploy the application package:\n", + "* [Deploy to local container](https://pyvespa.readthedocs.io/en/latest/deploy-docker.html)\n", + "* [Deploy to Vespa Cloud](https://pyvespa.readthedocs.io/en/latest/deploy-vespa-cloud.html)\n", + "\n", + "Use the guides on the pyvespa site to feed and query data." + ] } ], "metadata": { diff --git a/docs/sphinx/source/examples/semantic-retrieval-for-question-answering-applications.ipynb b/docs/sphinx/source/examples/semantic-retrieval-for-question-answering-applications.ipynb index 748b02dc..1a6378f0 100644 --- a/docs/sphinx/source/examples/semantic-retrieval-for-question-answering-applications.ipynb +++ b/docs/sphinx/source/examples/semantic-retrieval-for-question-answering-applications.ipynb @@ -916,11 +916,11 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "python3", "language": "python", "name": "python3" } }, "nbformat": 4, "nbformat_minor": 5 -} \ No newline at end of file +} From e24f0f64fac0e7ff4beaaac248345787de640fda Mon Sep 17 00:00:00 2001 From: Kristian Aune Date: Tue, 28 Feb 2023 08:50:21 +0100 Subject: [PATCH 3/4] Remove unintended commit --- ...semantic-retrieval-for-question-answering-applications.ipynb | 2 +- docs/sphinx/source/examples/text-search-quick-start.ipynb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/sphinx/source/examples/semantic-retrieval-for-question-answering-applications.ipynb b/docs/sphinx/source/examples/semantic-retrieval-for-question-answering-applications.ipynb index 1a6378f0..d7fb498b 100644 --- a/docs/sphinx/source/examples/semantic-retrieval-for-question-answering-applications.ipynb +++ b/docs/sphinx/source/examples/semantic-retrieval-for-question-answering-applications.ipynb @@ -916,7 +916,7 @@ ], "metadata": { "kernelspec": { - "display_name": "python3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" } diff --git a/docs/sphinx/source/examples/text-search-quick-start.ipynb b/docs/sphinx/source/examples/text-search-quick-start.ipynb index 6be4825a..fc481d01 100644 --- a/docs/sphinx/source/examples/text-search-quick-start.ipynb +++ b/docs/sphinx/source/examples/text-search-quick-start.ipynb @@ -439,7 +439,7 @@ ], "metadata": { "kernelspec": { - "display_name": "python3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" } From 28c92977bffa45967e980ea8bafffdd4484f63eb Mon Sep 17 00:00:00 2001 From: Kristian Aune Date: Tue, 28 Feb 2023 08:52:09 +0100 Subject: [PATCH 4/4] Removed on master --- .../examples/text-search-quick-start.ipynb | 449 ------------------ 1 file changed, 449 deletions(-) delete mode 100644 docs/sphinx/source/examples/text-search-quick-start.ipynb diff --git a/docs/sphinx/source/examples/text-search-quick-start.ipynb b/docs/sphinx/source/examples/text-search-quick-start.ipynb deleted file mode 100644 index fc481d01..00000000 --- a/docs/sphinx/source/examples/text-search-quick-start.ipynb +++ /dev/null @@ -1,449 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "given-adoption", - "metadata": {}, - "source": [ - "![Vespa logo](https://vespa.ai/assets/vespa-logo-color.png)\n", - "\n", - "# Text search\n", - "\n", - "This self-contained tutorial will create a basic text search application based on the MS MARCO dataset,\n", - "similar to Vespa's [text search tutorials](https://docs.vespa.ai/en/tutorials/text-search.html).\n", - "\n", - "[Install pyvespa](https://pyvespa.readthedocs.io/) and start Docker, validate minimum 4G available:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "03f3d0f2", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " Total Memory: 15.64GiB\r\n" - ] - } - ], - "source": [ - "!docker info | grep \"Total Memory\"" - ] - }, - { - "cell_type": "markdown", - "id": "db637322", - "metadata": {}, - "source": [ - "## Create an application package\n", - "\n", - "Create an [application package](https://pyvespa.readthedocs.io/en/latest/reference-api.html#vespa.package.ApplicationPackage) - do not use a `-` in the name:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "bd5c2629", - "metadata": {}, - "outputs": [], - "source": [ - "from vespa.package import ApplicationPackage\n", - "\n", - "app_package = ApplicationPackage(name=\"textsearch\")" - ] - }, - { - "cell_type": "markdown", - "id": "caa46d5a", - "metadata": {}, - "source": [ - "## Add fields to the schema\n", - "\n", - "Add [fields](https://pyvespa.readthedocs.io/en/latest/reference-api.html#vespa.package.Field)\n", - "to the application's [schema](https://pyvespa.readthedocs.io/en/latest/reference-api.html#vespa.package.Schema):" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f2d0bea7", - "metadata": {}, - "outputs": [], - "source": [ - "from vespa.package import Field\n", - "\n", - "app_package.schema.add_fields(\n", - " Field(name = \"id\", type = \"string\", indexing = [\"attribute\", \"summary\"]),\n", - " Field(name = \"title\", type = \"string\", indexing = [\"index\", \"summary\"], index = \"enable-bm25\"),\n", - " Field(name = \"body\", type = \"string\", indexing = [\"index\", \"summary\"], index = \"enable-bm25\")\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "0a4199cb", - "metadata": {}, - "source": [ - "* `id` holds the document ids, while `title` and `body` are the text fields of the documents.\n", - "\n", - "* Setting `\"index\"` in `indexing` means that a searchable index for `title` and `body` is created.\n", - " Read more about [indexing options](https://docs.vespa.ai/en/reference/schema-reference.html#indexing). \n", - "\n", - "* Setting `index = \"enable-bm25\"` will pre-compute quantities to make it fast to compute the BM25 score." - ] - }, - { - "cell_type": "markdown", - "id": "e1757dea", - "metadata": {}, - "source": [ - "## Search multiple fields when querying\n", - "\n", - "A [FieldSet](https://pyvespa.readthedocs.io/en/latest/reference-api.html#vespa.package.FieldSet)\n", - "groups fields together for searching -\n", - "it configures queries to look for matches both in the titles and bodies of the documents:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d0ecbb27", - "metadata": {}, - "outputs": [], - "source": [ - "from vespa.package import FieldSet\n", - "\n", - "app_package.schema.add_field_set(\n", - " FieldSet(name = \"default\", fields = [\"title\", \"body\"])\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "6573388c", - "metadata": {}, - "source": [ - "## Define how to rank the documents matched\n", - "\n", - "Specify how to rank the matched documents by defining a\n", - "[RankProfile](https://pyvespa.readthedocs.io/en/latest/reference-api.html#vespa.package.RankProfile).\n", - "Here, the `bm25` rank profile combines BM25 scores from `title` and `body`:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9bb438f4", - "metadata": {}, - "outputs": [], - "source": [ - "from vespa.package import RankProfile\n", - "\n", - "app_package.schema.add_rank_profile(\n", - " RankProfile(name = \"default\", first_phase = \"bm25(title) + bm25(body)\")\n", - ")\n", - "app_package.schema.add_rank_profile(\n", - " RankProfile(name = \"bm25\", first_phase = \"bm25(title) + bm25(body)\")\n", - ")\n", - "app_package.schema.add_rank_profile(\n", - " RankProfile(name = \"native_rank\", first_phase = \"nativeRank(title, body)\")\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "careful-savage", - "metadata": {}, - "source": [ - "## Deploy\n", - "\n", - "The text search app with fields, a fieldset to group fields together, and a rank profile to rank matched documents is now defined and ready to deploy.\n", - "Deploy `app_package` on the local machine using Docker,\n", - "without leaving the notebook, by creating an instance of\n", - "[VespaDocker](https://pyvespa.readthedocs.io/en/latest/reference-api.html#vespa.deployment.VespaDocker):" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "canadian-blood", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Waiting for configuration server, 0/300 seconds...\n", - "Waiting for configuration server, 5/300 seconds...\n", - "Waiting for application status, 0/300 seconds...\n", - "Waiting for application status, 5/300 seconds...\n", - "Waiting for application status, 10/300 seconds...\n", - "Waiting for application status, 15/300 seconds...\n", - "Waiting for application status, 20/300 seconds...\n", - "Waiting for application status, 25/300 seconds...\n", - "Finished deployment.\n" - ] - } - ], - "source": [ - "import os\n", - "from vespa.deployment import VespaDocker\n", - "\n", - "vespa_docker = VespaDocker()\n", - "app = vespa_docker.deploy(application_package=app_package)" - ] - }, - { - "cell_type": "markdown", - "id": "aaae2f91", - "metadata": {}, - "source": [ - "`app` now holds a [Vespa](https://pyvespa.readthedocs.io/en/latest/reference-api.html#vespa.application.Vespa) instance,\n", - "to be used to interact with the application.\n", - "`pyvespa` provides an API to define Vespa application packages from python.\n", - "`vespa_docker.deploy` exports Vespa configuration files to `disk_folder` -\n", - "going through these is a good way to learning about Vespa configuration." - ] - }, - { - "cell_type": "markdown", - "id": "sealed-mustang", - "metadata": {}, - "source": [ - "## Feed\n", - "\n", - "Download approx 10K documents:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "executed-reservoir", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
idtitlebody
0D1712962Can you eat crab or imitation krab when you ha...Answers com Wiki Answers Categories Health...
1D1817294How long is a tax refund check goodAnswers com Wiki Answers Categories Busine...
2D1761039The Suffolk Resolves 1774The Suffolk Resolves 1774 Across New England ...
3D2899268The eagle has flownDownload citation Share Download full text PDF...
4D327848122b Cotton and African American Life22b Cotton and African American Life Two thi...
\n", - "
" - ], - "text/plain": [ - " id title \\\n", - "0 D1712962 Can you eat crab or imitation krab when you ha... \n", - "1 D1817294 How long is a tax refund check good \n", - "2 D1761039 The Suffolk Resolves 1774 \n", - "3 D2899268 The eagle has flown \n", - "4 D3278481 22b Cotton and African American Life \n", - "\n", - " body \n", - "0 Answers com Wiki Answers Categories Health... \n", - "1 Answers com Wiki Answers Categories Busine... \n", - "2 The Suffolk Resolves 1774 Across New England ... \n", - "3 Download citation Share Download full text PDF... \n", - "4 22b Cotton and African American Life Two thi... " - ] - }, - "execution_count": null, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from pandas import read_csv\n", - "\n", - "docs = read_csv(\n", - " filepath_or_buffer=\"https://data.vespa.oath.cloud/blog/msmarco/sample_docs.csv\"\n", - ")\n", - "docs.head()" - ] - }, - { - "cell_type": "markdown", - "id": "4f0ca33f", - "metadata": {}, - "source": [ - "Feed the DataFrame to the application:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "bottom-memorabilia", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Successful documents fed: 992/1000.\n", - "Batch progress: 1/10.\n", - "Successful documents fed: 992/1000.\n", - "Batch progress: 2/10.\n", - "Successful documents fed: 987/1000.\n", - "Batch progress: 3/10.\n", - "Successful documents fed: 992/1000.\n", - "Batch progress: 4/10.\n", - "Successful documents fed: 995/1000.\n", - "Batch progress: 5/10.\n", - "Successful documents fed: 993/1000.\n", - "Batch progress: 6/10.\n", - "Successful documents fed: 995/1000.\n", - "Batch progress: 7/10.\n", - "Successful documents fed: 991/1000.\n", - "Batch progress: 8/10.\n", - "Successful documents fed: 993/1000.\n", - "Batch progress: 9/10.\n", - "Successful documents fed: 957/963.\n", - "Batch progress: 10/10.\n" - ] - } - ], - "source": [ - "feed_res = app.feed_df(docs)" - ] - }, - { - "cell_type": "markdown", - "id": "separated-insertion", - "metadata": {}, - "source": [ - "## Query\n", - "\n", - "Query the text search app using the [Vespa Query language](https://docs.vespa.ai/en/query-language.html)\n", - "by sending the parameters to the body argument of\n", - "[app.query](https://pyvespa.readthedocs.io/en/latest/reference-api.html#vespa.application.Vespa.query):" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "82b5f6fe", - "metadata": {}, - "outputs": [], - "source": [ - "query = {\n", - " 'yql': 'select * from sources * where userQuery()',\n", - " 'query': 'what keeps planes in the air',\n", - " 'ranking': 'bm25',\n", - " 'type': 'all',\n", - " 'hits': 10\n", - "}" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c930ef28", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'id': 'id:textsearch:textsearch::D1871659',\n", - " 'relevance': 25.629646778721735,\n", - " 'source': 'textsearch_content',\n", - " 'fields': {'sddocname': 'textsearch',\n", - " 'documentid': 'id:textsearch:textsearch::D1871659',\n", - " 'id': 'D1871659',\n", - " 'title': 'What keeps airplanes in the air ',\n", - " 'body': 'Answers com Wiki Answers Categories Cars Vehicles Airplanes and Aircraft What keeps airplanes in the air Flag What keeps airplanes in the air Answer by Karin L Confidence votes 95 0KThere s more to raising cattle than throwing them out to pasture Know your soil and plants to earn profit above ground and wealth below It is the combined forces of lift thrust and weight that keeps an airplane in the air Lift happens to be the largest force in this equation and is dependent on the speed of the wing or how fast an airplane is going vertical velocity of air and air density Well the elevator the rudder will help and something else I forgot what it was but don t judge me for that And that s how you be a bow done Like a boss Boss 15 people found this useful Was this answer useful Yes Somewhat No How do airplane windows keep out the cold Airplane windows The only way that heat can escape the warm cabin is to travel through something or radiate outward Since the windows are so small the radiation through Karin L There s more to raising cattle than throwing them out to pasture Know your soil and plants to earn profit above ground and wealth below Does speed keep an airplane in the air Yes to a degree speed is part of the equation Speed thrust and combine that with lift and weight though weight has to be smaller than lift and thrust combined then y Bala Surya 152 866 Contributions Adventurous Fun Dreaming High How does bernoulli s principle keep airplanes in the air Bernoulli s principle is that there is a region of high pressure under the wing So air rushes under the plane So it creates lift which in turn keeps the airplane in the air How airplane can fly in the air The airplane fly on the air by 4 main forces drag lift thrust and weight all these forces affect of the performances of the airplane to fly the high power of the e David Bäckman 388 346 Contributions Knowledge is a thing you can both share and keep Is the force that keeps an airplane in the air called lift or levitation Lift Sadia rulez 1 Contribution How does air help an airplane fly Air Helps An Aeroplane Fly Because Of The Up Thurst Up Thrust Is A Sort Of A Gravity That Pulls You Up Like A Float Floats In Water The Upthrust Pulls It Up But The Gravity Pull Djlax97 3 Contributions How do you keep your ears from popping on an airplane All you have to do is chew gum and swallow a lot Doing this has something to do with the place of your throat And yes it does work What does it mean when the air in airplanes are pressurized The air in an aircraft needs to be pressurised so that the people within the cabin don t pass out from oxygen starvation at higher altitudes The atmosphere can be described a Richard Loberger 26 278 Contributions Airplane can stop in the air It would depend on what you mean by stop in the air An airplane can have 0 MPH ground speed while in the air only IF the wind is going faster then the stall speed of the g How does an airplane stay stable in the air to keep a plane stable in the air it has different control surfaces or panels to allow the pilot to adjust the position of the plane in the air Some modern fighter jets such How do you recharge an airplane Air Conditioner An airplane airconditioner is completely different than the one in your house or car It doesn t rely on a refrigerant Rather it takes hot high pressure air from the hot comp What keeps an airplane up in the sky Bernoulli s Principle the statement that an increase in the speed of a fluid produces a decrease in pressure and a decrease in the speed produces an increase in pressure Win Karin L There s more to raising cattle than throwing them out to pasture Know your soil and plants to earn profit above ground and wealth below Answered In Physics What keeps the airplane from rolling unexpectedly On the tarmac there are triangular blocks that are placed in front and behind each wheel of the airplane called wheel chocks In the air a steady hand on the control sti David Bäckman 388 346 Contributions Knowledge is a thing you can both share and keep Answered In Airplanes and Aircraft What is a machine that keeps an airplane on course An auto pilot Charlie N 122 923 Contributions I have spent many years renovating buildings and leading a commercial handyman crew Answered In Airbus Machine that keeps an airplane on course Autopilot keeps an aircraft on course In modern times autopilot is assisted by GPS and radar Answered In Airplanes and Aircraft What keeps an airplane moving forward An engine producing THRUST keeps an airplane moving forward Types of engines used by airplanes include reciprocating engines turbo prop engines turbojet and turbofan engin Levyharaivan 396 Contributions Answered In Airplanes and Aircraft What keeps a airplane from rolling unexpectedly Brakes just like any other vehicle'}}" - ] - }, - "execution_count": null, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "res = app.query(body=query)\n", - "res.hits[0]" - ] - }, - { - "cell_type": "markdown", - "id": "28591491", - "metadata": {}, - "source": [ - "## Cleanup" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e5064bd2", - "metadata": {}, - "outputs": [], - "source": [ - "vespa_docker.container.stop()\n", - "vespa_docker.container.remove()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -}