New Python client and Jupyter extension

CHANGELOG.md

@@ -11,6 +11,14 @@ Recommendation: for ease of reading, use the following order:
 - Fixed
 -->
 
+## [Unreleased]
+### Changed
+- Flight SQL protocol now fully support authentication (anonymous and bearer token)
+- The `kamu notebook` command now defaults to `DataFusion` engine for speed, but you can switch to Spark with `--engine spark` argument
+- The `kamu notebook` command uses new image based on latest Jupyter and new [`kamu-client-python`](https://github.com/kamu-data/kamu-client-python) library
+- The `kamu sql server` command interface changed to use `--engine datafusion/spark`, removing the `--flight-sql` flag
+- Examples in `example/flightsql/python` were updated to new auth and showcasing `kamu` Python library
+
 ## [0.215.1] - 2024-12-30
 ### Fixed
 - GraphQL: in a multi-tenant workspace, `datasets.createEmpty` and `datasets.createFromSnapshot` mutations now return dataset aliases prefixed with account name.

examples/archive/commercial-fishing/.kamuconfig

@@ -0,0 +1,14 @@
+kind: CLIConfig
+version: 1
+content:
+  users:
+    predefined:
+      - accountName: kamu
+        isAdmin: true
+        avatarUrl: https://avatars.githubusercontent.com/u/50896974?s=200&v=4
+      - accountName: acme.fishing.co
+        accountType: Organization
+        avatarUrl: https://cdn-icons-png.flaticon.com/512/1090/1090630.png
+      - accountName: globalfishingwatch.org
+        accountType: Organization
+        avatarUrl: https://cdn-icons-png.flaticon.com/512/744/744480.png

examples/archive/commercial-fishing/init.sh

@@ -4,6 +4,7 @@ set -e
 KAMU_NODE_URL="odf+https://node.demo.kamu.dev/"
 
 kamu init --multi-tenant --exists-ok
+cp -f .kamuconfig .kamu/
 
 kamu --account acme.fishing.co pull "${KAMU_NODE_URL}acme.fishing.co/vessels.gps"
 kamu --account acme.fishing.co pull "${KAMU_NODE_URL}acme.fishing.co/vessels.trawl"

examples/archive/commercial-fishing/analysis.ipynb → examples/archive/commercial-fishing/notebook.ipynb

@@ -13,15 +13,14 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "f0d0f5b5",
+   "id": "d5947b7d-4cfd-4a8b-b12f-c1a76402438b",
    "metadata": {},
    "outputs": [],
    "source": [
-    "%import_dataset acme.fishing.co/vessels.gps --alias gps\n",
-    "%import_dataset acme.fishing.co/vessels.trawl --alias trawl\n",
-    "%import_dataset acme.fishing.co/vessels.fuel --alias fuel\n",
-    "%import_dataset acme.fishing.co/vessels.location-annotated --alias loc\n",
-    "%import_dataset globalfishingwatch.org/protected-areas --alias areas"
+    "import kamu\n",
+    "import kamu.utils\n",
+    "\n",
+    "con = kamu.connect(engine=\"spark\")"
    ]
   },
   {
@@ -31,14 +30,12 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%%local\n",
     "import os\n",
     "import pandas as pd\n",
     "import plotly.graph_objects as go\n",
     "import plotly.express as px\n",
     "from mapboxgl.viz import *\n",
     "from mapboxgl.utils import *\n",
-    "from utils.plotting import *\n",
     "\n",
     "# Must be a public token, starting with `pk`\n",
     "token = os.getenv('MAPBOX_ACCESS_TOKEN')\n",
@@ -68,7 +65,7 @@
     "    longitude,\n",
     "    latitude,\n",
     "    is_trawling\n",
-    "from loc"
+    "from `acme.fishing.co/vessels.location-annotated`"
    ]
   },
   {
@@ -78,7 +75,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%%local\n",
     "fig = go.Figure()\n",
     "\n",
     "for vessel_name in gps['vessel_name'].unique():\n",
@@ -136,7 +132,7 @@
     "    date,\n",
     "    wdpa_pid,\n",
     "    gis_area\n",
-    "from areas"
+    "from `globalfishingwatch.org/protected-areas`"
    ]
   },
   {
@@ -146,9 +142,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%%local\n",
     "viz = ChoroplethViz(\n",
-    "    df_to_geojson(areas),\n",
+    "    kamu.utils.df_to_geojson(areas),\n",
     "    style=mapbox_style,\n",
     "    center=(2, 51),\n",
     "    zoom=5,\n",
@@ -186,7 +181,8 @@
     "    name,\n",
     "    gis_area,\n",
     "    geometry\n",
-    "from areas where parent_iso in (\"NLD\", \"FRA\", \"DMK\", \"BEL\")"
+    "from `globalfishingwatch.org/protected-areas`\n",
+    "where parent_iso in (\"NLD\", \"FRA\", \"DMK\", \"BEL\")"
    ]
   },
   {
@@ -197,22 +193,27 @@
    "outputs": [],
    "source": [
     "%%sql -o isect -q\n",
-    "select\n",
-    "    gps.event_time,\n",
-    "    gps.vessel_name,\n",
-    "    gps.latitude,\n",
-    "    gps.longitude\n",
-    "from (\n",
+    "with location_trawling as (\n",
     "    select\n",
     "         event_time, vessel_name, latitude, longitude, st_point(longitude, latitude) as geometry \n",
-    "    from loc where is_trawling = 1\n",
-    ") gps,\n",
-    "(\n",
+    "    from `acme.fishing.co/vessels.location-annotated`\n",
+    "    where is_trawling = 1\n",
+    "),\n",
+    "protected_areas as (\n",
     "    select\n",
     "        st_geomfromgeojson(geometry) as geometry\n",
-    "    from areas where parent_iso = \"NLD\"\n",
-    ") areas\n",
-    "where st_contains(areas.geometry, gps.geometry)"
+    "    from `globalfishingwatch.org/protected-areas`\n",
+    "    where parent_iso = \"NLD\"\n",
+    ")\n",
+    "select\n",
+    "    loc.event_time,\n",
+    "    loc.vessel_name,\n",
+    "    loc.latitude,\n",
+    "    loc.longitude\n",
+    "from\n",
+    "    location_trawling as loc,\n",
+    "    protected_areas as area\n",
+    "where st_contains(area.geometry, loc.geometry)"
    ]
   },
   {
@@ -222,8 +223,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%%local\n",
-    "isect_areas_geojson = df_to_geojson(isect_areas)\n",
+    "isect_areas_geojson = kamu.utils.df_to_geojson(isect_areas)\n",
     "\n",
     "fig = go.Figure()\n",
     "\n",
@@ -283,19 +283,21 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "PySpark",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
-   "name": "pysparkkernel"
+   "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {
-    "name": "python",
+    "name": "ipython",
     "version": 3
    },
    "file_extension": ".py",
    "mimetype": "text/x-python",
-   "name": "pyspark",
-   "pygments_lexer": "python3"
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.8"
   }
  },
  "nbformat": 4,

examples/archive/water-management/.kamuconfig

@@ -0,0 +1,12 @@
+kind: CLIConfig
+version: 1
+content:
+  users:
+    predefined:
+      - accountName: kamu
+        isAdmin: true
+        avatarUrl: https://avatars.githubusercontent.com/u/50896974?s=200&v=4
+      - accountName: rijkswaterstaat.nl
+        avatarUrl: https://www.shutterstock.com/image-vector/royal-exclusive-badge-logo-two-260nw-236025661.jpg
+      - accountName: deltares.nl
+        avatarUrl: https://avatars.githubusercontent.com/u/6613768?s=200&v=4

examples/archive/water-management/init.sh

@@ -4,6 +4,7 @@ set -e
 KAMU_NODE_URL="odf+https://node.demo.kamu.dev/"
 
 kamu init --multi-tenant --exists-ok
+cp -f .kamuconfig .kamu/
 
 kamu --account rijkswaterstaat.nl pull "${KAMU_NODE_URL}rijkswaterstaat.nl/stations"
 kamu --account rijkswaterstaat.nl pull "${KAMU_NODE_URL}rijkswaterstaat.nl/measurements.boven-rijn"