Use poetry to manage the Python runtime project (#753)

langstream-runtime/langstream-runtime-base-docker-image/src/main/docker/Dockerfile

@@ -30,7 +30,7 @@ RUN echo 'Acquire::http::Timeout "30";\nAcquire::ftp::Timeout "30";\nAcquire::Re
 RUN apt-get update && add-apt-repository ppa:deadsnakes/ppa \
       && apt-get -y install --no-install-recommends python3.11-full \
       && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 \
-      && python3 -m ensurepip && python3 -m pip install pipenv
+      && python3 -m ensurepip && python3 -m pip install poetry-plugin-export
 
 
 RUN apt-get update \

langstream-runtime/langstream-runtime-impl/pom.xml

@@ -378,12 +378,10 @@
                 <copydir src="${basedir}/src/main/python/langstream" dest="${basedir}/target/python/langstream" />
                 <copydir src="${basedir}/src/main/python/langstream_grpc" dest="${basedir}/target/python/langstream_grpc" />
                 <copy file="${basedir}/src/main/python/LICENSE" todir="${basedir}/target/python" />
-                <copy file="${basedir}/src/main/python/Pipfile" todir="${basedir}/target/python" />
-                <copy file="${basedir}/src/main/python/Pipfile.lock" todir="${basedir}/target/python" />
+                <copy file="${basedir}/src/main/python/poetry.lock" todir="${basedir}/target/python" />
                 <copy file="${basedir}/src/main/python/pyproject.toml" todir="${basedir}/target/python" />
                 <replaceregexp file="${basedir}/target/python/pyproject.toml" match="version = &quot;0.0.0&quot;" replace="version=&quot;${pythonpackage.version}&quot;" />
                 <copy file="${basedir}/src/main/python/README.md" todir="${basedir}/target/python" />
-                <copy file="${basedir}/src/main/python/setup.py" todir="${basedir}/target/python" />
                 <copy file="${basedir}/src/main/python/tox.ini" todir="${basedir}/target/python" />
               </target>
             </configuration>
@@ -444,10 +442,9 @@
             </goals>
             <configuration>
               <workingDirectory>${project.build.directory}/python</workingDirectory>
-              <executable>python</executable>
+              <executable>poetry</executable>
               <skip>${skipPythonPackage}</skip>
               <arguments>
-                <argument>-m</argument>
                 <argument>build</argument>
               </arguments>
             </configuration>

langstream-runtime/langstream-runtime-impl/src/main/assemble/langstream-runtime.xml

@@ -51,7 +51,8 @@
     <fileSet>
       <directory>src/main/python/</directory>
       <includes>
-        <include>Pipfile.lock</include>
+        <include>pyproject.toml</include>
+        <include>poetry.lock</include>
       </includes>
       <outputDirectory>.</outputDirectory>
       <lineEnding>unix</lineEnding>

langstream-runtime/langstream-runtime-impl/src/main/docker/Dockerfile

@@ -21,12 +21,13 @@ ARG DEBIAN_FRONTEND=noninteractive
 
 RUN mkdir /app && chmod g+w /app
 
-ADD maven/Pipfile.lock /app/Pipfile.lock
+ADD maven/pyproject.toml /app/pyproject.toml
+ADD maven/poetry.lock /app/poetry.lock
 
 ENV NLTK_DATA="/app/nltk_data"
 
 # Install python runtime deps
-RUN cd /app && pipenv requirements --categories="packages full" > /app/requirements.txt \
+RUN cd /app && poetry export --with full -f requirements.txt --output /app/requirements.txt \
      && python3 -m pip install -r /app/requirements.txt \
      && python3 -m nltk.downloader -d /app/nltk_data punkt averaged_perceptron_tagger
 

langstream-runtime/langstream-runtime-impl/src/main/python/README.md

@@ -2,8 +2,8 @@
 
 This package contains helper classes to build LangStream agents.
 
-## Generate new Pipfile.lock
+## Generate new poetry.lock
 
 ```
-pipenv lock  
+poetry lock  
 ```

langstream-runtime/langstream-runtime-impl/src/main/python/pyproject.toml

@@ -15,19 +15,22 @@
 #
 
 [build-system]
-requires = ["setuptools>=61.0"]
-build-backend = "setuptools.build_meta"
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"
 
-[project]
+[tool.poetry]
 name = "langstream-ai"
 # === Warning: do not touch this section. The package version is set by regexp
 version = "0.0.0"
 # ===
-authors = [
-    {name = "Christophe Bornet", email = "[email protected]"}
-]
 description = "LangStream user API"
-requires-python = ">=3.5"
+license = "Apache-2.0"
+authors = ["LangStream"]
+readme = ["README.md"]
+homepage = "https://langstream.ai"
+repository = "https://github.com/LangStream/langstream"
+documentation = "https://docs.langstream.ai"
+keywords = ["streaming", "ai", "llm"]
 classifiers = [
     "Intended Audience :: Developers",
     "License :: OSI Approved :: Apache Software License",
@@ -46,16 +49,43 @@ classifiers = [
     "Topic :: Scientific/Engineering",
     "Topic :: Scientific/Engineering :: Artificial Intelligence",
 ]
-keywords = ["streaming", "ai", "llm"]
-dynamic = ["readme"]
+packages = [
+    { include = "langstream" }
+]
 
-[project.urls]
-"Homepage" = "https://github.com/LangStream/langstream"
+[tool.poetry.urls]
 "Bug Tracker" = "https://github.com/LangStream/langstream/issues"
 
-[tool.setuptools]
-packages = ["langstream"]
-py-modules = ['langstream.api', 'langstream.util']
+[tool.poetry.dependencies]
+python = "^3.8.1"
+grpcio = "^1.60.0"
+protobuf = "^4.25.1"
+fastavro = "^1.9.2"
+
+[tool.poetry.group.full]
+optional = true
+
+[tool.poetry.group.full.dependencies]
+openai = {extras = ["datalib"], version = "^1.6.1"}
+langchain = {extras = ["openai"], version = "^0.0.353"}
+langserve = "^0.0.37"
+fastapi = "^0.108.0"
+uvicorn = "^0.25.0"
+boto3 = "^1.34.11"
+unstructured = "^0.11.6"
+llama-index = "^0.9.24"
+cassio = "^0.1.3"
+
+# workaround for https://github.com/python-poetry/poetry-plugin-export/issues/183
+urllib3 = "<2"
+
+[tool.poetry.group.dev.dependencies]
+tox = "^4.11.4"
+black = "^23.12.1"
+ruff = "^0.1.9"
+pytest = "^7.4.4"
+pytest-asyncio = "^0.23.3"
+grpcio-tools = "^1.60.0"
 
 [tool.ruff]
 exclude = [

langstream-runtime/langstream-runtime-impl/src/main/python/tox.ini

@@ -6,13 +6,10 @@ env_list = lint, py311
 
 [testenv]
 description = run unit tests
-deps =
-    pipenv
-    pytest
-    pytest-asyncio
+allowlist_externals = poetry
 commands =
-    pipenv sync
-    pytest {posargs:langstream_grpc}
+    poetry install --no-root --sync
+    poetry run pytest {posargs:langstream_grpc}
 
 [testenv:lint]
 description = lint

pom.xml

@@ -680,8 +680,7 @@
                 <exclude>**/*.tpl</exclude>
                 <exclude>**/.idea/**</exclude>
                 <exclude>**/.mvn/**</exclude>
-                <exclude>**/Pipfile</exclude>
-                <exclude>**/Pipfile.lock</exclude>
+                <exclude>**/poetry.lock</exclude>
                 <exclude>**/*.ini</exclude>
                 <exclude>**/venv/**</exclude>
                 <exclude>**/.tox/**</exclude>

requirements.txt

@@ -1,4 +1,4 @@
 tox
 black
 ruff
-build
+poetry