Add a way to export SQLAlchemy models from Gel (EdgeDB). (#536)

Create separate intermediate objects to represent links with link
properties (because ORMs tend to view them as such).

Don't allow crazy names (not usual type of identifiers).Multi properties behave more like plain multi links than anything else
(because they have a separate table with the property value in it),
so they should be reflected like that establishing a relationship.

Reflect Gel modules into Python modules.

Handle backlink collisions for SQLAlchemy model generator.

Add tests that setup a Gel database and then generate the SQLAlchemy
models from it. The individual tests use a SQLAlchemy session to access
the database using postgres protocol.Add a CLI for generating ORM models.

Run `gel-orm --help` for more details

.flake8

@@ -1,3 +1,3 @@
 [flake8]
 ignore = B008,B023,B306,E203,E402,E731,D100,D101,D102,D103,D104,D105,W503,W504,E252,F999,F541
-exclude = .git,__pycache__,build,dist,.eggs
+exclude = .git,__pycache__,build,dist,.eggs,generated

.github/workflows/tests.yml

@@ -78,9 +78,15 @@ jobs:
 
     - name: Install Python Deps
       if: steps.release.outputs.version == 0
+      env:
+        PYTHON_VERSION: ${{ matrix.python-version }}
       run: |
         python -m pip install --upgrade setuptools pip wheel
-        python -m pip install -e .[test]
+        if [ "${PYTHON_VERSION}" = "3.10" -o "${PYTHON_VERSION}" = "3.11" -o "${PYTHON_VERSION}" = "3.12" ]; then
+          python -m pip install -e .[test,sqltest]
+        else
+          python -m pip install -e .[test]
+        fi
 
     - name: Test
       if: steps.release.outputs.version == 0

gel/_testbase.py

@@ -21,6 +21,7 @@
 import atexit
 import contextlib
 import functools
+import importlib.util
 import inspect
 import json
 import logging
@@ -35,6 +36,8 @@
 import gel
 from gel import asyncio_client
 from gel import blocking_client
+from gel.orm.introspection import get_schema_json
+from gel.orm.sqla import ModelGenerator
 
 
 log = logging.getLogger(__name__)
@@ -444,6 +447,7 @@ class DatabaseTestCase(ClusterTestCase, ConnectedTestCaseMixin):
     SETUP = None
     TEARDOWN = None
     SCHEMA = None
+    DEFAULT_MODULE = 'test'
 
     SETUP_METHOD = None
     TEARDOWN_METHOD = None
@@ -521,15 +525,18 @@ def get_database_name(cls):
     @classmethod
     def get_setup_script(cls):
         script = ''
+        schema = []
 
         # Look at all SCHEMA entries and potentially create multiple
-        # modules, but always create the 'test' module.
-        schema = ['\nmodule test {}']
+        # modules, but always create the test module, if not `default`.
+        if cls.DEFAULT_MODULE != 'default':
+            schema.append(f'\nmodule {cls.DEFAULT_MODULE} {{}}')
         for name, val in cls.__dict__.items():
             m = re.match(r'^SCHEMA(?:_(\w+))?', name)
             if m:
-                module_name = (m.group(1) or 'test').lower().replace(
-                    '__', '.')
+                module_name = (
+                    m.group(1) or cls.DEFAULT_MODULE
+                ).lower().replace('_', '::')
 
                 with open(val, 'r') as sf:
                     module = sf.read()
@@ -623,6 +630,54 @@ def adapt_call(cls, result):
         return result
 
 
+class SQLATestCase(SyncQueryTestCase):
+    SQLAPACKAGE = None
+    DEFAULT_MODULE = 'default'
+
+    @classmethod
+    def setUpClass(cls):
+        # SQLAlchemy relies on psycopg2 to connect to Postgres and thus we
+        # need it to run tests. Unfortunately not all test environemnts might
+        # have psycopg2 installed, as long as we run this in the test
+        # environments that have this, it is fine since we're not expecting
+        # different functionality based on flavours of psycopg2.
+        if importlib.util.find_spec("psycopg2") is None:
+            raise unittest.SkipTest("need psycopg2 for ORM tests")
+
+        super().setUpClass()
+
+        class_set_up = os.environ.get('EDGEDB_TEST_CASES_SET_UP')
+        if not class_set_up:
+            # Now that the DB is setup, generate the SQLAlchemy models from it
+            spec = get_schema_json(cls.client)
+            # We'll need a temp directory to setup the generated Python
+            # package
+            cls.tmpsqladir = tempfile.TemporaryDirectory()
+            gen = ModelGenerator(
+                outdir=os.path.join(cls.tmpsqladir.name, cls.SQLAPACKAGE),
+                basemodule=cls.SQLAPACKAGE,
+            )
+            gen.render_models(spec)
+            sys.path.append(cls.tmpsqladir.name)
+
+    @classmethod
+    def tearDownClass(cls):
+        super().tearDownClass()
+        # cleanup the temp modules
+        sys.path.remove(cls.tmpsqladir.name)
+        cls.tmpsqladir.cleanup()
+
+    @classmethod
+    def get_dsn_for_sqla(cls):
+        cargs = cls.get_connect_args(database=cls.get_database_name())
+        dsn = (
+            f'postgresql://{cargs["user"]}:{cargs["password"]}'
+            f'@{cargs["host"]}:{cargs["port"]}/{cargs["database"]}'
+        )
+
+        return dsn
+
+
 _lock_cnt = 0
 
 

gel/orm/cli.py

@@ -0,0 +1,86 @@
+#
+# This source file is part of the EdgeDB open source project.
+#
+# Copyright 2024-present MagicStack Inc. and the EdgeDB authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+import argparse
+
+import gel
+
+from gel.codegen.generator import _get_conn_args
+from .introspection import get_schema_json
+from .sqla import ModelGenerator
+
+
+class ArgumentParser(argparse.ArgumentParser):
+    def error(self, message):
+        self.exit(
+            2,
+            f"error: {message:s}\n",
+        )
+
+
+parser = ArgumentParser(
+    description="Generate Python ORM code for accessing a Gel database."
+)
+parser.add_argument(
+    "orm",
+    choices=['sqlalchemy', 'django'],
+    help="Pick which ORM to generate models for.",
+)
+parser.add_argument("--dsn")
+parser.add_argument("--credentials-file", metavar="PATH")
+parser.add_argument("-I", "--instance", metavar="NAME")
+parser.add_argument("-H", "--host")
+parser.add_argument("-P", "--port")
+parser.add_argument("-d", "--database", metavar="NAME")
+parser.add_argument("-u", "--user")
+parser.add_argument("--password")
+parser.add_argument("--password-from-stdin", action="store_true")
+parser.add_argument("--tls-ca-file", metavar="PATH")
+parser.add_argument(
+    "--tls-security",
+    choices=["default", "strict", "no_host_verification", "insecure"],
+)
+parser.add_argument(
+    "--out",
+    help="The output directory for the generated files.",
+    required=True,
+)
+parser.add_argument(
+    "--mod",
+    help="The fullname of the Python module corresponding to the output "
+         "directory.",
+    required=True,
+)
+
+
+def main():
+    args = parser.parse_args()
+    # setup client
+    client = gel.create_client(**_get_conn_args(args))
+    spec = get_schema_json(client)
+
+    match args.orm:
+        case 'sqlalchemy':
+            gen = ModelGenerator(
+                outdir=args.out,
+                basemodule=args.mod,
+            )
+            gen.render_models(spec)
+        case 'django':
+            print('Not available yet. Coming soon!')