Merge branch 'master' into qa

.github/workflows/build_breadbox.yml

@@ -211,6 +211,12 @@ jobs:
           key: venv-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('**/poetry.lock') }}
       - name: Generate breadbox client
         uses: ./.github/actions/generate-breadbox-client # defined as a re-usable action
+        # the above does a checkout which deletes the credential file, so we need to execute authenticate to google cloud after it
+      - name: Authenticate to Google Cloud
+        uses: google-github-actions/auth@v2
+        with:
+          # See instructions here: https://github.com/google-github-actions/auth?tab=readme-ov-file#service-account-key-json
+          credentials_json: ${{ secrets.DEPMAP_ARTIFACTS_SVC_ACCT }}
       - name: Set up for publishing breadbox client
         working-directory: ./breadbox-client
         run: |

.github/workflows/publish_breadbox_client.yaml

@@ -0,0 +1,54 @@
+---
+name: Publish breadbox client
+
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: "The tag/branch to checkout"
+        required: true
+        default: "master"
+jobs:
+  publish-breadbox-client:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out
+        uses: actions/checkout@v3
+        with:
+          token: "${{ secrets.GITHUB_TOKEN }}"
+          fetch-depth: 0
+      - run: |
+          git pull origin HEAD:${{ inputs.tag }}
+      - name: Authenticate to Google Cloud
+        uses: google-github-actions/auth@v2
+        with:
+          # See instructions here: https://github.com/google-github-actions/auth?tab=readme-ov-file#service-account-key-json
+          credentials_json: ${{ secrets.DEPMAP_ARTIFACTS_SVC_ACCT }}
+      - name: Install and configure Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: 1.7.1
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+      - name: Set up cache
+        uses: actions/cache@v2
+        id: cached-poetry-dependencies
+        with:
+          path: .venv
+          key: venv-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('**/poetry.lock') }}
+      - name: Generate breadbox client
+        uses: ./.github/actions/generate-breadbox-client # defined as a re-usable action
+        # the above does a checkout which deletes the credential file, so we need to execute authenticate to google cloud after it
+      - name: Authenticate to Google Cloud
+        uses: google-github-actions/auth@v2
+        with:
+          # See instructions here: https://github.com/google-github-actions/auth?tab=readme-ov-file#service-account-key-json
+          credentials_json: ${{ secrets.DEPMAP_ARTIFACTS_SVC_ACCT }}
+      - name: Set up for publishing breadbox client
+        working-directory: ./breadbox-client
+        run: |
+          poetry self add keyrings.google-artifactregistry-auth
+          poetry config repositories.public-python https://us-central1-python.pkg.dev/cds-artifacts/public-python/
+      - name: Publish new breadbox client version to Artifact Registry
+        working-directory: ./breadbox-client
+        run: poetry publish --build --repository public-python

breadbox-client/breadbox_facade/client.py

@@ -31,28 +31,28 @@
 from breadbox_client.api.groups import remove_group_access as remove_group_access_client
 from breadbox_client.api.groups import get_groups as get_groups_client
 from breadbox_client.api.types import add_dimension_type as add_dimension_type_client
-from breadbox_client.api.types import add_feature_type as add_feature_type_client
-from breadbox_client.api.types import add_sample_type as add_sample_type_client
 from breadbox_client.api.types import get_dimension_type as get_dimension_type_client
 from breadbox_client.api.types import get_dimension_types as get_dimension_types_client
 from breadbox_client.api.types import get_feature_types as get_feature_types_client
 from breadbox_client.api.types import get_sample_types as get_sample_types_client
 from breadbox_client.api.types import remove_dimension_type as remove_dimension_type_client
 from breadbox_client.api.types import update_dimension_type as update_dimension_type_client
-from breadbox_client.api.types import update_feature_type_metadata as update_feature_type_metadata_client
-from breadbox_client.api.types import update_sample_type_metadata as update_sample_type_metadata_client
+from breadbox_client.api.temp import get_associations as get_associations_client
+from breadbox_client.api.temp import add_associations as add_associations_client
+from breadbox_client.api.temp import get_associations_for_slice as get_associations_for_slice_client
+#
+
 from breadbox_client.models import (
     AccessType,
     AddDatasetResponse,
     AddDimensionType,
+    Associations,
+    AssociationTable,
+    AssociationsIn,
+    AssociationsInAxis,
     AddDimensionTypeAxis,
-    AnnotationTypeMap,
     BodyAddDataType,
-    BodyAddFeatureType,
-    BodyAddSampleType,
     BodyGetDatasetData,
-    BodyUpdateFeatureTypeMetadata,
-    BodyUpdateSampleTypeMetadata,
     BodyUploadFile,
     ColumnMetadata,
     ComputeParams,
@@ -66,12 +66,13 @@
     GroupEntry,
     GroupEntryIn,
     GroupOut,
-    IdMapping,
     MatrixDatasetParams,
     MatrixDatasetParamsDatasetMetadataType0,
     MatrixDatasetParamsFormat,
     MatrixDatasetResponse,
     SampleTypeOut,
+    SliceQuery,
+    SliceQueryIdentifierType,
     TableDatasetParams,
     TableDatasetParamsColumnsMetadata,
     TableDatasetParamsDatasetMetadataType0,
@@ -472,6 +473,24 @@ def remove_group_access(self, group_entry_id: str) -> str:
         breadbox_response = remove_group_access_client.sync_detailed(client=self.client, group_entry_id=group_entry_id)
         return self._parse_client_response(breadbox_response)
 
+    # ASSOCIATIONS
+    def get_associations(self) -> List[AssociationTable]:
+        breadbox_response = get_associations_client.sync_detailed(client=self.client)
+        return self._parse_client_response(breadbox_response)
+
+    def add_associations(self, dataset_1_id:str, dataset_2_id: str, axis :str, associations_table_filename : str) -> AssociationTable:
+        with open(associations_table_filename, "rb") as fd:
+            uploaded_file = self.upload_file(fd)
+
+        associations_in = AssociationsIn(axis = AssociationsInAxis(axis), dataset_1_id=dataset_1_id, dataset_2_id=dataset_2_id, file_ids=uploaded_file.file_ids, md5=uploaded_file.md5)
+        breadbox_response = add_associations_client.sync_detailed(client=self.client, body=associations_in)
+
+        return self._parse_client_response(breadbox_response)
+
+    def get_associations_for_slice(self, dataset_id: str, identifier: str, identifier_type: str) -> Associations:
+        breadbox_response = get_associations_for_slice_client.sync_detailed(client=self.client, body=SliceQuery(dataset_id=dataset_id, identifier=identifier,
+                                                                                    identifier_type=SliceQueryIdentifierType(identifier_type)))
+        return self._parse_client_response(breadbox_response)
 
     # API
 

breadbox-client/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "breadbox-client"
-version = "3.18.1"
+version = "3.20.1"
 description = "A client library for accessing Breadbox"
 
 authors = []

breadbox/alembic/versions/52a899219efc_add_associations_tables.py

@@ -0,0 +1,60 @@
+"""add associations tables
+
+Revision ID: 52a899219efc
+Revises: 73587e8936b2
+Create Date: 2025-01-07 17:05:27.637553
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "52a899219efc"
+down_revision = "73587e8936b2"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "precomputed_association",
+        sa.Column("id", sa.String(length=36), nullable=False),
+        sa.Column("dataset_1_id", sa.String(), nullable=False),
+        sa.Column("dataset_2_id", sa.String(), nullable=False),
+        sa.Column("axis", sa.String(), nullable=False),
+        sa.Column("filename", sa.String(), nullable=False),
+        sa.CheckConstraint(
+            "(axis == 'feature') OR (axis == 'sample')", name="ck_assoc_axis"
+        ),
+        sa.ForeignKeyConstraint(
+            ["dataset_1_id"],
+            ["dataset.id"],
+            name=op.f("fk_precomputed_association_dataset_1_id_dataset"),
+        ),
+        sa.ForeignKeyConstraint(
+            ["dataset_2_id"],
+            ["dataset.id"],
+            name=op.f("fk_precomputed_association_dataset_2_id_dataset"),
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_precomputed_association")),
+        sa.UniqueConstraint(
+            "dataset_1_id", "dataset_2_id", "axis", name="assoc_params_uc"
+        ),
+    )
+    with op.batch_alter_table("precomputed_association", schema=None) as batch_op:
+        batch_op.create_index("idx_assoc_dataset_1", ["dataset_1_id"], unique=False)
+        batch_op.create_index("idx_assoc_dataset_2", ["dataset_2_id"], unique=False)
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("precomputed_association", schema=None) as batch_op:
+        batch_op.drop_index("idx_assoc_dataset_2")
+        batch_op.drop_index("idx_assoc_dataset_1")
+
+    op.drop_table("precomputed_association")
+    # ### end Alembic commands ###

breadbox/breadbox/api/temp/__init__.py

@@ -0,0 +1,8 @@
+from .router import router
+
+# import the following which register endpoints onto `router` as a side effect of being imported
+import breadbox.api.temp.cas
+import breadbox.api.temp.context
+import breadbox.api.temp.associations
+
+__all__ = ["router"]

breadbox/breadbox/api/temp/associations.py

@@ -0,0 +1,129 @@
+import os.path
+from typing import Annotated
+
+from depmap_compute.slice import SliceQuery
+from fastapi import Body, Depends
+from itsdangerous import URLSafeSerializer
+
+from breadbox.api.dependencies import get_db_with_user
+from breadbox.config import Settings, get_settings
+from breadbox.db.session import SessionWithUser
+from breadbox.api.uploads import construct_file_from_ids
+from breadbox.schemas.associations import Associations
+from breadbox.schemas.associations import AssociationTable, AssociationsIn
+from typing import List
+from breadbox.service import associations as associations_service
+from breadbox.crud import associations as associations_crud
+import uuid
+from breadbox.db.util import transaction
+from typing import cast, Literal
+
+from .router import router
+
+
+@router.post(
+    "/associations/query-slice",
+    operation_id="get_associations_for_slice",
+    response_model=Associations,
+    response_model_exclude_none=False,
+)
+def query_associations_for_slice(
+    db: Annotated[SessionWithUser, Depends(get_db_with_user)],
+    settings: Annotated[Settings, Depends(get_settings)],
+    slice_query: Annotated[
+        SliceQuery, Body(description="A Data Explorer 2 context expression")
+    ],
+):
+    return associations_service.get_associations(
+        db, settings.filestore_location, slice_query
+    )
+
+
+@router.get(
+    "/associations",
+    operation_id="get_associations",
+    response_model=List[AssociationTable],
+)
+def get_associations(db: Annotated[SessionWithUser, Depends(get_db_with_user)]):
+    return [
+        AssociationTable(
+            id=a.id,
+            dataset_1_id=a.dataset_1_id,
+            dataset_2_id=a.dataset_2_id,
+            dataset_1_name=a.dataset_1.name,
+            dataset_2_name=a.dataset_2.name,
+            axis=a.axis,
+        )
+        for a in associations_crud.get_association_tables(db, None)
+    ]
+
+
+@router.delete("/associations/{id}")
+def delete_associations(
+    db: Annotated[SessionWithUser, Depends(get_db_with_user)],
+    settings: Annotated[Settings, Depends(get_settings)],
+    id: str,
+):
+    with transaction(db):
+        return associations_crud.delete_association_table(
+            db, id, settings.filestore_location
+        )
+
+
+@router.post(
+    "/associations",
+    operation_id="add_associations",
+    response_model=AssociationTable,
+    response_model_exclude_none=False,
+)
+def add_associations(
+    db: Annotated[SessionWithUser, Depends(get_db_with_user)],
+    settings: Annotated[Settings, Depends(get_settings)],
+    associations_in: Annotated[
+        AssociationsIn,
+        Body(
+            description="The associations table and which two datasets table references"
+        ),
+    ],
+):
+
+    serializer = URLSafeSerializer(settings.breadbox_secret)
+
+    full_file = construct_file_from_ids(
+        associations_in.file_ids,
+        associations_in.md5,
+        serializer,
+        settings.compute_results_location,
+    )
+
+    dest_filename = f"associations/{uuid.uuid4()}.sqlite3"
+    full_dest_filename = os.path.join(settings.filestore_location, dest_filename)
+    # make sure the directory exists before trying to move the file to it's final name
+    os.makedirs(os.path.dirname(full_dest_filename), exist_ok=True)
+
+    os.rename(full_file, full_dest_filename)
+
+    try:
+        with transaction(db):
+            assoc_table = associations_crud.add_association_table(
+                db,
+                associations_in.dataset_1_id,
+                associations_in.dataset_2_id,
+                associations_in.axis,
+                settings.filestore_location,
+                dest_filename,
+            )
+    except:
+        # if add_association_table fails, clean up the sqlite3 file
+        os.remove(full_dest_filename)
+        # ...before continuing on raising the exception
+        raise
+
+    return AssociationTable(
+        id=assoc_table.id,
+        dataset_1_id=assoc_table.dataset_1_id,
+        dataset_2_id=assoc_table.dataset_2_id,
+        dataset_1_name=assoc_table.dataset_1.name,
+        dataset_2_name=assoc_table.dataset_2.name,
+        axis=cast(Literal["sample", "feature"], assoc_table.axis),
+    )

breadbox/breadbox/api/temp/cas.py

@@ -0,0 +1,27 @@
+from .router import router
+from breadbox.schemas.cas import CASKey, CASValue
+from breadbox.io import cas
+import os.path
+from typing import Annotated
+from fastapi import APIRouter, Body, Depends, HTTPException
+from breadbox.api.dependencies import get_db_with_user, get_cas_db_path
+
+# Methods for getting/setting values in Content-addressable-storage (CAS)
+@router.get(
+    "/cas/{key}", operation_id="get_cas_value", response_model=CASValue,
+)
+def get_cas_value(key: str, cas_db_path: Annotated[str, Depends(get_cas_db_path)]):
+    value = cas.get_value(cas_db_path, key)
+    if value is None:
+        raise HTTPException(status_code=404)
+    return CASValue(value=value)
+
+
+@router.post(
+    "/cas", operation_id="set_cas_value", response_model=CASKey,
+)
+def set_cas_value(
+    value: CASValue, cas_db_path: Annotated[str, Depends(get_cas_db_path)]
+):
+    key = cas.set_value(cas_db_path, value.value)
+    return CASKey(key=key)