Wow, a messy merge

.github/workflows/build.yml

@@ -1,4 +1,6 @@
-name: Scala tests with coverage
+# Github action to Run Unit Tests and Publish Sam Client
+
+name: Run Unit Tests and Publish Client
 
 on:
   pull_request:
@@ -68,9 +70,15 @@ jobs:
         id: coverageReport
         run: source env/test.env; sbt coverage "testOnly --" coverageReport
 
+  tag:
+    uses: ./.github/workflows/tag.yml
+    with:
+      release-branches: develop
+    secrets: inherit
+
   generateAndPublishClient:
     runs-on: ubuntu-20.04
-    needs: run-unit-tests
+    needs: [ tag, run-unit-tests ]
 
     env:
       ARTIFACTORY_USERNAME: ${{secrets.ARTIFACTORY_USERNAME}}
@@ -92,13 +100,13 @@ jobs:
         working-directory: codegen_java
         id: publishJavaClient
         if: ${{ github.ref_name == 'develop' }}
-        run: sbt "+ publish" -Dproject.isSnapshot=false
+        run: sbt "+ publish" -Dproject.isSnapshot=false -Dproject.semVer=${{ needs.tag.outputs.tag }}
 
       - name: Publish java client as snapshot for PRs
         working-directory: codegen_java
         id: publishJavaClientSnapshot
         if: ${{ github.ref_name != 'develop' }}
-        run: sbt "+ publish" -Dproject.isSnapshot=true
+        run: sbt "+ publish" -Dproject.isSnapshot=true -Dproject.semVer=${{ needs.tag.outputs.tag }}
 
       - name: Generate old javax java client
         id: generateOldJavaClient
@@ -108,10 +116,10 @@ jobs:
         working-directory: codegen_java_old
         id: publishOldJavaClient
         if: ${{ github.ref_name == 'develop' }}
-        run: sbt "+ publish" -Dproject.isSnapshot=false
+        run: sbt "+ publish" -Dproject.isSnapshot=false -Dproject.semVer=${{ needs.tag.outputs.tag }}
 
       - name: Publish old javax java client as snapshot for PRs
         working-directory: codegen_java_old
         id: publishOldJavaClientSnapshot
         if: ${{ github.ref_name != 'develop' }}
-        run: sbt "+ publish" -Dproject.isSnapshot=true
+        run: sbt "+ publish" -Dproject.isSnapshot=true -Dproject.semVer=${{ needs.tag.outputs.tag }}

.github/workflows/sam-build-tag-publish-and-run-tests.yaml

@@ -1,6 +1,6 @@
 # Github action to Build Sam artifact, add/promote semantic tagging and then run tests
 
-name: sam-build-tag-publish-and-run-tests
+name: Build, Publish to Sherlock, and Run Integration Tests
 
 on:
   pull_request:
@@ -13,6 +13,7 @@ on:
 env:
   BEE_CREATE_RUN_NAME: 'bee-create-${{ github.event.repository.name }}-${{ github.run_id }}-${{ github.run_attempt }}'
   BEE_DESTROY_RUN_NAME: 'bee-destroy-${{ github.event.repository.name }}-${{ github.run_id }}-${{ github.run_attempt }}'
+  RAWLS_SWAT_TESTS_RUN_NAME: 'rawls-swat-tests-${{ github.event.repository.name }}-${{ github.run_id }}-${{ github.run_attempt }}'
 
 jobs:
   tag:
@@ -195,8 +196,8 @@ jobs:
         terra-env: [ dev ] # what versions of apps do we use to emulate types of environments
         testing-env: [ qa ] # what env resources to use, e.g. SA keys
         test-group: [
-          { group_name: workspaces, tag: "-n org.broadinstitute.dsde.test.api.AuthDomainsTest -n org.broadinstitute.dsde.test.api.BillingsTest -n org.broadinstitute.dsde.test.api.WorkspacesTest" },
-          { group_name: analysis_journeys, tag: "-n org.broadinstitute.dsde.test.api.DataRepoSnapshotsTest" },
+          { group_name: workspaces, tag: "-n org.broadinstitute.dsde.test.api.BillingsTest -n org.broadinstitute.dsde.test.api.WorkspacesTest" },
+          { group_name: workspacesAuthDomains, tag: "-n org.broadinstitute.dsde.test.api.AuthDomainsTest" },
           { group_name: workflows, tag: "-n org.broadinstitute.dsde.test.api.MethodsTest" }
         ] # Rawls test groups
     runs-on: ubuntu-latest
@@ -209,15 +210,17 @@ jobs:
         env:
           rawls_base_test_entrypoint: "testOnly -- -l ProdTest -l NotebooksCanaryTest"
           test-context: ${{ needs.prepare-configs.outputs.test-context }}
-        uses: broadinstitute/workflow-dispatch@v3
+        uses: broadinstitute/workflow-dispatch@v4
         with:
+          run-name: "${{ env.RAWLS_SWAT_TESTS_RUN_NAME }}-${{ matrix.terra-env }}-${{ matrix.testing-env }}-${{ matrix.test-group.group_name }}"
           workflow: .github/workflows/rawls-swat-tests.yaml
           repo: broadinstitute/terra-github-workflows
           ref: refs/heads/main
           token: ${{ secrets.BROADBOT_TOKEN }} # github token for access to kick off a job in the private repo
           # manually recalculate b/c env context is broken https://github.com/actions/runner/issues/480
           inputs: '{
             "additional-args": "{\"logging\":\"true\",\"java-version\":\"17\",\"billing-project\":\"\"}",
+            "run-name": "${{ env.RAWLS_SWAT_TESTS_RUN_NAME }}-${{ matrix.terra-env }}-${{ matrix.testing-env }}-${{ matrix.test-group.group_name }}",
             "bee-name": "${{ github.event.repository.name }}-${{ github.run_id }}-${{ matrix.terra-env }}",
             "ENV": "${{ matrix.testing-env }}",
             "test-group-name": "${{ matrix.test-group.group_name }}",

.github/workflows/tag.yml

@@ -13,11 +13,6 @@ on:
         default: false
         required: false
         type: string
-      print-tag:
-        description: "Echo generated tag to console"
-        default: "true"
-        required: false
-        type: string
       release-branches:
         description: "Default branch (main, develop, etc)"
         default: 'main'
@@ -30,6 +25,9 @@ on:
       new-tag:
         description: "The value of the newly created tag"
         value: ${{ jobs.tag-job.outputs.new-tag }}
+      app-version:
+        description: "The app version"
+        value: ${{ jobs.tag-job.outputs.app-version }}
     secrets:
       BROADBOT_TOKEN:
         required: true
@@ -44,6 +42,7 @@ jobs:
     outputs:
       tag: ${{ steps.tag.outputs.tag }}
       new-tag: ${{ steps.tag.outputs.new_tag }}
+      app-version: ${{ steps.output-version.outputs.app-version }}
     steps:
       - name: Checkout current code
         uses: actions/checkout@v3
@@ -60,7 +59,14 @@ jobs:
           DRY_RUN: ${{ inputs.dry-run }}
           RELEASE_BRANCHES: ${{ inputs.release-branches }}
           WITH_V: true
-      - name: Echo generated tag to console
-        if: ${{ inputs.print-tag == 'true' }}
+      - name: Output app version
+        id: output-version
         run: |
-          echo "Newly created version tag: '${{ steps.tag.outputs.new_tag }}'"
+          # See https://broadworkbench.atlassian.net/browse/QA-2282 for context
+          if [[ -z "${{ steps.tag.outputs.new_tag }}" ]]; then
+            echo "App version tag for this commit has already been dispatched: '${{ steps.tag.outputs.tag }}'"
+            echo "app-version=${{ steps.tag.outputs.tag }}" >> $GITHUB_OUTPUT
+          else
+            echo "New app version tag: '${{ steps.tag.outputs.new_tag }}'"
+            echo "app-version=${{ steps.tag.outputs.new_tag }}" >> $GITHUB_OUTPUT
+          fi

.github/workflows/verify_consumer_pacts.yml

@@ -151,7 +151,7 @@ jobs:
           # for publishing the results of provider verification.
           if [[ -z "${{ inputs.pb-event-type }}" ]]; then
             echo "PROVIDER_BRANCH=${{ env.CURRENT_BRANCH }}" >> $GITHUB_ENV
-            echo "PROVIDER_VERSION=${{ needs.regulated-tag-job.outputs.new-tag }}" >> $GITHUB_ENV
+            echo "PROVIDER_VERSION=${{ needs.regulated-tag-job.outputs.app-version }}" >> $GITHUB_ENV
           else
             echo "PROVIDER_VERSION=${{ env.PROVIDER_TAG }}" >> $GITHUB_ENV
           fi

README.md

@@ -178,6 +178,9 @@ sudo sh -c "echo '127.0.0.1       local.dsde-dev.broadinstitute.org' >> /etc/hos
 
 You can then start Sam against dev DBs or local DBs following the instructions below.
 
+#### Get on the Broad VPN
+You need to be on the non-split VPN to perform the next steps
+
 #### Credentials
 You'll need to authenticate with Google to run the `kubectl` commands. To run Sam locally, dev credentials are required.
 
@@ -209,6 +212,8 @@ sh docker/run-proxy.sh start
 ```
 
 #### Run Sam!
+You can use sbt, but sometimes it's easier to run from your IDE. To do so, just set up a run configuration as described in the debugging instructions and run it with IntelliJ
+
 ```shell
 sbt run
 ```

automation/project/Dependencies.scala

@@ -7,10 +7,11 @@ object Dependencies {
   val akkaV = "2.6.19"
   val akkaHttpV = "10.2.2"
 
-  val workbenchLibV = "a0519cb"
-  val workbenchGoogleV = s"0.30-$workbenchLibV"
-  val workbenchGoogle2V = s"0.34-$workbenchLibV"
-  val workbenchServiceTestV = "2.0-5863cbd"
+  val workbenchLibV = "9138393"
+
+  val workbenchGoogleV = s"0.32-$workbenchLibV"
+  val workbenchGoogle2V = s"0.36-$workbenchLibV"
+  val workbenchServiceTestV = s"5.0-$workbenchLibV"
 
   val excludeWorkbenchModel = ExclusionRule(organization = "org.broadinstitute.dsde.workbench", name = "workbench-model_" + scalaV)
 

automation/src/test/scala/org/broadinstitute/dsde/workbench/sam/api/SamApiSpec.scala

@@ -173,65 +173,6 @@ class SamApiSpec extends AnyFreeSpec with Matchers with ScalaFutures with CleanU
       petEmail2 shouldBe petEmail1
     }
 
-    "should synchronize groups with Google" taggedAs Tags.ExcludeInAlpha in {
-      val managedGroupId = UUID.randomUUID.toString
-      val adminPolicyName = "admin"
-      val Seq(user1: Credentials, user2: Credentials, user3: Credentials) = UserPool.chooseStudents(3)
-      val user1AuthToken = user1.makeAuthToken()
-      val Seq(user1Proxy: WorkbenchEmail, user2Proxy: WorkbenchEmail, user3Proxy: WorkbenchEmail) =
-        Seq(user1, user2, user3).map(user => Sam.user.proxyGroup(user.email)(user1AuthToken))
-
-      val waitTime = 10.minutes
-
-      Sam.user.createGroup(managedGroupId)(user1AuthToken)
-      register cleanUp Sam.user.deleteGroup(managedGroupId)(user1AuthToken)
-
-      val policies = Sam.user.listResourcePolicies("managed-group", managedGroupId)(user1AuthToken)
-      val policyEmail = policies.collect {
-        case SamModel.AccessPolicyResponseEntry(_, policy, email) if policy.memberEmails.nonEmpty => email
-      }
-      assert(policyEmail.size == 1) // Only the admin policy should be non-empty after creation
-
-      // The admin policy should contain only the user that created the group
-      awaitAssert(
-        Await
-          .result(googleDirectoryDAO.listGroupMembers(policyEmail.head), waitTime)
-          .getOrElse(Set.empty) should contain theSameElementsAs Set(user1Proxy.value),
-        waitTime,
-        5.seconds
-      )
-
-      // Change the membership of the admin policy to include users 1 and 2
-      Sam.user.setPolicyMembers(managedGroupId, adminPolicyName, Set(user1.email, user2.email))(user1AuthToken)
-      awaitAssert(
-        Await
-          .result(googleDirectoryDAO.listGroupMembers(policyEmail.head), waitTime)
-          .getOrElse(Set.empty) should contain theSameElementsAs Set(user1Proxy.value, user2Proxy.value),
-        waitTime,
-        5.seconds
-      )
-
-      // Add user 3 to the admin policy
-      Sam.user.addUserToPolicy(managedGroupId, adminPolicyName, user3.email)(user1AuthToken)
-      awaitAssert(
-        Await
-          .result(googleDirectoryDAO.listGroupMembers(policyEmail.head), waitTime)
-          .getOrElse(Set.empty) should contain theSameElementsAs Set(user1Proxy.value, user2Proxy.value, user3Proxy.value),
-        waitTime,
-        5.seconds
-      )
-
-      // Remove user 2 from the admin policy
-      Sam.user.removeUserFromPolicy(managedGroupId, adminPolicyName, user2.email)(user1AuthToken)
-      awaitAssert(
-        Await
-          .result(googleDirectoryDAO.listGroupMembers(policyEmail.head), waitTime)
-          .getOrElse(Set.empty) should contain theSameElementsAs Set(user1Proxy.value, user3Proxy.value),
-        waitTime,
-        5.seconds
-      )
-    }
-
     "should only synchronize the intersection group for policies constrained by auth domains" taggedAs Tags.ExcludeInAlpha in {
       val waitTime = 10.minutes
       val authDomainId = UUID.randomUUID.toString

codegen_java/project/Version.scala

@@ -11,10 +11,12 @@ object Version {
     val lastCommit = sys.env.getOrElse("GIT_HASH", getLastCommitFromGit).trim()
     val version = baseVersion + "-" + lastCommit
 
+    val semVer = sys.props.get("project.semVer").getOrElse(version)
+
     // The project isSnapshot string passed in via command line settings, if desired.
     val isSnapshot = sys.props.getOrElse("project.isSnapshot", "true").toBoolean
 
     // For now, obfuscate SNAPSHOTs from sbt's developers: https://github.com/sbt/sbt/issues/2687#issuecomment-236586241
-    if (isSnapshot) s"$version-SNAP" else version
+    if (isSnapshot) s"$semVer-SNAP" else semVer
   }
 }

codegen_java/templates/libraries/okhttp-gson/build.sbt.mustache

@@ -39,7 +39,7 @@ lazy val root = (project in file(".")).
       "com.google.code.findbugs" % "jsr305" % "3.0.2" % "compile",
       "org.junit.jupiter" % "junit-jupiter-api" % "5.9.1" % "test",
       "com.novocode" % "junit-interface" % "0.10" % "test",
-      "org.mockito" % "mockito-core" % "3.12.4" % "test"
-    ),
-    dependencyOverrides += "org.json" % "json" % "20240205") ++ publishSettings:_*
+      "org.mockito" % "mockito-core" % "3.12.4" % "test",
+      "org.json" % "json" % "20240205"
+    )) ++ publishSettings:_*
   )

codegen_java_old/project/Version.scala

@@ -11,10 +11,12 @@ object Version {
     val lastCommit = sys.env.getOrElse("GIT_HASH", getLastCommitFromGit).trim()
     val version = baseVersion + "-" + lastCommit
 
+    val semVer = sys.props.get("project.semVer").getOrElse(version)
+
     // The project isSnapshot string passed in via command line settings, if desired.
     val isSnapshot = sys.props.getOrElse("project.isSnapshot", "true").toBoolean
 
     // For now, obfuscate SNAPSHOTs from sbt's developers: https://github.com/sbt/sbt/issues/2687#issuecomment-236586241
-    if (isSnapshot) s"$version-SNAP" else version
+    if (isSnapshot) s"$semVer-SNAP" else semVer
   }
 }

codegen_java_old/templates/libraries/okhttp-gson/build.sbt.mustache

@@ -38,8 +38,8 @@ lazy val root = (project in file(".")).
       "jakarta.annotation" % "jakarta.annotation-api" % "1.3.5" % "compile",
       "org.junit.jupiter" % "junit-jupiter-api" % "5.9.1" % "test",
       "com.novocode" % "junit-interface" % "0.10" % "test",
-      "javax.annotation" % "javax.annotation-api" % "1.3.2"
-    ),
-    dependencyOverrides += "org.json" % "json" % "20240205") ++ publishSettings:_*
+      "javax.annotation" % "javax.annotation-api" % "1.3.2",
+      "org.json" % "json" % "20240205"
+    )) ++ publishSettings:_*
   )
 

docker/run-postgres.sh

@@ -1,7 +1,9 @@
 #!/usr/bin/env bash
 
-# The CloudSQL console simply states "PostgreSQL 9.6" so we may not match the minor version number
-POSTGRES_VERSION=9.6
+# NOTE this script has to be run from the root of the project or else it wont work.
+
+# The CloudSQL console simply states "PostgreSQL 15" so we may not match the minor version number
+POSTGRES_VERSION=15
 start() {
 
 
@@ -12,6 +14,8 @@ start() {
     echo "starting up postgres container..."
     docker run --name $CONTAINER -e POSTGRES_USER=sam-test -e POSTGRES_PASSWORD=sam-test -e POSTGRES_DB=testdb -d -p "$POSTGRES_PORT:5432" postgres:$POSTGRES_VERSION
 
+    echo "Created DB: testdb"
+
     # validate postgres
     echo "running postgres validation..."
     docker run --rm --link $CONTAINER:postgres -v $PWD/docker/sql_validate.sh:/working/sql_validate.sh postgres:$POSTGRES_VERSION /working/sql_validate.sh sam

docker/run-proxy.sh

@@ -11,6 +11,7 @@ start() {
     docker create --name "${CONTAINER}" \
         --restart "always" \
         -p 40080:80 -p 50443:443 \
+        -e "B2C_APPLICATION_ID=${OIDC_CLIENT_ID}" \
         us.gcr.io/broad-dsp-gcr-public/httpd-terra-proxy:v0.1.17
 
     docker cp "${SECRETS_DIR}/server.crt" sam-proxy:/etc/ssl/certs/server.crt

env/local.env

@@ -5,6 +5,8 @@ export ADMIN_SERVICE_ACCOUNT_4="src/main/resources/rendered/admin-service-accoun
 export ADMIN_SERVICE_ACCOUNT_5="src/main/resources/rendered/admin-service-account-5.json"
 export SERVICE_ACCOUNT_ADMINS="[email protected], [email protected]"
 export AZURE_ENABLED="false"
+export AZURE_SERVICE_CATALOG_APPS_ENABLED="false"
+export AZURE_MANAGED_APP_WORKLOAD_CLIENT_ID="661e243c-5ef9-4a9c-9be3-b7f5585828b3"
 export EMAIL_DOMAIN="dev.test.firecloud.org"
 export ENVIRONMENT="dev"
 export GOOGLE_APPS_DOMAIN="test.firecloud.org"
@@ -16,6 +18,7 @@ export GOOGLE_PROJECT="broad-dsde-dev"
 export GOOGLE_PROJECT_SERVICE_ACCOUNT="[email protected]"
 export GOOGLE_PUB_SUB_SUBSCRIPTION="sam-group-sync-local-sub"
 export GOOGLE_PUB_SUB_TOPIC="sam-group-sync-local"
+export GOOGLE_GROUP_SYNC_WORKER_COUNT="1"
 export GOOGLE_SERVICE_ACCOUNT_DOMAIN="broad-dsde-dev.iam.gserviceaccount.com"
 export GOOGLE_SUBDOMAIN_EMAIL="[email protected]"
 export GOOGLE_DIRECTORY_API_EMAIL_1="[email protected]"

env/test.env

@@ -1,8 +1,12 @@
 export SERVICE_ACCOUNT_ADMINS="[email protected], [email protected]"
 export AZURE_ENABLED="false"
+export CREATE_BEE_WORKFLOW_TEST="true"
 export AZURE_MANAGED_APP_CLIENT_ID="foo"
 export AZURE_MANAGED_APP_CLIENT_SECRET="foo"
 export AZURE_MANAGED_APP_TENANT_ID="foo"
+export AZURE_MANAGED_APP_WORKLOAD_CLIENT_ID="foo"
+export AZURE_ALLOW_MANAGED_IDENTITY_USER_CREATION="false"
+export AZURE_SERVICE_CATALOG_APPS_ENABLED="false"
 export EMAIL_DOMAIN="dev.test.firecloud.org"
 export ENVIRONMENT="local"
 export GOOGLE_APPS_DOMAIN="test.firecloud.org"

pact4s/src/test/resources/reference.conf

@@ -119,8 +119,6 @@ testStuff = {
 oidc {
   authorityEndpoint = "https://accounts.google.com"
   oidcClientId = "some-client"
-  oidcClientSecret = "some-secret"
-  legacyGoogleClientId = "another-client"
 }
 
 liquibase {