tests: simplify derived data test configuration

Summary: Now that we are using python for integration test setup, we can simplify how this works. Instead of having to provide the full JSON-formatted replacement config, we define the enabled derived data types using space-separated lists in environment variables. * `ENABLED_DERIVED_DATA` overrides the full set of enabled derived data types. * `ADDITIONAL_DERIVED_DATA` augments the default set with additional types. * `DISABLED_DERIVED_DATA` removes derived data types from the default set * `OTHER_DERIVED_DATA` enables derived data types in the `other` config. Existing tests which had overrides for this config that are now covered by the defaults have their overrides removed. Tests that just need to disable specific types (e.g. the Mercurial types) can now explicitly define this. Reviewed By: lmvasquezg Differential Revision: D65270448 fbshipit-source-id: a25c841defc3bf75ced8cb69f53eb73f189ec5f5
facebook · Nov 3, 2024 · d61e313 · d61e313
1 parent e66b8d5
commit d61e313
Show file tree

Hide file tree

Showing 21 changed files with 61 additions and 80 deletions.
diff --git a/eden/mononoke/tests/integration/git_lfs/test-git-lfs-derive-and-clone.t b/eden/mononoke/tests/integration/git_lfs/test-git-lfs-derive-and-clone.t
@@ -6,7 +6,7 @@
 
   $ . "${TEST_FIXTURES}/library.sh"
   $ REPOTYPE="blob_files"
-  $ GIT_LFS_INTERPRET_POINTERS=1 ENABLED_DERIVED_DATA='["git_commits", "git_trees", "git_delta_manifests_v2", "unodes", "filenodes", "hgchangesets", "ccsm", "skeleton_manifests"]' setup_common_config $REPOTYPE
+  $ GIT_LFS_INTERPRET_POINTERS=1 setup_common_config $REPOTYPE
   $ testtool_drawdag -R repo << EOF
   > A-B-C
   > # bookmark: C heads/master_bookmark

diff --git a/eden/mononoke/tests/integration/gitbundle/test-gitbundle-mononoke-repo-submodule.t b/eden/mononoke/tests/integration/gitbundle/test-gitbundle-mononoke-repo-submodule.t
@@ -5,14 +5,14 @@
 # directory of this source tree.
 
   $ . "${TEST_FIXTURES}/library.sh"
-  $ REPOTYPE="blob_files"
-  $ ENABLED_DERIVED_DATA='["ccsm", "skeleton_manifests", "skeleton_manifests_v2", "git_commits", "git_trees", "git_delta_manifests_v2", "unodes"]' setup_common_config $REPOTYPE
   $ GIT_REPO_ORIGIN="${TESTTMP}/origin/repo-git"
   $ GIT_REPO_SUBMODULE="${TESTTMP}/origin/repo-submodule"
   $ GIT_REPO="${TESTTMP}/repo-git"
   $ HG_REPO="${TESTTMP}/repo"
   $ BUNDLE_PATH="${TESTTMP}/repo_bundle.bundle"
 
+Disable Mercurial types as they do not support git submodules
+  $ DISABLED_DERIVED_DATA="filenodes hgchangesets hg_augmented_manifests" setup_common_config blob_files
 
 # Setup submodule git repository
   $ mkdir -p "$GIT_REPO_SUBMODULE"

diff --git a/eden/mononoke/tests/integration/gitimport/test-gitimport-bad-tag.t b/eden/mononoke/tests/integration/gitimport/test-gitimport-bad-tag.t
@@ -5,10 +5,8 @@
 # directory of this source tree.
 
   $ . "${TEST_FIXTURES}/library.sh"
-  $ ENABLED_DERIVED_DATA='["ccsm", "unodes", "git_commits", "git_trees", "git_delta_manifests_v2"]' setup_common_config
   $ GIT_REPO="${TESTTMP}/repo-git"
-  $ REPOTYPE="blob_files"
-  $ ENABLED_DERIVED_DATA='["ccsm", "unodes", "git_commits", "git_trees", "git_delta_manifests_v2"]' setup_common_config $REPOTYPE
+  $ setup_common_config blob_files
 
 # Setup git repository
   $ mkdir -p "$GIT_REPO"

diff --git a/eden/mononoke/tests/integration/gitimport/test-gitimport-commit-with-timezone-offset.t b/eden/mononoke/tests/integration/gitimport/test-gitimport-commit-with-timezone-offset.t
@@ -5,10 +5,8 @@
 # directory of this source tree.
 
   $ . "${TEST_FIXTURES}/library.sh"
-  $ ENABLED_DERIVED_DATA='["unodes", "git_commits", "git_trees", "git_delta_manifests_v2"]' setup_common_config
   $ GIT_REPO="${TESTTMP}/repo-git"
-  $ REPOTYPE="blob_files"
-  $ ENABLED_DERIVED_DATA='["unodes", "git_commits", "git_trees", "git_delta_manifests_v2"]' setup_common_config $REPOTYPE
+  $ setup_common_config blob_files
 
 # Setup git repository
   $ mkdir -p "$GIT_REPO"

diff --git a/eden/mononoke/tests/integration/gitimport/test-gitimport-committer.t b/eden/mononoke/tests/integration/gitimport/test-gitimport-committer.t
@@ -5,9 +5,9 @@
 # directory of this source tree.
 
   $ . "${TEST_FIXTURES}/library.sh"
-  $ HG_SET_COMMITTER_EXTRA=true ENABLED_DERIVED_DATA='["git_trees", "filenodes", "hgchangesets"]' setup_common_config
   $ GIT_REPO="${TESTTMP}/repo-git"
   $ HG_REPO="${TESTTMP}/repo"
+  $ HG_SET_COMMITTER_EXTRA=true setup_common_config
 
 # Setup git repsitory
   $ mkdir "$GIT_REPO"

diff --git a/eden/mononoke/tests/integration/gitimport/test-gitimport-discard-submodules.t b/eden/mononoke/tests/integration/gitimport/test-gitimport-discard-submodules.t
@@ -5,8 +5,7 @@
 # directory of this source tree.
 
   $ . "${TEST_FIXTURES}/library.sh"
-  $ REPOTYPE="blob_files"
-  $ ENABLED_DERIVED_DATA='["ccsm", "skeleton_manifests", "skeleton_manifests_v2", "git_commits", "git_trees", "git_delta_manifests_v2", "unodes", "filenodes", "hgchangesets"]' setup_common_config $REPOTYPE
+  $ setup_common_config blob_files
   $ GIT_REPO_ORIGIN="${TESTTMP}/origin/repo-git"
   $ GIT_REPO_SUBMODULE="${TESTTMP}/origin/repo-submodule"
   $ GIT_REPO="${TESTTMP}/repo-git"

diff --git a/eden/mononoke/tests/integration/gitimport/test-gitimport-filestore.t b/eden/mononoke/tests/integration/gitimport/test-gitimport-filestore.t
@@ -7,9 +7,9 @@
   $ export FILESTORE=1
   $ export FILESTORE_CHUNK_SIZE=10
   $ . "${TEST_FIXTURES}/library.sh"
-  $ ENABLED_DERIVED_DATA='["git_trees", "filenodes", "hgchangesets"]' setup_common_config
   $ GIT_REPO="${TESTTMP}/repo-git"
   $ HG_REPO="${TESTTMP}/repo"
+  $ setup_common_config blob_files
 
 # Setup git repsitory
   $ mkdir "$GIT_REPO"

diff --git a/eden/mononoke/tests/integration/gitimport/test-gitimport-lfs-disabled.t b/eden/mononoke/tests/integration/gitimport/test-gitimport-lfs-disabled.t
@@ -6,8 +6,7 @@
 
   $ . "${TEST_FIXTURES}/library.sh"
   $ . "${TEST_FIXTURES}/library-git-lfs.sh"
-  $ REPOTYPE="blob_files"
-  $ ENABLED_DERIVED_DATA='["git_commits", "git_trees", "git_delta_manifests_v2", "unodes", "filenodes", "hgchangesets"]' setup_common_config $REPOTYPE
+  $ setup_common_config blob_files
 Without that bit gitimport is unable to set bookmarks
 
 Use common repo setup

diff --git a/eden/mononoke/tests/integration/gitimport/test-gitimport-lfs-enabled-dangling-pointer.t b/eden/mononoke/tests/integration/gitimport/test-gitimport-lfs-enabled-dangling-pointer.t
@@ -6,9 +6,7 @@
 
   $ . "${TEST_FIXTURES}/library.sh"
   $ . "${TEST_FIXTURES}/library-git-lfs.sh"
-  $ REPOTYPE="blob_files"
-  $ ENABLED_DERIVED_DATA='["git_commits", "git_trees", "git_delta_manifests_v2", "unodes", "filenodes", "hgchangesets"]' setup_common_config $REPOTYPE
-Without that bit gitimport is unable to set bookmarks
+  $ setup_common_config blob_files
 
 Use common repo setup
   $ GIT_LFS_INTERPRET_POINTERS=1 test_repos_for_git_lfs_import

diff --git a/eden/mononoke/tests/integration/gitimport/test-gitimport-lfs-enabled.t b/eden/mononoke/tests/integration/gitimport/test-gitimport-lfs-enabled.t
@@ -6,9 +6,7 @@
 
   $ . "${TEST_FIXTURES}/library.sh"
   $ . "${TEST_FIXTURES}/library-git-lfs.sh"
-  $ REPOTYPE="blob_files"
-  $ ENABLED_DERIVED_DATA='["git_commits", "git_trees", "git_delta_manifests_v2", "unodes", "filenodes", "hgchangesets"]' setup_common_config $REPOTYPE
-Without that bit gitimport is unable to set bookmarks
+  $ setup_common_config blob_files
 
 Use common repo setup
   $ GIT_LFS_INTERPRET_POINTERS=1 test_repos_for_git_lfs_import

diff --git a/eden/mononoke/tests/integration/gitimport/test-gitimport-octopus.t b/eden/mononoke/tests/integration/gitimport/test-gitimport-octopus.t
@@ -5,11 +5,9 @@
 # directory of this source tree.
 
   $ . "${TEST_FIXTURES}/library.sh"
-  $ ENABLED_DERIVED_DATA='["git_trees", "filenodes", "hgchangesets"]' setup_common_config "blob_files"
   $ GIT_REPO="${TESTTMP}/repo-git"
   $ HG_REPO="${TESTTMP}/repo"
-  $ REPOTYPE="blob_files"
-  $ setup_common_config $REPOTYPE
+  $ setup_common_config blob_files
   $ setconfig remotenames.selectivepulldefault=master_bookmark,branch1,branch2
 
 # Setup git repsitory

diff --git a/eden/mononoke/tests/integration/gitimport/test-gitimport-single-commit.t b/eden/mononoke/tests/integration/gitimport/test-gitimport-single-commit.t
@@ -5,11 +5,9 @@
 # directory of this source tree.
 
   $ . "${TEST_FIXTURES}/library.sh"
-  $ ENABLED_DERIVED_DATA='["git_trees", "filenodes", "hgchangesets", "git_delta_manifests_v2"]' setup_common_config
   $ GIT_REPO="${TESTTMP}/repo-git"
   $ HG_REPO="${TESTTMP}/repo"
-  $ REPOTYPE="blob_files"
-  $ setup_common_config $REPOTYPE
+  $ setup_common_config blob_files
 
 # Setup git repsitory
   $ mkdir "$GIT_REPO"

diff --git a/eden/mononoke/tests/integration/gitimport/test-gitimport-timezone.t b/eden/mononoke/tests/integration/gitimport/test-gitimport-timezone.t
@@ -5,9 +5,9 @@
 # directory of this source tree.
 
   $ . "${TEST_FIXTURES}/library.sh"
-  $ ENABLED_DERIVED_DATA='["git_trees", "filenodes", "hgchangesets"]' setup_common_config
   $ GIT_REPO="${TESTTMP}/repo-git"
   $ HG_REPO="${TESTTMP}/repo"
+  $ setup_common_config blob_files
 
 # Setup git repsitory
   $ mkdir "$GIT_REPO"

diff --git a/eden/mononoke/tests/integration/gitimport/test-gitimport-unroundtripable.t b/eden/mononoke/tests/integration/gitimport/test-gitimport-unroundtripable.t
@@ -5,10 +5,8 @@
 # directory of this source tree.
 
   $ . "${TEST_FIXTURES}/library.sh"
-  $ ENABLED_DERIVED_DATA='["unodes", "git_commits", "git_trees", "git_delta_manifests_v2"]' setup_common_config
   $ GIT_REPO="${TESTTMP}/repo-git"
-  $ REPOTYPE="blob_files"
-  $ ENABLED_DERIVED_DATA='["unodes", "git_commits", "git_trees", "git_delta_manifests_v2"]' setup_common_config $REPOTYPE
+  $ setup_common_config blob_files
 
 # Setup git repository
   $ mkdir -p "$GIT_REPO"

diff --git a/eden/mononoke/tests/integration/gitimport/test-gitimport-with-filters.t b/eden/mononoke/tests/integration/gitimport/test-gitimport-with-filters.t
@@ -5,12 +5,10 @@
 # directory of this source tree.
 
   $ . "${TEST_FIXTURES}/library.sh"
-  $ ENABLED_DERIVED_DATA='["git_trees", "filenodes", "hgchangesets"]' setup_common_config
   $ GIT_REPO_ORIGIN="${TESTTMP}/origin/repo-git"
   $ GIT_REPO="${TESTTMP}/repo-git"
   $ HG_REPO="${TESTTMP}/repo"
-  $ REPOTYPE="blob_files"
-  $ setup_common_config $REPOTYPE
+  $ setup_common_config blob_files
 
 # Setup git repository
   $ mkdir -p "$GIT_REPO_ORIGIN"

diff --git a/eden/mononoke/tests/integration/gitimport/test-gitimport.t b/eden/mononoke/tests/integration/gitimport/test-gitimport.t
@@ -5,12 +5,10 @@
 # directory of this source tree.
 
   $ . "${TEST_FIXTURES}/library.sh"
-  $ ENABLED_DERIVED_DATA='["git_trees", "filenodes", "hgchangesets"]' setup_common_config
   $ GIT_REPO_ORIGIN="${TESTTMP}/origin/repo-git"
   $ GIT_REPO="${TESTTMP}/repo-git"
   $ HG_REPO="${TESTTMP}/repo"
-  $ REPOTYPE="blob_files"
-  $ setup_common_config $REPOTYPE
+  $ setup_common_config blob_files
 
 # Setup git repository
   $ mkdir -p "$GIT_REPO_ORIGIN"

diff --git a/...s/integration/mononoke_git_server/test-mononoke-git-server-clone-with-submodule-in-root.t b/...s/integration/mononoke_git_server/test-mononoke-git-server-clone-with-submodule-in-root.t
@@ -5,12 +5,13 @@
 # directory of this source tree.
 
   $ . "${TEST_FIXTURES}/library.sh"
-  $ REPOTYPE="blob_files"
-  $ ENABLED_DERIVED_DATA='["git_trees", "git_commits", "git_delta_manifests_v2", "ccsm", "skeleton_manifests", "skeleton_manifests_v2"]' setup_common_config $REPOTYPE
   $ GIT_REPO_ORIGIN="${TESTTMP}/origin/repo-git"
   $ GIT_REPO_SUBMODULE="${TESTTMP}/origin/repo-submodule"
   $ GIT_REPO="${TESTTMP}/repo-git"
 
+Disable Mercurial types as they do not support git submodules
+  $ DISABLED_DERIVED_DATA="filenodes hgchangesets hg_augmented_manifests" setup_common_config blob_files
+
 # Setup submodule git repository
   $ mkdir -p "$GIT_REPO_SUBMODULE"
   $ cd "$GIT_REPO_SUBMODULE"

diff --git a/eden/mononoke/tests/integration/newadmin/test-newadmin-derive-regenerate.t b/eden/mononoke/tests/integration/newadmin/test-newadmin-derive-regenerate.t
@@ -5,9 +5,9 @@
 # directory of this source tree.
 
   $ . "${TEST_FIXTURES}/library.sh"
-  $ ENABLED_DERIVED_DATA='["git_trees", "filenodes", "hgchangesets"]' setup_common_config
   $ GIT_REPO="${TESTTMP}/repo-git"
   $ HG_REPO="${TESTTMP}/repo"
+  $ setup_common_config blob_files
 
 # Setup git repository
   $ mkdir "$GIT_REPO"
@@ -68,7 +68,7 @@
 
 # Now rederive HG_SET_COMMITTER_EXTRA=true. This changes hg hash, so we need to run with --rederive and make sure
 # hg hash was overwritten.
-  $ HG_SET_COMMITTER_EXTRA=true ENABLED_DERIVED_DATA='["git_trees", "filenodes", "hgchangesets"]' setup_common_config
+  $ HG_SET_COMMITTER_EXTRA=true setup_common_config blob_files
 
 # If we call derive without the rederive flag, we have the same hg hash as before
   $ mononoke_newadmin derived-data -R repo derive -T hgchangesets -i 1213979c6023f23e70dbe8845d773078ac1e0506bc2ab98382a329da0cb379a7

diff --git a/eden/mononoke/tests/integration/newadmin/test-newadmin-derived-data-multiple-configs.t b/eden/mononoke/tests/integration/newadmin/test-newadmin-derived-data-multiple-configs.t
@@ -8,7 +8,7 @@
   $ . "${TEST_FIXTURES}/library.sh"
 
 setup configuration
-  $ ENABLED_DERIVED_DATA='["unodes"]' OTHER_DERIVED_DATA='["skeleton_manifests"]' setup_common_config
+  $ ENABLED_DERIVED_DATA="unodes" OTHER_DERIVED_DATA="skeleton_manifests" setup_common_config
 
   $ testtool_drawdag -R repo << 'EOF'
   > D F

diff --git a/eden/mononoke/tests/integration/server/test-getpackv2-multibyte-filenames.t b/eden/mononoke/tests/integration/server/test-getpackv2-multibyte-filenames.t
@@ -12,7 +12,7 @@
   $ NAME='file-multibyte-€'
   $ BOOKMARK='master_bookmark'
 
-  $ ENABLED_DERIVED_DATA='["git_trees", "filenodes", "hgchangesets"]' setup_common_config
+  $ setup_common_config
 
 # THis is a bit clowntown. There is a bug in our tests right now that prevents
 # us from creating files using UTF-8 names on Mercurial Python 3, so we

diff --git a/eden/scm/sapling/testing/ext/mononoke.py b/eden/scm/sapling/testing/ext/mononoke.py
@@ -917,51 +917,51 @@ def append_def_config(content: str, mode: str = "a"):
 """.format(env.getenv("TESTTMP"))
     )
 
-    if env.getenv("ENABLED_DERIVED_DATA"):
-        enabled_derived_data = env.getenv("ENABLED_DERIVED_DATA")
-        append_config(
-            f"""
-[derived_data_config.available_configs.default]
-types = {enabled_derived_data}
-git_delta_manifest_version = 2
-git_delta_manifest_v2_config.max_inlined_object_size = 20
-git_delta_manifest_v2_config.max_inlined_delta_size = 20
-git_delta_manifest_v2_config.delta_chunk_size = 1000
-"""
-        )
+    if enabled_derived_data := env.getenv("ENABLED_DERIVED_DATA"):
+        derived_data_types = enabled_derived_data.split()
     else:
-        append_config(
-            """
+        derived_data_types = [
+            "blame",
+            "changeset_info",
+            "deleted_manifest",
+            "fastlog",
+            "filenodes",
+            "fsnodes",
+            "git_commits",
+            "git_delta_manifests_v2",
+            "git_trees",
+            "unodes",
+            "hgchangesets",
+            "hg_augmented_manifests",
+            "skeleton_manifests",
+            "skeleton_manifests_v2",
+            "bssm_v3",
+            "ccsm",
+            "test_manifests",
+            "test_sharded_manifests",
+        ]
+
+    if additional_derived_data := env.getenv("ADDITIONAL_DERIVED_DATA"):
+        derived_data_types.extend(additional_derived_data.split())
+    if disabled_derived_data := env.getenv("DISABLED_DERIVED_DATA"):
+        derived_data_types = list(
+            set(derived_data_types) - set(disabled_derived_data.split())
+        )
+
+    enabled_derived_data = json.dumps(derived_data_types)
+    append_config(
+        f"""
 [derived_data_config.available_configs.default]
-types=[
-  "blame",
-  "changeset_info",
-  "deleted_manifest",
-  "fastlog",
-  "filenodes",
-  "fsnodes",
-  "git_commits",
-  "git_delta_manifests_v2",
-  "git_trees",
-  "unodes",
-  "hgchangesets",
-  "hg_augmented_manifests",
-  "skeleton_manifests",
-  "skeleton_manifests_v2",
-  "bssm_v3",
-  "ccsm",
-  "test_manifests",
-  "test_sharded_manifests"
-]
+types = {enabled_derived_data}
 git_delta_manifest_version = 2
 git_delta_manifest_v2_config.max_inlined_object_size = 20
 git_delta_manifest_v2_config.max_inlined_delta_size = 20
 git_delta_manifest_v2_config.delta_chunk_size = 1000
 """
-        )
+    )
 
-    if env.getenv("OTHER_DERIVED_DATA"):
-        other_derived_data = env.getenv("OTHER_DERIVED_DATA")
+    if other_derived_data := env.getenv("OTHER_DERIVED_DATA"):
+        other_derived_data = json.dumps(other_derived_data.split())
         append_config(
             f"""
 [derived_data_config.available_configs.other]