From d61e3131671e52e2381dc2cff5f82615f67718bd Mon Sep 17 00:00:00 2001 From: Mark Juggurnauth-Thomas Date: Sun, 3 Nov 2024 14:46:46 -0800 Subject: [PATCH] tests: simplify derived data test configuration Summary: Now that we are using python for integration test setup, we can simplify how this works. Instead of having to provide the full JSON-formatted replacement config, we define the enabled derived data types using space-separated lists in environment variables. * `ENABLED_DERIVED_DATA` overrides the full set of enabled derived data types. * `ADDITIONAL_DERIVED_DATA` augments the default set with additional types. * `DISABLED_DERIVED_DATA` removes derived data types from the default set * `OTHER_DERIVED_DATA` enables derived data types in the `other` config. Existing tests which had overrides for this config that are now covered by the defaults have their overrides removed. Tests that just need to disable specific types (e.g. the Mercurial types) can now explicitly define this. Reviewed By: lmvasquezg Differential Revision: D65270448 fbshipit-source-id: a25c841defc3bf75ced8cb69f53eb73f189ec5f5 --- .../git_lfs/test-git-lfs-derive-and-clone.t | 2 +- .../test-gitbundle-mononoke-repo-submodule.t | 4 +- .../gitimport/test-gitimport-bad-tag.t | 4 +- ...st-gitimport-commit-with-timezone-offset.t | 4 +- .../gitimport/test-gitimport-committer.t | 2 +- .../test-gitimport-discard-submodules.t | 3 +- .../gitimport/test-gitimport-filestore.t | 2 +- .../gitimport/test-gitimport-lfs-disabled.t | 3 +- ...t-gitimport-lfs-enabled-dangling-pointer.t | 4 +- .../gitimport/test-gitimport-lfs-enabled.t | 4 +- .../gitimport/test-gitimport-octopus.t | 4 +- .../gitimport/test-gitimport-single-commit.t | 4 +- .../gitimport/test-gitimport-timezone.t | 2 +- .../test-gitimport-unroundtripable.t | 4 +- .../gitimport/test-gitimport-with-filters.t | 4 +- .../integration/gitimport/test-gitimport.t | 4 +- ...-git-server-clone-with-submodule-in-root.t | 5 +- .../test-newadmin-derive-regenerate.t | 4 +- ...t-newadmin-derived-data-multiple-configs.t | 2 +- .../test-getpackv2-multibyte-filenames.t | 2 +- eden/scm/sapling/testing/ext/mononoke.py | 74 +++++++++---------- 21 files changed, 61 insertions(+), 80 deletions(-) diff --git a/eden/mononoke/tests/integration/git_lfs/test-git-lfs-derive-and-clone.t b/eden/mononoke/tests/integration/git_lfs/test-git-lfs-derive-and-clone.t index 4a94135a5658d..530580882a5d6 100644 --- a/eden/mononoke/tests/integration/git_lfs/test-git-lfs-derive-and-clone.t +++ b/eden/mononoke/tests/integration/git_lfs/test-git-lfs-derive-and-clone.t @@ -6,7 +6,7 @@ $ . "${TEST_FIXTURES}/library.sh" $ REPOTYPE="blob_files" - $ GIT_LFS_INTERPRET_POINTERS=1 ENABLED_DERIVED_DATA='["git_commits", "git_trees", "git_delta_manifests_v2", "unodes", "filenodes", "hgchangesets", "ccsm", "skeleton_manifests"]' setup_common_config $REPOTYPE + $ GIT_LFS_INTERPRET_POINTERS=1 setup_common_config $REPOTYPE $ testtool_drawdag -R repo << EOF > A-B-C > # bookmark: C heads/master_bookmark diff --git a/eden/mononoke/tests/integration/gitbundle/test-gitbundle-mononoke-repo-submodule.t b/eden/mononoke/tests/integration/gitbundle/test-gitbundle-mononoke-repo-submodule.t index 342ed7fbee7a0..c3a41729c1c09 100644 --- a/eden/mononoke/tests/integration/gitbundle/test-gitbundle-mononoke-repo-submodule.t +++ b/eden/mononoke/tests/integration/gitbundle/test-gitbundle-mononoke-repo-submodule.t @@ -5,14 +5,14 @@ # directory of this source tree. $ . "${TEST_FIXTURES}/library.sh" - $ REPOTYPE="blob_files" - $ ENABLED_DERIVED_DATA='["ccsm", "skeleton_manifests", "skeleton_manifests_v2", "git_commits", "git_trees", "git_delta_manifests_v2", "unodes"]' setup_common_config $REPOTYPE $ GIT_REPO_ORIGIN="${TESTTMP}/origin/repo-git" $ GIT_REPO_SUBMODULE="${TESTTMP}/origin/repo-submodule" $ GIT_REPO="${TESTTMP}/repo-git" $ HG_REPO="${TESTTMP}/repo" $ BUNDLE_PATH="${TESTTMP}/repo_bundle.bundle" +Disable Mercurial types as they do not support git submodules + $ DISABLED_DERIVED_DATA="filenodes hgchangesets hg_augmented_manifests" setup_common_config blob_files # Setup submodule git repository $ mkdir -p "$GIT_REPO_SUBMODULE" diff --git a/eden/mononoke/tests/integration/gitimport/test-gitimport-bad-tag.t b/eden/mononoke/tests/integration/gitimport/test-gitimport-bad-tag.t index 98842911fa229..a5fd7b77f344e 100644 --- a/eden/mononoke/tests/integration/gitimport/test-gitimport-bad-tag.t +++ b/eden/mononoke/tests/integration/gitimport/test-gitimport-bad-tag.t @@ -5,10 +5,8 @@ # directory of this source tree. $ . "${TEST_FIXTURES}/library.sh" - $ ENABLED_DERIVED_DATA='["ccsm", "unodes", "git_commits", "git_trees", "git_delta_manifests_v2"]' setup_common_config $ GIT_REPO="${TESTTMP}/repo-git" - $ REPOTYPE="blob_files" - $ ENABLED_DERIVED_DATA='["ccsm", "unodes", "git_commits", "git_trees", "git_delta_manifests_v2"]' setup_common_config $REPOTYPE + $ setup_common_config blob_files # Setup git repository $ mkdir -p "$GIT_REPO" diff --git a/eden/mononoke/tests/integration/gitimport/test-gitimport-commit-with-timezone-offset.t b/eden/mononoke/tests/integration/gitimport/test-gitimport-commit-with-timezone-offset.t index 0e482020c3587..2a98246b6e418 100644 --- a/eden/mononoke/tests/integration/gitimport/test-gitimport-commit-with-timezone-offset.t +++ b/eden/mononoke/tests/integration/gitimport/test-gitimport-commit-with-timezone-offset.t @@ -5,10 +5,8 @@ # directory of this source tree. $ . "${TEST_FIXTURES}/library.sh" - $ ENABLED_DERIVED_DATA='["unodes", "git_commits", "git_trees", "git_delta_manifests_v2"]' setup_common_config $ GIT_REPO="${TESTTMP}/repo-git" - $ REPOTYPE="blob_files" - $ ENABLED_DERIVED_DATA='["unodes", "git_commits", "git_trees", "git_delta_manifests_v2"]' setup_common_config $REPOTYPE + $ setup_common_config blob_files # Setup git repository $ mkdir -p "$GIT_REPO" diff --git a/eden/mononoke/tests/integration/gitimport/test-gitimport-committer.t b/eden/mononoke/tests/integration/gitimport/test-gitimport-committer.t index 66a5326d62971..40948c79c0322 100644 --- a/eden/mononoke/tests/integration/gitimport/test-gitimport-committer.t +++ b/eden/mononoke/tests/integration/gitimport/test-gitimport-committer.t @@ -5,9 +5,9 @@ # directory of this source tree. $ . "${TEST_FIXTURES}/library.sh" - $ HG_SET_COMMITTER_EXTRA=true ENABLED_DERIVED_DATA='["git_trees", "filenodes", "hgchangesets"]' setup_common_config $ GIT_REPO="${TESTTMP}/repo-git" $ HG_REPO="${TESTTMP}/repo" + $ HG_SET_COMMITTER_EXTRA=true setup_common_config # Setup git repsitory $ mkdir "$GIT_REPO" diff --git a/eden/mononoke/tests/integration/gitimport/test-gitimport-discard-submodules.t b/eden/mononoke/tests/integration/gitimport/test-gitimport-discard-submodules.t index e84cfdf9d9bdd..efe954dcd8e61 100644 --- a/eden/mononoke/tests/integration/gitimport/test-gitimport-discard-submodules.t +++ b/eden/mononoke/tests/integration/gitimport/test-gitimport-discard-submodules.t @@ -5,8 +5,7 @@ # directory of this source tree. $ . "${TEST_FIXTURES}/library.sh" - $ REPOTYPE="blob_files" - $ ENABLED_DERIVED_DATA='["ccsm", "skeleton_manifests", "skeleton_manifests_v2", "git_commits", "git_trees", "git_delta_manifests_v2", "unodes", "filenodes", "hgchangesets"]' setup_common_config $REPOTYPE + $ setup_common_config blob_files $ GIT_REPO_ORIGIN="${TESTTMP}/origin/repo-git" $ GIT_REPO_SUBMODULE="${TESTTMP}/origin/repo-submodule" $ GIT_REPO="${TESTTMP}/repo-git" diff --git a/eden/mononoke/tests/integration/gitimport/test-gitimport-filestore.t b/eden/mononoke/tests/integration/gitimport/test-gitimport-filestore.t index bf751bffce82b..e92e022a22009 100644 --- a/eden/mononoke/tests/integration/gitimport/test-gitimport-filestore.t +++ b/eden/mononoke/tests/integration/gitimport/test-gitimport-filestore.t @@ -7,9 +7,9 @@ $ export FILESTORE=1 $ export FILESTORE_CHUNK_SIZE=10 $ . "${TEST_FIXTURES}/library.sh" - $ ENABLED_DERIVED_DATA='["git_trees", "filenodes", "hgchangesets"]' setup_common_config $ GIT_REPO="${TESTTMP}/repo-git" $ HG_REPO="${TESTTMP}/repo" + $ setup_common_config blob_files # Setup git repsitory $ mkdir "$GIT_REPO" diff --git a/eden/mononoke/tests/integration/gitimport/test-gitimport-lfs-disabled.t b/eden/mononoke/tests/integration/gitimport/test-gitimport-lfs-disabled.t index ce7d1b29a4743..635b2dcd3267e 100644 --- a/eden/mononoke/tests/integration/gitimport/test-gitimport-lfs-disabled.t +++ b/eden/mononoke/tests/integration/gitimport/test-gitimport-lfs-disabled.t @@ -6,8 +6,7 @@ $ . "${TEST_FIXTURES}/library.sh" $ . "${TEST_FIXTURES}/library-git-lfs.sh" - $ REPOTYPE="blob_files" - $ ENABLED_DERIVED_DATA='["git_commits", "git_trees", "git_delta_manifests_v2", "unodes", "filenodes", "hgchangesets"]' setup_common_config $REPOTYPE + $ setup_common_config blob_files Without that bit gitimport is unable to set bookmarks Use common repo setup diff --git a/eden/mononoke/tests/integration/gitimport/test-gitimport-lfs-enabled-dangling-pointer.t b/eden/mononoke/tests/integration/gitimport/test-gitimport-lfs-enabled-dangling-pointer.t index 049646753c2c4..3135859b2bce4 100644 --- a/eden/mononoke/tests/integration/gitimport/test-gitimport-lfs-enabled-dangling-pointer.t +++ b/eden/mononoke/tests/integration/gitimport/test-gitimport-lfs-enabled-dangling-pointer.t @@ -6,9 +6,7 @@ $ . "${TEST_FIXTURES}/library.sh" $ . "${TEST_FIXTURES}/library-git-lfs.sh" - $ REPOTYPE="blob_files" - $ ENABLED_DERIVED_DATA='["git_commits", "git_trees", "git_delta_manifests_v2", "unodes", "filenodes", "hgchangesets"]' setup_common_config $REPOTYPE -Without that bit gitimport is unable to set bookmarks + $ setup_common_config blob_files Use common repo setup $ GIT_LFS_INTERPRET_POINTERS=1 test_repos_for_git_lfs_import diff --git a/eden/mononoke/tests/integration/gitimport/test-gitimport-lfs-enabled.t b/eden/mononoke/tests/integration/gitimport/test-gitimport-lfs-enabled.t index d90fcb2aef441..881237635869d 100644 --- a/eden/mononoke/tests/integration/gitimport/test-gitimport-lfs-enabled.t +++ b/eden/mononoke/tests/integration/gitimport/test-gitimport-lfs-enabled.t @@ -6,9 +6,7 @@ $ . "${TEST_FIXTURES}/library.sh" $ . "${TEST_FIXTURES}/library-git-lfs.sh" - $ REPOTYPE="blob_files" - $ ENABLED_DERIVED_DATA='["git_commits", "git_trees", "git_delta_manifests_v2", "unodes", "filenodes", "hgchangesets"]' setup_common_config $REPOTYPE -Without that bit gitimport is unable to set bookmarks + $ setup_common_config blob_files Use common repo setup $ GIT_LFS_INTERPRET_POINTERS=1 test_repos_for_git_lfs_import diff --git a/eden/mononoke/tests/integration/gitimport/test-gitimport-octopus.t b/eden/mononoke/tests/integration/gitimport/test-gitimport-octopus.t index c43d07e1771ec..9d6b9da354079 100644 --- a/eden/mononoke/tests/integration/gitimport/test-gitimport-octopus.t +++ b/eden/mononoke/tests/integration/gitimport/test-gitimport-octopus.t @@ -5,11 +5,9 @@ # directory of this source tree. $ . "${TEST_FIXTURES}/library.sh" - $ ENABLED_DERIVED_DATA='["git_trees", "filenodes", "hgchangesets"]' setup_common_config "blob_files" $ GIT_REPO="${TESTTMP}/repo-git" $ HG_REPO="${TESTTMP}/repo" - $ REPOTYPE="blob_files" - $ setup_common_config $REPOTYPE + $ setup_common_config blob_files $ setconfig remotenames.selectivepulldefault=master_bookmark,branch1,branch2 # Setup git repsitory diff --git a/eden/mononoke/tests/integration/gitimport/test-gitimport-single-commit.t b/eden/mononoke/tests/integration/gitimport/test-gitimport-single-commit.t index 893e3453ced54..a3a01cf86d8d6 100644 --- a/eden/mononoke/tests/integration/gitimport/test-gitimport-single-commit.t +++ b/eden/mononoke/tests/integration/gitimport/test-gitimport-single-commit.t @@ -5,11 +5,9 @@ # directory of this source tree. $ . "${TEST_FIXTURES}/library.sh" - $ ENABLED_DERIVED_DATA='["git_trees", "filenodes", "hgchangesets", "git_delta_manifests_v2"]' setup_common_config $ GIT_REPO="${TESTTMP}/repo-git" $ HG_REPO="${TESTTMP}/repo" - $ REPOTYPE="blob_files" - $ setup_common_config $REPOTYPE + $ setup_common_config blob_files # Setup git repsitory $ mkdir "$GIT_REPO" diff --git a/eden/mononoke/tests/integration/gitimport/test-gitimport-timezone.t b/eden/mononoke/tests/integration/gitimport/test-gitimport-timezone.t index 5d6b73d277dbe..37547b645c214 100644 --- a/eden/mononoke/tests/integration/gitimport/test-gitimport-timezone.t +++ b/eden/mononoke/tests/integration/gitimport/test-gitimport-timezone.t @@ -5,9 +5,9 @@ # directory of this source tree. $ . "${TEST_FIXTURES}/library.sh" - $ ENABLED_DERIVED_DATA='["git_trees", "filenodes", "hgchangesets"]' setup_common_config $ GIT_REPO="${TESTTMP}/repo-git" $ HG_REPO="${TESTTMP}/repo" + $ setup_common_config blob_files # Setup git repsitory $ mkdir "$GIT_REPO" diff --git a/eden/mononoke/tests/integration/gitimport/test-gitimport-unroundtripable.t b/eden/mononoke/tests/integration/gitimport/test-gitimport-unroundtripable.t index c52de0c2f7f57..f6532d672af12 100644 --- a/eden/mononoke/tests/integration/gitimport/test-gitimport-unroundtripable.t +++ b/eden/mononoke/tests/integration/gitimport/test-gitimport-unroundtripable.t @@ -5,10 +5,8 @@ # directory of this source tree. $ . "${TEST_FIXTURES}/library.sh" - $ ENABLED_DERIVED_DATA='["unodes", "git_commits", "git_trees", "git_delta_manifests_v2"]' setup_common_config $ GIT_REPO="${TESTTMP}/repo-git" - $ REPOTYPE="blob_files" - $ ENABLED_DERIVED_DATA='["unodes", "git_commits", "git_trees", "git_delta_manifests_v2"]' setup_common_config $REPOTYPE + $ setup_common_config blob_files # Setup git repository $ mkdir -p "$GIT_REPO" diff --git a/eden/mononoke/tests/integration/gitimport/test-gitimport-with-filters.t b/eden/mononoke/tests/integration/gitimport/test-gitimport-with-filters.t index 8eda6bfcd2f83..7d6a683b0d896 100644 --- a/eden/mononoke/tests/integration/gitimport/test-gitimport-with-filters.t +++ b/eden/mononoke/tests/integration/gitimport/test-gitimport-with-filters.t @@ -5,12 +5,10 @@ # directory of this source tree. $ . "${TEST_FIXTURES}/library.sh" - $ ENABLED_DERIVED_DATA='["git_trees", "filenodes", "hgchangesets"]' setup_common_config $ GIT_REPO_ORIGIN="${TESTTMP}/origin/repo-git" $ GIT_REPO="${TESTTMP}/repo-git" $ HG_REPO="${TESTTMP}/repo" - $ REPOTYPE="blob_files" - $ setup_common_config $REPOTYPE + $ setup_common_config blob_files # Setup git repository $ mkdir -p "$GIT_REPO_ORIGIN" diff --git a/eden/mononoke/tests/integration/gitimport/test-gitimport.t b/eden/mononoke/tests/integration/gitimport/test-gitimport.t index 8b937bbbc2d4a..afb414baddaa4 100644 --- a/eden/mononoke/tests/integration/gitimport/test-gitimport.t +++ b/eden/mononoke/tests/integration/gitimport/test-gitimport.t @@ -5,12 +5,10 @@ # directory of this source tree. $ . "${TEST_FIXTURES}/library.sh" - $ ENABLED_DERIVED_DATA='["git_trees", "filenodes", "hgchangesets"]' setup_common_config $ GIT_REPO_ORIGIN="${TESTTMP}/origin/repo-git" $ GIT_REPO="${TESTTMP}/repo-git" $ HG_REPO="${TESTTMP}/repo" - $ REPOTYPE="blob_files" - $ setup_common_config $REPOTYPE + $ setup_common_config blob_files # Setup git repository $ mkdir -p "$GIT_REPO_ORIGIN" diff --git a/eden/mononoke/tests/integration/mononoke_git_server/test-mononoke-git-server-clone-with-submodule-in-root.t b/eden/mononoke/tests/integration/mononoke_git_server/test-mononoke-git-server-clone-with-submodule-in-root.t index 4aa55e3ca55c6..1b70fa550c53d 100644 --- a/eden/mononoke/tests/integration/mononoke_git_server/test-mononoke-git-server-clone-with-submodule-in-root.t +++ b/eden/mononoke/tests/integration/mononoke_git_server/test-mononoke-git-server-clone-with-submodule-in-root.t @@ -5,12 +5,13 @@ # directory of this source tree. $ . "${TEST_FIXTURES}/library.sh" - $ REPOTYPE="blob_files" - $ ENABLED_DERIVED_DATA='["git_trees", "git_commits", "git_delta_manifests_v2", "ccsm", "skeleton_manifests", "skeleton_manifests_v2"]' setup_common_config $REPOTYPE $ GIT_REPO_ORIGIN="${TESTTMP}/origin/repo-git" $ GIT_REPO_SUBMODULE="${TESTTMP}/origin/repo-submodule" $ GIT_REPO="${TESTTMP}/repo-git" +Disable Mercurial types as they do not support git submodules + $ DISABLED_DERIVED_DATA="filenodes hgchangesets hg_augmented_manifests" setup_common_config blob_files + # Setup submodule git repository $ mkdir -p "$GIT_REPO_SUBMODULE" $ cd "$GIT_REPO_SUBMODULE" diff --git a/eden/mononoke/tests/integration/newadmin/test-newadmin-derive-regenerate.t b/eden/mononoke/tests/integration/newadmin/test-newadmin-derive-regenerate.t index 66c085bf880b6..4b21e421159e1 100644 --- a/eden/mononoke/tests/integration/newadmin/test-newadmin-derive-regenerate.t +++ b/eden/mononoke/tests/integration/newadmin/test-newadmin-derive-regenerate.t @@ -5,9 +5,9 @@ # directory of this source tree. $ . "${TEST_FIXTURES}/library.sh" - $ ENABLED_DERIVED_DATA='["git_trees", "filenodes", "hgchangesets"]' setup_common_config $ GIT_REPO="${TESTTMP}/repo-git" $ HG_REPO="${TESTTMP}/repo" + $ setup_common_config blob_files # Setup git repository $ mkdir "$GIT_REPO" @@ -68,7 +68,7 @@ # Now rederive HG_SET_COMMITTER_EXTRA=true. This changes hg hash, so we need to run with --rederive and make sure # hg hash was overwritten. - $ HG_SET_COMMITTER_EXTRA=true ENABLED_DERIVED_DATA='["git_trees", "filenodes", "hgchangesets"]' setup_common_config + $ HG_SET_COMMITTER_EXTRA=true setup_common_config blob_files # If we call derive without the rederive flag, we have the same hg hash as before $ mononoke_newadmin derived-data -R repo derive -T hgchangesets -i 1213979c6023f23e70dbe8845d773078ac1e0506bc2ab98382a329da0cb379a7 diff --git a/eden/mononoke/tests/integration/newadmin/test-newadmin-derived-data-multiple-configs.t b/eden/mononoke/tests/integration/newadmin/test-newadmin-derived-data-multiple-configs.t index 20d91fbe0dd2f..f4f4a52739644 100644 --- a/eden/mononoke/tests/integration/newadmin/test-newadmin-derived-data-multiple-configs.t +++ b/eden/mononoke/tests/integration/newadmin/test-newadmin-derived-data-multiple-configs.t @@ -8,7 +8,7 @@ $ . "${TEST_FIXTURES}/library.sh" setup configuration - $ ENABLED_DERIVED_DATA='["unodes"]' OTHER_DERIVED_DATA='["skeleton_manifests"]' setup_common_config + $ ENABLED_DERIVED_DATA="unodes" OTHER_DERIVED_DATA="skeleton_manifests" setup_common_config $ testtool_drawdag -R repo << 'EOF' > D F diff --git a/eden/mononoke/tests/integration/server/test-getpackv2-multibyte-filenames.t b/eden/mononoke/tests/integration/server/test-getpackv2-multibyte-filenames.t index f0cce4a491264..206bdc670f49d 100644 --- a/eden/mononoke/tests/integration/server/test-getpackv2-multibyte-filenames.t +++ b/eden/mononoke/tests/integration/server/test-getpackv2-multibyte-filenames.t @@ -12,7 +12,7 @@ $ NAME='file-multibyte-€' $ BOOKMARK='master_bookmark' - $ ENABLED_DERIVED_DATA='["git_trees", "filenodes", "hgchangesets"]' setup_common_config + $ setup_common_config # THis is a bit clowntown. There is a bug in our tests right now that prevents # us from creating files using UTF-8 names on Mercurial Python 3, so we diff --git a/eden/scm/sapling/testing/ext/mononoke.py b/eden/scm/sapling/testing/ext/mononoke.py index 1f95448d20d8d..5afe5c418b3ef 100644 --- a/eden/scm/sapling/testing/ext/mononoke.py +++ b/eden/scm/sapling/testing/ext/mononoke.py @@ -917,51 +917,51 @@ def append_def_config(content: str, mode: str = "a"): """.format(env.getenv("TESTTMP")) ) - if env.getenv("ENABLED_DERIVED_DATA"): - enabled_derived_data = env.getenv("ENABLED_DERIVED_DATA") - append_config( - f""" -[derived_data_config.available_configs.default] -types = {enabled_derived_data} -git_delta_manifest_version = 2 -git_delta_manifest_v2_config.max_inlined_object_size = 20 -git_delta_manifest_v2_config.max_inlined_delta_size = 20 -git_delta_manifest_v2_config.delta_chunk_size = 1000 -""" - ) + if enabled_derived_data := env.getenv("ENABLED_DERIVED_DATA"): + derived_data_types = enabled_derived_data.split() else: - append_config( - """ + derived_data_types = [ + "blame", + "changeset_info", + "deleted_manifest", + "fastlog", + "filenodes", + "fsnodes", + "git_commits", + "git_delta_manifests_v2", + "git_trees", + "unodes", + "hgchangesets", + "hg_augmented_manifests", + "skeleton_manifests", + "skeleton_manifests_v2", + "bssm_v3", + "ccsm", + "test_manifests", + "test_sharded_manifests", + ] + + if additional_derived_data := env.getenv("ADDITIONAL_DERIVED_DATA"): + derived_data_types.extend(additional_derived_data.split()) + if disabled_derived_data := env.getenv("DISABLED_DERIVED_DATA"): + derived_data_types = list( + set(derived_data_types) - set(disabled_derived_data.split()) + ) + + enabled_derived_data = json.dumps(derived_data_types) + append_config( + f""" [derived_data_config.available_configs.default] -types=[ - "blame", - "changeset_info", - "deleted_manifest", - "fastlog", - "filenodes", - "fsnodes", - "git_commits", - "git_delta_manifests_v2", - "git_trees", - "unodes", - "hgchangesets", - "hg_augmented_manifests", - "skeleton_manifests", - "skeleton_manifests_v2", - "bssm_v3", - "ccsm", - "test_manifests", - "test_sharded_manifests" -] +types = {enabled_derived_data} git_delta_manifest_version = 2 git_delta_manifest_v2_config.max_inlined_object_size = 20 git_delta_manifest_v2_config.max_inlined_delta_size = 20 git_delta_manifest_v2_config.delta_chunk_size = 1000 """ - ) + ) - if env.getenv("OTHER_DERIVED_DATA"): - other_derived_data = env.getenv("OTHER_DERIVED_DATA") + if other_derived_data := env.getenv("OTHER_DERIVED_DATA"): + other_derived_data = json.dumps(other_derived_data.split()) append_config( f""" [derived_data_config.available_configs.other]