From 5d9bcdbea238f25b5b09e2ccc790c08c0c50bdb9 Mon Sep 17 00:00:00 2001 From: Paulo Castello da Costa Date: Tue, 16 Jul 2024 16:58:32 -0700 Subject: [PATCH] [benchmark] Add a scalable_unrestricted benchmark category. The following benchmark hangs on 8-core machines, but passes on 30-core machines: ``` cpp-protobuf-async-client-unary-1channel-64wide-128breq-8mbresp-insecure ``` This change creates a `scalable_unrestricted` category in addition to the `scalable` category typically run in CI, and applies it to all `scalable` benchmarks except the one that fails on 8-core machines. This change also changes the set of benchmarks run on 8-core machines to use the `scalable_unrestricted` category. Benchmarks run on 30-core machines remain unchanged. --- .../linux/grpc_e2e_performance_gke.sh | 6 +- .../grpc_e2e_performance_gke_experiment.sh | 6 +- .../run_tests/performance/loadtest_config.py | 1 + .../performance/loadtest_examples.sh | 4 +- .../run_tests/performance/scenario_config.py | 155 ++++++++++-------- .../performance/scenario_config_exporter.py | 1 + 6 files changed, 99 insertions(+), 74 deletions(-) diff --git a/tools/internal_ci/linux/grpc_e2e_performance_gke.sh b/tools/internal_ci/linux/grpc_e2e_performance_gke.sh index 6525bd5e54ac1..23146bc4d25be 100755 --- a/tools/internal_ci/linux/grpc_e2e_performance_gke.sh +++ b/tools/internal_ci/linux/grpc_e2e_performance_gke.sh @@ -113,7 +113,7 @@ buildConfigs() { -a ci_gitCommit_java="${GRPC_JAVA_COMMIT}" \ -a ci_gitActualCommit="${KOKORO_GIT_COMMIT}" \ --prefix="${LOAD_TEST_PREFIX}" -u "${UNIQUE_IDENTIFIER}" -u "${pool}" \ - -a pool="${pool}" --category=scalable \ + -a pool="${pool}" \ --allow_client_language=c++ --allow_server_language=c++ \ --allow_server_language=node \ -o "loadtest_with_prebuilt_workers_${pool}.yaml" @@ -139,8 +139,8 @@ for language in "${disabledLanguages[@]}"; do done # Add arguments for languages. -declare -a configLangArgs8core=() -declare -a configLangArgs32core=() +declare -a configLangArgs8core=("--category=scalable_unrestricted") +declare -a configLangArgs32core=("--category=scalable") declare -a runnerLangArgs=() # c++ diff --git a/tools/internal_ci/linux/grpc_e2e_performance_gke_experiment.sh b/tools/internal_ci/linux/grpc_e2e_performance_gke_experiment.sh index f15a716028a83..5351b79f54b82 100755 --- a/tools/internal_ci/linux/grpc_e2e_performance_gke_experiment.sh +++ b/tools/internal_ci/linux/grpc_e2e_performance_gke_experiment.sh @@ -108,7 +108,7 @@ buildConfigs() { -a ci_gitCommit_java="${GRPC_JAVA_COMMIT}" \ -a ci_gitActualCommit="${KOKORO_GIT_COMMIT}" \ --prefix="${LOAD_TEST_PREFIX}" -u "${UNIQUE_IDENTIFIER}" -u "${pool}" \ - -a pool="${pool}" --category=scalable \ + -a pool="${pool}" \ --allow_client_language=c++ --allow_server_language=c++ \ --allow_server_language=node \ -o "loadtest_with_prebuilt_workers_${pool}.yaml" @@ -134,8 +134,8 @@ for language in "${disabledLanguages[@]}"; do done # Add arguments for languages. -declare -a configLangArgs8core=() -declare -a configLangArgs32core=() +declare -a configLangArgs8core=("--category=scalable_unrestricted") +declare -a configLangArgs32core=("--category=scalable") declare -a runnerLangArgs=() # c++ diff --git a/tools/run_tests/performance/loadtest_config.py b/tools/run_tests/performance/loadtest_config.py index 5ad1085c15175..a58f8f663d778 100755 --- a/tools/run_tests/performance/loadtest_config.py +++ b/tools/run_tests/performance/loadtest_config.py @@ -462,6 +462,7 @@ def main() -> None: "all", "inproc", "scalable", + "scalable_unrestricted", "smoketest", "sweep", "psm", diff --git a/tools/run_tests/performance/loadtest_examples.sh b/tools/run_tests/performance/loadtest_examples.sh index 4e7eb7c3ca08a..758a04ba0afc9 100755 --- a/tools/run_tests/performance/loadtest_examples.sh +++ b/tools/run_tests/performance/loadtest_examples.sh @@ -131,8 +131,8 @@ prebuilt_example() { } # PSM basic examples are intended to be runnable with only subsituding the -# xds-server and sidecar images, so substitution keys for xds-server and -# sidecar images are kept. +# xds-server and sidecar images, so substitution keys for xds-server and +# sidecar images are kept. psm_basic_example() { local -r scenario="${1}" local -r outputdir="${2}" diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index 0445f6bb9205c..17a31dc1d8f62 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -23,13 +23,17 @@ SMOKETEST = "smoketest" SCALABLE = "scalable" +# The category below is for C++ benchmarks that can run on 8 cores. +# TODO research why this particular benchmark hangs on 8 cores but not on 30: +# cpp-protobuf-async-client-unary-1channel-64wide-128breq-8mbresp-insecure. +SCALABLE_UNRESTRICTED = "scalable_unrestricted" INPROC = "inproc" SWEEP = "sweep" PSM = "psm" # A small superset of the benchmarks required to produce # https://grafana-dot-grpc-testing.appspot.com/ DASHBOARD = "dashboard" -DEFAULT_CATEGORIES = (SCALABLE, SMOKETEST) +DEFAULT_CATEGORIES = (SCALABLE, SCALABLE_UNRESTRICTED, SMOKETEST) SECURE_SECARGS = { "use_test_ca": True, @@ -321,7 +325,7 @@ def scenarios(self): ) # Scenario was added in https://github.com/grpc/grpc/pull/12987, but its purpose is unclear - # (beyond excercising some params that other scenarios don't) + # (beyond exercising some params that other scenarios don't) yield _ping_pong_scenario( "cpp_protobuf_async_unary_75Kqps_600channel_60Krpcs_300Breq_50Bresp", rpc_type="UNARY", @@ -336,7 +340,7 @@ def scenarios(self): secure=False, async_server_threads=16, server_threads_per_cq=1, - categories=[SCALABLE], + categories=[SCALABLE, SCALABLE_UNRESTRICTED], warmup_seconds=CXX_WARMUP_SECONDS, ) @@ -355,7 +359,7 @@ def scenarios(self): secure=secure, categories=smoketest_categories + inproc_categories - + [SCALABLE], + + [SCALABLE, SCALABLE_UNRESTRICTED], warmup_seconds=CXX_WARMUP_SECONDS, ) @@ -372,7 +376,7 @@ def scenarios(self): minimal_stack=not secure, categories=smoketest_categories + inproc_categories - + [SCALABLE], + + [SCALABLE, SCALABLE_UNRESTRICTED], warmup_seconds=CXX_WARMUP_SECONDS, ) @@ -390,7 +394,7 @@ def scenarios(self): minimal_stack=not secure, categories=smoketest_categories + inproc_categories - + [SCALABLE], + + [SCALABLE, SCALABLE_UNRESTRICTED], warmup_seconds=CXX_WARMUP_SECONDS, ) @@ -421,7 +425,8 @@ def scenarios(self): use_generic_payload=True, secure=secure, minimal_stack=not secure, - categories=inproc_categories + [SCALABLE], + categories=inproc_categories + + [SCALABLE, SCALABLE_UNRESTRICTED], channels=1, outstanding=100, warmup_seconds=CXX_WARMUP_SECONDS, @@ -439,7 +444,8 @@ def scenarios(self): use_generic_payload=True, secure=secure, minimal_stack=not secure, - categories=inproc_categories + [SCALABLE], + categories=inproc_categories + + [SCALABLE, SCALABLE_UNRESTRICTED], warmup_seconds=CXX_WARMUP_SECONDS, ) @@ -467,7 +473,8 @@ def scenarios(self): secure=secure, client_threads_per_cq=1000000, server_threads_per_cq=1000000, - categories=inproc_categories + [SCALABLE], + categories=inproc_categories + + [SCALABLE, SCALABLE_UNRESTRICTED], warmup_seconds=CXX_WARMUP_SECONDS, ) @@ -480,7 +487,8 @@ def scenarios(self): secure=secure, client_threads_per_cq=1000000, server_threads_per_cq=1000000, - categories=inproc_categories + [SCALABLE], + categories=inproc_categories + + [SCALABLE, SCALABLE_UNRESTRICTED], warmup_seconds=CXX_WARMUP_SECONDS, ) @@ -509,10 +517,11 @@ def scenarios(self): minimal_stack=not secure, categories=smoketest_categories + inproc_categories - + [SCALABLE], + + [SCALABLE, SCALABLE_UNRESTRICTED], warmup_seconds=CXX_WARMUP_SECONDS, ) + # Insecure test hangs on 8 cores: excluded from SCALABLE_UNRESTRICTED. yield _ping_pong_scenario( "cpp_protobuf_async_client_unary_1channel_64wide_128Breq_8MBresp_%s" % (secstr), @@ -525,7 +534,8 @@ def scenarios(self): resp_size=8 * 1024 * 1024, secure=secure, minimal_stack=not secure, - categories=inproc_categories + [SCALABLE], + categories=inproc_categories + + ([SCALABLE, SCALABLE_UNRESTRICTED] if secure else [SCALABLE]), warmup_seconds=CXX_WARMUP_SECONDS, ) @@ -553,7 +563,7 @@ def scenarios(self): minimal_stack=not secure, categories=smoketest_categories + inproc_categories - + [SCALABLE, DASHBOARD], + + [SCALABLE, SCALABLE_UNRESTRICTED, DASHBOARD], warmup_seconds=CXX_WARMUP_SECONDS, ) @@ -604,7 +614,7 @@ def scenarios(self): warmup_seconds=CXX_WARMUP_SECONDS, ) - maybe_scalable = [SCALABLE] + maybe_scalable = [SCALABLE, SCALABLE_UNRESTRICTED] if ( rpc_type == "streaming_from_server" and synchronicity == "async" @@ -647,7 +657,7 @@ def scenarios(self): # server_type='%s_SERVER' % synchronicity.upper(), # unconstrained_client=synchronicity, # secure=secure, - # categories=smoketest_categories+[SCALABLE], + # categories=smoketest_categories+[SCALABLE, SCALABLE_UNRESTRICTED], # warmup_seconds=CXX_WARMUP_SECONDS, # resource_quota_size=500*1024) @@ -663,7 +673,8 @@ def scenarios(self): secure=secure, messages_per_stream=mps, minimal_stack=not secure, - categories=inproc_categories + [SCALABLE], + categories=inproc_categories + + [SCALABLE, SCALABLE_UNRESTRICTED], warmup_seconds=CXX_WARMUP_SECONDS, ) @@ -733,7 +744,7 @@ def scenarios(self): client_type="ASYNC_CLIENT", server_type="ASYNC_GENERIC_SERVER", use_generic_payload=True, - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) yield _ping_pong_scenario( @@ -745,7 +756,7 @@ def scenarios(self): resp_size=1024 * 1024, use_generic_payload=True, secure=False, - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) yield _ping_pong_scenario( @@ -756,7 +767,7 @@ def scenarios(self): unconstrained_client="async", use_generic_payload=True, secure=False, - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) yield _ping_pong_scenario( @@ -771,7 +782,7 @@ def scenarios(self): rpc_type="UNARY", client_type="ASYNC_CLIENT", server_type="ASYNC_SERVER", - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) yield _ping_pong_scenario( @@ -787,7 +798,7 @@ def scenarios(self): client_type="ASYNC_CLIENT", server_type="ASYNC_SERVER", unconstrained_client="async", - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) yield _ping_pong_scenario( @@ -796,7 +807,7 @@ def scenarios(self): client_type="ASYNC_CLIENT", server_type="ASYNC_SERVER", unconstrained_client="async", - categories=[SCALABLE], + categories=[SCALABLE, SCALABLE_UNRESTRICTED], ) yield _ping_pong_scenario( @@ -806,7 +817,7 @@ def scenarios(self): server_type="SYNC_SERVER", server_language="c++", async_server_threads=1, - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) yield _ping_pong_scenario( @@ -825,7 +836,7 @@ def scenarios(self): server_type="ASYNC_SERVER", unconstrained_client="async", server_language="c++", - categories=[SCALABLE], + categories=[SCALABLE, SCALABLE_UNRESTRICTED], ) yield _ping_pong_scenario( @@ -835,7 +846,7 @@ def scenarios(self): server_type="ASYNC_SERVER", unconstrained_client="sync", server_language="c++", - categories=[SCALABLE], + categories=[SCALABLE, SCALABLE_UNRESTRICTED], ) yield _ping_pong_scenario( @@ -845,7 +856,7 @@ def scenarios(self): server_type="ASYNC_SERVER", unconstrained_client="async", client_language="c++", - categories=[SCALABLE], + categories=[SCALABLE, SCALABLE_UNRESTRICTED], ) yield _ping_pong_scenario( @@ -855,7 +866,7 @@ def scenarios(self): server_type="ASYNC_SERVER", req_size=1024 * 1024, resp_size=1024 * 1024, - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) def __str__(self): @@ -880,7 +891,7 @@ def scenarios(self): client_type="ASYNC_CLIENT", server_type="ASYNC_GENERIC_SERVER", use_generic_payload=True, - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) yield _ping_pong_scenario( @@ -892,7 +903,7 @@ def scenarios(self): resp_size=1024 * 1024, use_generic_payload=True, secure=False, - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) yield _ping_pong_scenario( @@ -903,7 +914,7 @@ def scenarios(self): unconstrained_client="async", use_generic_payload=True, secure=False, - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) yield _ping_pong_scenario( @@ -918,7 +929,7 @@ def scenarios(self): rpc_type="UNARY", client_type="ASYNC_CLIENT", server_type="ASYNC_SERVER", - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) yield _ping_pong_scenario( @@ -934,7 +945,7 @@ def scenarios(self): client_type="ASYNC_CLIENT", server_type="ASYNC_SERVER", unconstrained_client="async", - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) yield _ping_pong_scenario( @@ -943,7 +954,7 @@ def scenarios(self): client_type="ASYNC_CLIENT", server_type="ASYNC_SERVER", unconstrained_client="async", - categories=[SCALABLE], + categories=[SCALABLE, SCALABLE_UNRESTRICTED], ) yield _ping_pong_scenario( @@ -953,7 +964,7 @@ def scenarios(self): server_type="SYNC_SERVER", server_language="c++", async_server_threads=1, - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) yield _ping_pong_scenario( @@ -972,7 +983,7 @@ def scenarios(self): server_type="ASYNC_SERVER", unconstrained_client="async", server_language="c++", - categories=[SCALABLE], + categories=[SCALABLE, SCALABLE_UNRESTRICTED], ) yield _ping_pong_scenario( @@ -982,7 +993,7 @@ def scenarios(self): server_type="ASYNC_SERVER", unconstrained_client="sync", server_language="c++", - categories=[SCALABLE], + categories=[SCALABLE, SCALABLE_UNRESTRICTED], ) yield _ping_pong_scenario( @@ -992,7 +1003,7 @@ def scenarios(self): server_type="ASYNC_SERVER", unconstrained_client="async", client_language="c++", - categories=[SCALABLE], + categories=[SCALABLE, SCALABLE_UNRESTRICTED], ) yield _ping_pong_scenario( @@ -1002,7 +1013,7 @@ def scenarios(self): server_type="ASYNC_SERVER", req_size=1024 * 1024, resp_size=1024 * 1024, - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) def __str__(self): @@ -1038,7 +1049,7 @@ def scenarios(self): client_type="SYNC_CLIENT", server_type="ASYNC_GENERIC_SERVER", use_generic_payload=True, - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) yield _ping_pong_scenario( @@ -1060,7 +1071,7 @@ def scenarios(self): rpc_type="UNARY", client_type="SYNC_CLIENT", server_type="ASYNC_SERVER", - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) yield _ping_pong_scenario( @@ -1086,7 +1097,7 @@ def scenarios(self): server_type="ASYNC_SERVER", server_language="c++", async_server_threads=0, - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) yield _ping_pong_scenario( @@ -1105,7 +1116,7 @@ def scenarios(self): server_type="ASYNC_SERVER", req_size=1024 * 1024, resp_size=1024 * 1024, - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) def __str__(self): @@ -1151,7 +1162,7 @@ def scenarios(self): client_processes=0, server_processes=0, unconstrained_client="async", - categories=[SCALABLE], + categories=[SCALABLE, SCALABLE_UNRESTRICTED], ) yield _ping_pong_scenario( @@ -1165,7 +1176,7 @@ def scenarios(self): client_processes=1, server_processes=1, unconstrained_client="async", - categories=[SCALABLE], + categories=[SCALABLE, SCALABLE_UNRESTRICTED], ) yield _ping_pong_scenario( @@ -1177,7 +1188,7 @@ def scenarios(self): client_processes=1, server_processes=1, use_generic_payload=True, - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) yield _ping_pong_scenario( @@ -1188,7 +1199,7 @@ def scenarios(self): channels=1, client_processes=1, server_processes=1, - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) yield _ping_pong_scenario( @@ -1198,7 +1209,7 @@ def scenarios(self): server_type="ASYNC_SERVER", client_processes=1, server_processes=1, - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) yield _ping_pong_scenario( @@ -1209,7 +1220,7 @@ def scenarios(self): channels=1, client_processes=1, server_processes=1, - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) yield _ping_pong_scenario( @@ -1239,7 +1250,7 @@ def scenarios(self): channels=1, client_processes=1, unconstrained_client="async", - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) yield _ping_pong_scenario( @@ -1251,7 +1262,7 @@ def scenarios(self): channels=1, client_processes=0, server_language="c++", - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) yield _ping_pong_scenario( @@ -1276,7 +1287,7 @@ def scenarios(self): channels=1, client_processes=1, server_processes=1, - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) def __str__(self): @@ -1296,7 +1307,7 @@ def scenarios(self): rpc_type="STREAMING", client_type="SYNC_CLIENT", server_type="SYNC_SERVER", - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) yield _ping_pong_scenario( @@ -1304,7 +1315,7 @@ def scenarios(self): rpc_type="UNARY", client_type="SYNC_CLIENT", server_type="SYNC_SERVER", - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) yield _ping_pong_scenario( @@ -1348,7 +1359,7 @@ def scenarios(self): server_type="SYNC_SERVER", req_size=1024 * 1024, resp_size=1024 * 1024, - categories=[SMOKETEST, SCALABLE], + categories=list(DEFAULT_CATEGORIES), ) def __str__(self): @@ -1471,7 +1482,10 @@ def scenarios(self): for secure in [True, False]: secstr = "secure" if secure else "insecure" - smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE] + smoketest_categories = ([SMOKETEST] if secure else []) + [ + SCALABLE, + SCALABLE_UNRESTRICTED, + ] yield _ping_pong_scenario( "java_generic_async_streaming_ping_pong_%s" % secstr, @@ -1524,7 +1538,8 @@ def scenarios(self): unconstrained_client="async", secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS, - categories=smoketest_categories + [SCALABLE], + categories=smoketest_categories + + [SCALABLE, SCALABLE_UNRESTRICTED], ) yield _ping_pong_scenario( @@ -1535,7 +1550,7 @@ def scenarios(self): unconstrained_client="async", secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS, - categories=[SCALABLE], + categories=[SCALABLE, SCALABLE_UNRESTRICTED], ) yield _ping_pong_scenario( @@ -1547,7 +1562,7 @@ def scenarios(self): use_generic_payload=True, secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS, - categories=[SCALABLE], + categories=[SCALABLE, SCALABLE_UNRESTRICTED], ) yield _ping_pong_scenario( @@ -1593,7 +1608,10 @@ def scenarios(self): for secure in [True, False]: secstr = "secure" if secure else "insecure" - smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE] + smoketest_categories = ([SMOKETEST] if secure else []) + [ + SCALABLE, + SCALABLE_UNRESTRICTED, + ] # ASYNC_GENERIC_SERVER for Go actually uses a sync streaming server, # but that's mostly because of lack of better name of the enum value. @@ -1635,7 +1653,8 @@ def scenarios(self): server_type="SYNC_SERVER", unconstrained_client="async", secure=secure, - categories=smoketest_categories + [SCALABLE], + categories=smoketest_categories + + [SCALABLE, SCALABLE_UNRESTRICTED], ) # unconstrained_client='async' is intended (client uses goroutines) @@ -1646,7 +1665,7 @@ def scenarios(self): server_type="SYNC_SERVER", unconstrained_client="async", secure=secure, - categories=[SCALABLE], + categories=[SCALABLE, SCALABLE_UNRESTRICTED], ) # unconstrained_client='async' is intended (client uses goroutines) @@ -1660,7 +1679,7 @@ def scenarios(self): unconstrained_client="async", use_generic_payload=True, secure=secure, - categories=[SCALABLE], + categories=[SCALABLE, SCALABLE_UNRESTRICTED], ) # TODO(jtattermusch): add scenarios go vs C++ @@ -1699,7 +1718,10 @@ def scenarios(self): for secure in [True, False]: secstr = "secure" if secure else "insecure" - smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE] + smoketest_categories = ([SMOKETEST] if secure else []) + [ + SCALABLE, + SCALABLE_UNRESTRICTED, + ] yield _ping_pong_scenario( "node_to_node_generic_async_streaming_ping_pong_%s" % secstr, @@ -1739,7 +1761,8 @@ def scenarios(self): server_type="ASYNC_SERVER", unconstrained_client="async", secure=secure, - categories=smoketest_categories + [SCALABLE], + categories=smoketest_categories + + [SCALABLE, SCALABLE_UNRESTRICTED], ) yield _ping_pong_scenario( @@ -1750,7 +1773,7 @@ def scenarios(self): server_type="ASYNC_SERVER", unconstrained_client="async", secure=secure, - categories=[SCALABLE], + categories=[SCALABLE, SCALABLE_UNRESTRICTED], ) yield _ping_pong_scenario( @@ -1762,7 +1785,7 @@ def scenarios(self): unconstrained_client="async", use_generic_payload=True, secure=secure, - categories=[SCALABLE], + categories=[SCALABLE, SCALABLE_UNRESTRICTED], ) # TODO(murgatroid99): add scenarios node vs C++ diff --git a/tools/run_tests/performance/scenario_config_exporter.py b/tools/run_tests/performance/scenario_config_exporter.py index 436557f5691d3..218f8d4364798 100755 --- a/tools/run_tests/performance/scenario_config_exporter.py +++ b/tools/run_tests/performance/scenario_config_exporter.py @@ -192,6 +192,7 @@ def main() -> None: "all", "inproc", "scalable", + "scalable_unrestricted", "smoketest", "sweep", "psm",