From 2136a38252ca79846f18380aad312b6b2c7bf69c Mon Sep 17 00:00:00 2001 From: antoshkka Date: Thu, 5 Oct 2023 11:51:28 +0300 Subject: [PATCH] feat core: implement generic tracing logic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ref: https://github.com/userver-framework/userver/issues/366 Tests: протестировано локально и в CI --- .mapping.json | 13 +- core/functional_tests/CMakeLists.txt | 3 + core/functional_tests/tracing/CMakeLists.txt | 6 + .../tracing/dynamic_config_fallback.json | 76 +++++ .../functional_tests/tracing/echo_no_body.cpp | 64 ++++ .../tracing/static_config.yaml | 60 ++++ .../tracing/tests/conftest.py | 22 ++ .../tests/test_trace_headers_propagation.py | 199 +++++++++++++ core/include/userver/clients/http/request.hpp | 5 +- .../components/common_component_list.hpp | 1 + .../common_server_component_list.hpp | 1 - core/include/userver/tracing/manager.hpp | 64 +++- .../userver/tracing/manager_component.hpp | 22 +- core/include/userver/tracing/span.hpp | 17 +- core/src/clients/http/client.cpp | 19 +- core/src/clients/http/component.cpp | 10 +- core/src/clients/http/request.cpp | 9 +- core/src/clients/http/request_state.cpp | 5 +- core/src/clients/http/request_state.hpp | 3 +- .../src/clients/http/tracing_manager_test.cpp | 23 +- core/src/components/common_component_list.cpp | 2 + .../common_server_component_list.cpp | 1 - core/src/components/component_list_test.hpp | 3 +- core/src/components/tracer.cpp | 3 +- core/src/tracing/manager.cpp | 277 +++++++++++++++++- core/src/tracing/manager_component.cpp | 44 ++- ...opentracing.cpp => opentracing_logger.cpp} | 2 +- .../tracing/opentracing_logger.hpp} | 1 + core/src/tracing/span_opentracing.cpp | 5 +- core/src/tracing/span_test.cpp | 3 +- core/src/tracing/tracing_benchmark.cpp | 3 +- .../include/userver/utest/http_client.hpp | 7 + core/testing/src/utest/http_client.cpp | 20 +- dynamic_config_fallbacks.yaml | 1 + samples/hello_service/tests/test_hello.py | 2 + samples/production_service/tests/test_ping.py | 5 + scripts/docs/en/userver/logging.md | 6 +- .../include/userver/http/common_headers.hpp | 15 + .../userver/http/predefined_header.hpp | 8 +- .../include/userver/tracing/opentelemetry.hpp | 28 ++ universal/include/userver/utils/expected.hpp | 18 +- .../userver/yaml_config/yaml_config.hpp | 2 +- universal/src/tracing/opentelemetry.cpp | 95 ++++++ universal/src/tracing/opentelemetry_test.cpp | 58 ++++ 44 files changed, 1158 insertions(+), 73 deletions(-) create mode 100644 core/functional_tests/tracing/CMakeLists.txt create mode 100644 core/functional_tests/tracing/dynamic_config_fallback.json create mode 100644 core/functional_tests/tracing/echo_no_body.cpp create mode 100644 core/functional_tests/tracing/static_config.yaml create mode 100644 core/functional_tests/tracing/tests/conftest.py create mode 100644 core/functional_tests/tracing/tests/test_trace_headers_propagation.py rename core/src/tracing/{opentracing.cpp => opentracing_logger.cpp} (92%) rename core/{include/userver/tracing/opentracing.hpp => src/tracing/opentracing_logger.hpp} (99%) create mode 100644 universal/include/userver/tracing/opentelemetry.hpp create mode 100644 universal/src/tracing/opentelemetry.cpp create mode 100644 universal/src/tracing/opentelemetry_test.cpp diff --git a/.mapping.json b/.mapping.json index 495906a36778..225d9f5b4c14 100644 --- a/.mapping.json +++ b/.mapping.json @@ -235,6 +235,12 @@ "core/functional_tests/metrics/tests/static/metrics_values.txt":"taxi/uservices/userver/core/functional_tests/metrics/tests/static/metrics_values.txt", "core/functional_tests/metrics/tests/test_metrics.py":"taxi/uservices/userver/core/functional_tests/metrics/tests/test_metrics.py", "core/functional_tests/service.yaml":"taxi/uservices/userver/core/functional_tests/service.yaml", + "core/functional_tests/tracing/CMakeLists.txt":"taxi/uservices/userver/core/functional_tests/tracing/CMakeLists.txt", + "core/functional_tests/tracing/dynamic_config_fallback.json":"taxi/uservices/userver/core/functional_tests/tracing/dynamic_config_fallback.json", + "core/functional_tests/tracing/echo_no_body.cpp":"taxi/uservices/userver/core/functional_tests/tracing/echo_no_body.cpp", + "core/functional_tests/tracing/static_config.yaml":"taxi/uservices/userver/core/functional_tests/tracing/static_config.yaml", + "core/functional_tests/tracing/tests/conftest.py":"taxi/uservices/userver/core/functional_tests/tracing/tests/conftest.py", + "core/functional_tests/tracing/tests/test_trace_headers_propagation.py":"taxi/uservices/userver/core/functional_tests/tracing/tests/test_trace_headers_propagation.py", "core/functional_tests/uctl/CMakeLists.txt":"taxi/uservices/userver/core/functional_tests/uctl/CMakeLists.txt", "core/functional_tests/uctl/config_vars.yaml":"taxi/uservices/userver/core/functional_tests/uctl/config_vars.yaml", "core/functional_tests/uctl/dynamic_config_fallback.json":"taxi/uservices/userver/core/functional_tests/uctl/dynamic_config_fallback.json", @@ -527,7 +533,6 @@ "core/include/userver/tracing/manager.hpp":"taxi/uservices/userver/core/include/userver/tracing/manager.hpp", "core/include/userver/tracing/manager_component.hpp":"taxi/uservices/userver/core/include/userver/tracing/manager_component.hpp", "core/include/userver/tracing/noop.hpp":"taxi/uservices/userver/core/include/userver/tracing/noop.hpp", - "core/include/userver/tracing/opentracing.hpp":"taxi/uservices/userver/core/include/userver/tracing/opentracing.hpp", "core/include/userver/tracing/scope_time.hpp":"taxi/uservices/userver/core/include/userver/tracing/scope_time.hpp", "core/include/userver/tracing/set_throttle_reason.hpp":"taxi/uservices/userver/core/include/userver/tracing/set_throttle_reason.hpp", "core/include/userver/tracing/span.hpp":"taxi/uservices/userver/core/include/userver/tracing/span.hpp", @@ -1235,7 +1240,8 @@ "core/src/tracing/no_log_spans.cpp":"taxi/uservices/userver/core/src/tracing/no_log_spans.cpp", "core/src/tracing/no_log_spans.hpp":"taxi/uservices/userver/core/src/tracing/no_log_spans.hpp", "core/src/tracing/noop.cpp":"taxi/uservices/userver/core/src/tracing/noop.cpp", - "core/src/tracing/opentracing.cpp":"taxi/uservices/userver/core/src/tracing/opentracing.cpp", + "core/src/tracing/opentracing_logger.cpp":"taxi/uservices/userver/core/src/tracing/opentracing_logger.cpp", + "core/src/tracing/opentracing_logger.hpp":"taxi/uservices/userver/core/src/tracing/opentracing_logger.hpp", "core/src/tracing/scope_time.cpp":"taxi/uservices/userver/core/src/tracing/scope_time.cpp", "core/src/tracing/scope_time_test.cpp":"taxi/uservices/userver/core/src/tracing/scope_time_test.cpp", "core/src/tracing/set_throttle_reason.cpp":"taxi/uservices/userver/core/src/tracing/set_throttle_reason.cpp", @@ -3127,6 +3133,7 @@ "universal/include/userver/logging/log_helper_fwd.hpp":"taxi/uservices/userver/universal/include/userver/logging/log_helper_fwd.hpp", "universal/include/userver/logging/null_logger.hpp":"taxi/uservices/userver/universal/include/userver/logging/null_logger.hpp", "universal/include/userver/logging/stacktrace_cache.hpp":"taxi/uservices/userver/universal/include/userver/logging/stacktrace_cache.hpp", + "universal/include/userver/tracing/opentelemetry.hpp":"taxi/uservices/userver/universal/include/userver/tracing/opentelemetry.hpp", "universal/include/userver/utest/assert_macros.hpp":"taxi/uservices/userver/universal/include/userver/utest/assert_macros.hpp", "universal/include/userver/utest/death_tests.hpp":"taxi/uservices/userver/universal/include/userver/utest/death_tests.hpp", "universal/include/userver/utest/impl/assert_macros.hpp":"taxi/uservices/userver/universal/include/userver/utest/impl/assert_macros.hpp", @@ -3373,6 +3380,8 @@ "universal/src/logging/rate_limit.cpp":"taxi/uservices/userver/universal/src/logging/rate_limit.cpp", "universal/src/logging/rate_limit.hpp":"taxi/uservices/userver/universal/src/logging/rate_limit.hpp", "universal/src/logging/stacktrace_cache.cpp":"taxi/uservices/userver/universal/src/logging/stacktrace_cache.cpp", + "universal/src/tracing/opentelemetry.cpp":"taxi/uservices/userver/universal/src/tracing/opentelemetry.cpp", + "universal/src/tracing/opentelemetry_test.cpp":"taxi/uservices/userver/universal/src/tracing/opentelemetry_test.cpp", "universal/src/utest/assert_macros.cpp":"taxi/uservices/userver/universal/src/utest/assert_macros.cpp", "universal/src/utest/assert_macros_test.cpp":"taxi/uservices/userver/universal/src/utest/assert_macros_test.cpp", "universal/src/utils/algo_test.cpp":"taxi/uservices/userver/universal/src/utils/algo_test.cpp", diff --git a/core/functional_tests/CMakeLists.txt b/core/functional_tests/CMakeLists.txt index cf29e03911a2..768ddc098d0e 100644 --- a/core/functional_tests/CMakeLists.txt +++ b/core/functional_tests/CMakeLists.txt @@ -8,6 +8,9 @@ add_dependencies(${PROJECT_NAME} ${PROJECT_NAME}-basic-chaos) add_subdirectory(metrics) add_dependencies(${PROJECT_NAME} ${PROJECT_NAME}-metrics) +add_subdirectory(tracing) +add_dependencies(${PROJECT_NAME} ${PROJECT_NAME}-tracing) + add_subdirectory(uctl) add_dependencies(${PROJECT_NAME} ${PROJECT_NAME}-uctl) diff --git a/core/functional_tests/tracing/CMakeLists.txt b/core/functional_tests/tracing/CMakeLists.txt new file mode 100644 index 000000000000..2e19e8ccd01d --- /dev/null +++ b/core/functional_tests/tracing/CMakeLists.txt @@ -0,0 +1,6 @@ +project(userver-core-tests-tracing CXX) + +add_executable(${PROJECT_NAME} "echo_no_body.cpp") +target_link_libraries(${PROJECT_NAME} userver-core) + +userver_chaos_testsuite_add() diff --git a/core/functional_tests/tracing/dynamic_config_fallback.json b/core/functional_tests/tracing/dynamic_config_fallback.json new file mode 100644 index 000000000000..f14f60bb851e --- /dev/null +++ b/core/functional_tests/tracing/dynamic_config_fallback.json @@ -0,0 +1,76 @@ +{ + "BAGGAGE_SETTINGS": { + "allowed_keys": [] + }, + "HTTP_CLIENT_CONNECTION_POOL_SIZE": 1000, + "HTTP_CLIENT_CONNECT_THROTTLE": { + "max-size": 100, + "token-update-interval-ms": 0 + }, + "USERVER_BAGGAGE_ENABLED": false, + "USERVER_CACHES": {}, + "USERVER_CANCEL_HANDLE_REQUEST_BY_DEADLINE": false, + "USERVER_CHECK_AUTH_IN_HANDLERS": true, + "USERVER_DEADLINE_PROPAGATION_ENABLED": true, + "USERVER_DUMPS": {}, + "USERVER_FILES_CONTENT_TYPE_MAP": { + ".css": "text/css", + ".gif": "image/gif", + ".htm": "text/html", + ".html": "text/html", + ".jpeg": "image/jpeg", + ".js": "application/javascript", + ".json": "application/json", + ".md": "text/markdown", + ".png": "image/png", + ".svg": "image/svg+xml", + "__default__": "text/plain" + }, + "USERVER_HANDLER_STREAM_API_ENABLED": false, + "USERVER_HTTP_PROXY": "", + "USERVER_LOG_DYNAMIC_DEBUG": { + "force-disabled": [], + "force-enabled": [] + }, + "USERVER_LOG_REQUEST": true, + "USERVER_LOG_REQUEST_HEADERS": false, + "USERVER_LRU_CACHES": {}, + "USERVER_NO_LOG_SPANS": { + "names": [], + "prefixes": [] + }, + "USERVER_RPS_CCONTROL": { + "down-level": 1, + "down-rate-percent": 2, + "min-limit": 10, + "no-limit-seconds": 1000, + "overload-off-seconds": 3, + "overload-on-seconds": 3, + "up-level": 2, + "up-rate-percent": 2 + }, + "USERVER_RPS_CCONTROL_ACTIVATED_FACTOR_METRIC": 5, + "USERVER_RPS_CCONTROL_CUSTOM_STATUS": {}, + "USERVER_RPS_CCONTROL_ENABLED": false, + "USERVER_TASK_PROCESSOR_PROFILER_DEBUG": { + "fs-task-processor": { + "enabled": false, + "execution-slice-threshold-us": 1000000 + }, + "main-task-processor": { + "enabled": false, + "execution-slice-threshold-us": 2000 + } + }, + "USERVER_TASK_PROCESSOR_QOS": { + "default-service": { + "default-task-processor": { + "wait_queue_overload": { + "action": "ignore", + "length_limit": 5000, + "time_limit_us": 3000 + } + } + } + } +} diff --git a/core/functional_tests/tracing/echo_no_body.cpp b/core/functional_tests/tracing/echo_no_body.cpp new file mode 100644 index 000000000000..fa5d5f162236 --- /dev/null +++ b/core/functional_tests/tracing/echo_no_body.cpp @@ -0,0 +1,64 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +class EchoNoBody final : public server::handlers::HttpHandlerBase { + public: + static constexpr std::string_view kName = "handler-echo-no-body"; + + EchoNoBody(const components::ComponentConfig& config, + const components::ComponentContext& context) + : HttpHandlerBase(config, context), + echo_url_{config["echo-url"].As()}, + http_client_( + context.FindComponent().GetHttpClient()) {} + + std::string HandleRequestThrow( + const server::http::HttpRequest&, + server::request::RequestContext&) const override { + auto response = http_client_.CreateRequest() + .get(echo_url_) + .retry(2) + .timeout(std::chrono::seconds{5}) + .perform(); + response->raise_for_status(); + return {}; + } + + static yaml_config::Schema GetStaticConfigSchema() { + return yaml_config::MergeSchemas(R"( + type: object + description: HTTP echo without body component + additionalProperties: false + properties: + echo-url: + type: string + description: some other microservice listens on this URL + )"); + } + + private: + const std::string echo_url_; + clients::http::Client& http_client_; +}; + +int main(int argc, char* argv[]) { + const auto component_list = components::MinimalServerComponentList() + .Append() + .Append() + .Append() + .Append() + .Append(); + return utils::DaemonMain(argc, argv, component_list); +} diff --git a/core/functional_tests/tracing/static_config.yaml b/core/functional_tests/tracing/static_config.yaml new file mode 100644 index 000000000000..5b203afedde2 --- /dev/null +++ b/core/functional_tests/tracing/static_config.yaml @@ -0,0 +1,60 @@ +components_manager: + components: + handler-echo-no-body: + echo-url: 'mockserver/v1/translations' # Some other microservice listens on this URL + path: /echo-no-body + method: GET + task_processor: main-task-processor + + http-client: + fs-task-processor: fs-task-processor + + dns-client: + fs-task-processor: fs-task-processor + + testsuite-support: + tests-control: + # Some options from server::handlers::HttpHandlerBase + path: /tests/{action} + method: POST + task_processor: main-task-processor + + server: + listener: + port: 8089 + task_processor: main-task-processor + logging: + fs-task-processor: fs-task-processor + loggers: + default: + file_path: '@stderr' + level: debug + overflow_behavior: discard + + tracer: + service-name: http-trcin-test + + tracing-manager-locator: + incomming-format: ['taxi', 'yandex', 'b3-alternative', 'opentelemetry'] + new-requests-format: ['taxi', 'yandex', 'b3-alternative', 'opentelemetry'] + + dynamic-config: # Dynamic config storage options, do nothing + fs-cache-path: '' + dynamic-config-fallbacks: # Load options from file and push them into the dynamic config storage. + fallback-path: /etc/http_caching/dynamic_config_fallback.json + + coro_pool: + initial_size: 500 # Preallocate 500 coroutines at startup. + max_size: 1000 # Do not keep more than 1000 preallocated coroutines. + + task_processors: # Task processor is an executor for coroutine tasks + + main-task-processor: # Make a task processor for CPU-bound coroutine tasks. + worker_threads: 4 # Process tasks in 4 threads. + thread_name: main-worker # OS will show the threads of this task processor with 'main-worker' prefix. + + fs-task-processor: # Make a separate task processor for filesystem bound tasks. + thread_name: fs-worker + worker_threads: 4 + + default_task_processor: main-task-processor diff --git a/core/functional_tests/tracing/tests/conftest.py b/core/functional_tests/tracing/tests/conftest.py new file mode 100644 index 000000000000..9689a875d2fb --- /dev/null +++ b/core/functional_tests/tracing/tests/conftest.py @@ -0,0 +1,22 @@ +import pytest + + +pytest_plugins = ['pytest_userver.plugins.core'] + +USERVER_CONFIG_HOOKS = ['userver_config_echo_url'] + + +@pytest.fixture(scope='session') +def userver_config_echo_url(mockserver_info): + def do_patch(config_yaml, config_vars): + components = config_yaml['components_manager']['components'] + components['handler-echo-no-body']['echo-url'] = mockserver_info.url( + '/test-service/echo-no-body', + ) + + return do_patch + + +@pytest.fixture +def taxi_test_service(service_client): + return service_client diff --git a/core/functional_tests/tracing/tests/test_trace_headers_propagation.py b/core/functional_tests/tracing/tests/test_trace_headers_propagation.py new file mode 100644 index 000000000000..22fb80221885 --- /dev/null +++ b/core/functional_tests/tracing/tests/test_trace_headers_propagation.py @@ -0,0 +1,199 @@ +B3_HEADERS = { + 'X-B3-TraceId': '10e1afed08e019fc1110464cfa66635c', + 'X-B3-SpanId': '7a085853722dc6d2', + 'X-B3-Sampled': '1', +} + +OPENTELEMETRY_TRACE_ID = '20e1afed08e019fc1110464cfa66635c' +OPENTELEMETRY_HEADERS = { + 'traceparent': f'00-{OPENTELEMETRY_TRACE_ID}-7a085853722dc6d2-01', + 'tracestate': '42', +} + +TAXI_HEADERS = { + 'X-YaTraceId': '30e1afed08e019fc1110464cfa66635c', + 'X-YaSpanId': '7a085853722dc6d2', +} + +TAXI_HEADERS_EXT = {'X-YaRequestId': '40e1afed08e019fc1110464cfa66635c'} + +YANDEX_HEADERS = {'X-RequestId': '50e1afed08e019fc1110464cfa66635c'} + + +async def test_empty_b3_tracing_headers(taxi_test_service, mockserver): + @mockserver.json_handler('/test-service/echo-no-body') + async def _handler(request): + assert 'X-B3-TraceId' in request.headers + assert 'X-B3-Sampled' in request.headers + return mockserver.make_response() + + response = await taxi_test_service.get('/echo-no-body') + assert _handler.times_called >= 1 + assert response.status_code == 200 + + +async def test_empty_otel_tracing_headers(taxi_test_service, mockserver): + @mockserver.json_handler('/test-service/echo-no-body') + async def _handler(request): + assert 'traceparent' in request.headers + return mockserver.make_response() + + response = await taxi_test_service.get('/echo-no-body') + assert _handler.times_called >= 1 + assert response.status_code == 200 + + +async def test_empty_taxi_tracing_headers(taxi_test_service, mockserver): + @mockserver.json_handler('/test-service/echo-no-body') + async def _handler(request): + assert 'X-YaTraceId' in request.headers + assert 'X-YaRequestId' in request.headers + assert 'X-YaSpanId' in request.headers + return mockserver.make_response() + + response = await taxi_test_service.get('/echo-no-body') + assert _handler.times_called >= 1 + assert response.status_code == 200 + + +async def test_empty_yandex_tracing_headers(taxi_test_service, mockserver): + @mockserver.json_handler('/test-service/echo-no-body') + async def _handler(request): + assert 'X-RequestId' in request.headers + assert request.headers['X-RequestId'] == request.headers['X-YaTraceId'] + return mockserver.make_response() + + response = await taxi_test_service.get('/echo-no-body') + assert _handler.times_called >= 1 + assert response.status_code == 200 + + +async def test_b3_tracing_headers(taxi_test_service, mockserver): + @mockserver.json_handler('/test-service/echo-no-body') + async def _handler(request): + assert request.headers['X-YaTraceId'] == B3_HEADERS['X-B3-TraceId'] + assert request.headers['X-B3-TraceId'] == B3_HEADERS['X-B3-TraceId'] + assert request.headers['X-B3-Sampled'] == B3_HEADERS['X-B3-Sampled'] + assert request.headers['X-YaSpanId'] == request.headers['X-B3-SpanId'] + return mockserver.make_response() + + response = await taxi_test_service.get('/echo-no-body', headers=B3_HEADERS) + assert _handler.times_called >= 1 + assert response.status_code == 200 + + +async def test_otel_tracing_headers(taxi_test_service, mockserver): + @mockserver.json_handler('/test-service/echo-no-body') + async def _handler(request): + assert request.headers['X-YaTraceId'] == OPENTELEMETRY_TRACE_ID + assert ( + request.headers['traceparent'].split('-')[0] + == OPENTELEMETRY_HEADERS['traceparent'].split('-')[0] + ) + assert ( + request.headers['traceparent'].split('-')[1] + == OPENTELEMETRY_HEADERS['traceparent'].split('-')[1] + ) + assert ( + request.headers['traceparent'].split('-')[3] + == OPENTELEMETRY_HEADERS['traceparent'].split('-')[3] + ), request.headers['traceparent'] + assert ( + request.headers['tracestate'] + == OPENTELEMETRY_HEADERS['tracestate'] + ) + return mockserver.make_response() + + response = await taxi_test_service.get( + '/echo-no-body', headers=OPENTELEMETRY_HEADERS, + ) + assert _handler.times_called >= 1 + assert response.status_code == 200 + + +async def test_taxi_tracing_headers(taxi_test_service, mockserver): + @mockserver.json_handler('/test-service/echo-no-body') + async def _handler(request): + assert request.headers['X-YaTraceId'] == TAXI_HEADERS['X-YaTraceId'] + assert 'X-YaRequestId' in request.headers + assert request.headers['X-YaSpanId'] != TAXI_HEADERS['X-YaSpanId'] + return mockserver.make_response() + + response = await taxi_test_service.get( + '/echo-no-body', headers=TAXI_HEADERS, + ) + assert _handler.times_called >= 1 + assert response.status_code == 200 + + +async def test_taxi_tracing_headers_ext(taxi_test_service, mockserver): + @mockserver.json_handler('/test-service/echo-no-body') + async def _handler(request): + assert request.headers['X-YaTraceId'] == TAXI_HEADERS['X-YaTraceId'] + assert ( + request.headers['X-YaRequestId'] + != TAXI_HEADERS_EXT['X-YaRequestId'] + ) + assert request.headers['X-YaSpanId'] != TAXI_HEADERS['X-YaSpanId'] + return mockserver.make_response() + + response = await taxi_test_service.get( + '/echo-no-body', headers={**TAXI_HEADERS, **TAXI_HEADERS_EXT}, + ) + assert _handler.times_called >= 1 + assert response.status_code == 200 + + +async def test_taxi_tracing_headers_min(taxi_test_service, mockserver): + @mockserver.json_handler('/test-service/echo-no-body') + async def _handler(request): + assert request.headers['X-YaTraceId'] == TAXI_HEADERS['X-YaTraceId'] + assert 'X-YaRequestId' in request.headers + assert 'X-YaSpanId' in request.headers + return mockserver.make_response() + + response = await taxi_test_service.get( + '/echo-no-body', headers={'X-YaTraceId': TAXI_HEADERS['X-YaTraceId']}, + ) + assert _handler.times_called >= 1 + assert response.status_code == 200 + + +async def test_yandex_tracing_headers(taxi_test_service, mockserver): + @mockserver.json_handler('/test-service/echo-no-body') + async def _handler(request): + assert request.headers['X-RequestId'] == YANDEX_HEADERS['X-RequestId'] + assert request.headers['X-RequestId'] == request.headers['X-YaTraceId'] + return mockserver.make_response() + + response = await taxi_test_service.get( + '/echo-no-body', headers=YANDEX_HEADERS, + ) + assert _handler.times_called >= 1 + assert response.status_code == 200 + + +async def test_priority_otel_tracing_headers(taxi_test_service, mockserver): + @mockserver.json_handler('/test-service/echo-no-body') + async def _handler(request): + assert request.headers['X-YaTraceId'] == OPENTELEMETRY_TRACE_ID + return mockserver.make_response() + + response = await taxi_test_service.get( + '/echo-no-body', headers={**TAXI_HEADERS, **OPENTELEMETRY_HEADERS}, + ) + assert _handler.times_called >= 1 + assert response.status_code == 200 + + +async def test_priority_b3_tracing_headers(taxi_test_service, mockserver): + @mockserver.json_handler('/test-service/echo-no-body') + async def _handler(request): + assert request.headers['X-YaTraceId'] == B3_HEADERS['X-B3-TraceId'] + return mockserver.make_response() + + response = await taxi_test_service.get( + '/echo-no-body', headers={**TAXI_HEADERS, **B3_HEADERS}, + ) + assert _handler.times_called >= 1 + assert response.status_code == 200 diff --git a/core/include/userver/clients/http/request.hpp b/core/include/userver/clients/http/request.hpp index ffbcac808df1..3815fe1e6ff6 100644 --- a/core/include/userver/clients/http/request.hpp +++ b/core/include/userver/clients/http/request.hpp @@ -96,7 +96,8 @@ class Request final { std::shared_ptr&& req_stats, const std::shared_ptr& dest_stats, clients::dns::Resolver* resolver, - impl::PluginPipeline& plugin_pipeline); + impl::PluginPipeline& plugin_pipeline, + const tracing::TracingManagerBase& tracing_manager); /// @endcond /// Specifies method @@ -300,6 +301,8 @@ class Request final { Request& DisableReplyDecoding() &; Request DisableReplyDecoding() &&; + /// Override the default tracing manager from HTTP client for this + /// particular request. Request& SetTracingManager(const tracing::TracingManagerBase&) &; Request SetTracingManager(const tracing::TracingManagerBase&) &&; diff --git a/core/include/userver/components/common_component_list.hpp b/core/include/userver/components/common_component_list.hpp index 477e9d1b7aae..4cd838d0f3b0 100644 --- a/core/include/userver/components/common_component_list.hpp +++ b/core/include/userver/components/common_component_list.hpp @@ -30,6 +30,7 @@ namespace components { /// * components::DynamicConfigClient /// * components::DynamicConfigClientUpdater /// * engine::TaskProcessorsLoadMonitor +/// * tracing::DefaultTracingManagerLocator ComponentList CommonComponentList(); } // namespace components diff --git a/core/include/userver/components/common_server_component_list.hpp b/core/include/userver/components/common_server_component_list.hpp index 565f0f113773..743aba56b9cb 100644 --- a/core/include/userver/components/common_server_component_list.hpp +++ b/core/include/userver/components/common_server_component_list.hpp @@ -30,7 +30,6 @@ namespace components { /// * server::handlers::auth::NonceCacheSettingsComponent /// * congestion_control::Component /// * components::HttpServerSettings -/// * tracing::DefaultTracingManagerLocator ComponentList CommonServerComponentList(); } // namespace components diff --git a/core/include/userver/tracing/manager.hpp b/core/include/userver/tracing/manager.hpp index 45281532f184..61f8f548a7e8 100644 --- a/core/include/userver/tracing/manager.hpp +++ b/core/include/userver/tracing/manager.hpp @@ -7,6 +7,7 @@ #include #include #include +#include USERVER_NAMESPACE_BEGIN @@ -35,7 +36,7 @@ class TracingManagerBase { const server::http::HttpRequest& request, SpanBuilder& span_builder) const = 0; - /// Fill request with tracing information + /// Fill new client requests with tracing information virtual void FillRequestWithTracingContext( const Span& span, clients::http::RequestTracingEditor request) const = 0; @@ -44,10 +45,59 @@ class TracingManagerBase { const Span& span, server::http::HttpResponse& response) const = 0; }; -/// @brief Used as default tracing manager. -/// Provides methods for working with usual Yandex.Taxi tracing headers. -class DefaultTracingManager final : public TracingManagerBase { +// clang-format off +enum class Format : short { + /// Yandex Taxi/Lavka/Eda/... tracing: + /// @code + /// http::headers::kXYaTraceId -> tracing::Span::GetTraceId() -> http::headers::kXYaTraceId + /// http::headers::kXYaRequestId -> tracing::Span::GetParentLink(); tracing::Span::GetLink() -> http::headers::kXYaRequestId + /// http::headers::kXYaSpanId -> tracing::Span::GetParentId(); tracing::Span::GetSpanId() -> http::headers::kXYaSpanId + /// @endcode + kYandexTaxi = 1 << 1, + + /// Yandex Search tracing: + /// http::headers::kXRequestId -> tracing::Span::GetTraceId() -> http::headers::kXRequestId + kYandex = 1 << 2, + + /// Use http::headers::opentelemetry::kTraceState and + /// http::headers::opentelemetry::kTraceParent headers to fill the + /// tracing::opentelemetry::TraceParentData as per OpenTelemetry. + kOpenTelemetry = 1 << 3, + + /// Openzipkin b3 alternative propagation, where Span ID goes to partern ID: + /// @code + /// b3::kTraceId -> tracing::Span::GetTraceId() -> b3::kTraceId + /// b3::kSpanId -> tracing::Span::GetParentId(); tracing::Span::GetSpanId() -> b3::kSpanId + /// span.GetParentId() -> b3::kParentSpanId + /// @endcode + /// See https://github.com/openzipkin/b3-propagation for more info. + kB3Alternative = 1 << 4, +}; +// clang-format on + +/// Converts a textual representation of format into tracing::Format enum. +Format FormatFromString(std::string_view format); + +bool TryFillSpanBuilderFromRequest(Format format, + const server::http::HttpRequest& request, + SpanBuilder& span_builder); + +void FillRequestWithTracingContext(Format format, const tracing::Span& span, + clients::http::RequestTracingEditor request); + +void FillResponseWithTracingContext(Format format, const Span& span, + server::http::HttpResponse& response); + +/// @brief Generic tracing manager that knows about popular tracing +/// headers and allows customising input and output headers. +class GenericTracingManager final : public TracingManagerBase { public: + GenericTracingManager() = delete; + + GenericTracingManager(utils::Flags in_request_response, + utils::Flags new_request) + : in_request_response_{in_request_response}, new_request_{new_request} {} + bool TryFillSpanBuilderFromRequest(const server::http::HttpRequest& request, SpanBuilder& span_builder) const override; @@ -57,9 +107,11 @@ class DefaultTracingManager final : public TracingManagerBase { void FillResponseWithTracingContext( const Span& span, server::http::HttpResponse& response) const override; -}; -extern const DefaultTracingManager kDefaultTracingManager; + private: + const utils::Flags in_request_response_; + const utils::Flags new_request_; +}; } // namespace tracing diff --git a/core/include/userver/tracing/manager_component.hpp b/core/include/userver/tracing/manager_component.hpp index ca1ac64b46b7..1169771be89d 100644 --- a/core/include/userver/tracing/manager_component.hpp +++ b/core/include/userver/tracing/manager_component.hpp @@ -26,16 +26,29 @@ class TracingManagerComponentBase : public components::LoggableComponentBase, /// @ingroup userver_components /// -/// @brief Locator component that provides access to the actual TracingManager -/// that will be used in handlers and clients unless specified otherwise +/// @brief Component that provides access to the actual TracingManager +/// that is used in handlers and clients. +/// +/// This component allows conversion of tracing formats and allows working with +/// multiple tracing formats. For example: +/// @code +/// # yaml +/// incomming-format: ['opentelemetry', 'taxi'] +/// new-requests-format: ['b3-alternative', 'opentelemetry'] +/// @endcode +/// means that tracing data is extracted from OpenTelemetry headers if they +/// were received or from Yandex-Taxi specific headers. The outgoing requests +/// will have the tracing::Format::kB3Alternative headers and OpenTelemetry +/// headers at the same time. /// /// The component can be configured in service config. -/// If the config is not provided, then tracing::kDefaultTracingManager will be used /// /// ## Static options: /// Name | Description | Default value /// ---- | ----------- | ------------- -/// component-name | name of the component, that implements TracingManagerComponentBase | +/// component-name | name of the component, that implements TracingManagerComponentBase | +/// incomming-format | Array of incomming tracing formats supported by tracing::FormatFromString | ['taxi'] +/// new-requests-format | Send tracing data in those formats supported by tracing::FormatFromString | ['taxi'] /// // clang-format on class DefaultTracingManagerLocator final @@ -53,6 +66,7 @@ class DefaultTracingManagerLocator final static yaml_config::Schema GetStaticConfigSchema(); private: + GenericTracingManager default_manager_; const TracingManagerBase& tracing_manager_; }; diff --git a/core/include/userver/tracing/span.hpp b/core/include/userver/tracing/span.hpp index 05612fd8c2d2..92bacb55eeae 100644 --- a/core/include/userver/tracing/span.hpp +++ b/core/include/userver/tracing/span.hpp @@ -133,17 +133,30 @@ class Span final { /// it is set and greater than the main log level of the Span. std::optional GetLocalLogLevel() const; - /// Set link. Can be called only once. + /// Set link - a requst ID within a service. Can be called only once. + /// + /// Propagates within a single service, but not from client to server. A new + /// link is generated for the "root" request handling task void SetLink(std::string link); - /// Set parent_link. Can be called only once. + /// Set parent_link - an ID . Can be called only once. void SetParentLink(std::string parent_link); + /// Get link - a request ID within the service. + /// + /// Propagates within a single service, but not from client to server. A new + /// link is generated for the "root" request handling task std::string GetLink() const; std::string GetParentLink() const; + /// An ID of the request that does not change from service to service. + /// + /// Propagates both to sub-spans within a single service, and from client + /// to server const std::string& GetTraceId() const; + + /// Identifies a specific span. It does not propagate const std::string& GetSpanId() const; const std::string& GetParentId() const; diff --git a/core/src/clients/http/client.cpp b/core/src/clients/http/client.cpp index 4f8442e6a3af..19454035efe0 100644 --- a/core/src/clients/http/client.cpp +++ b/core/src/clients/http/client.cpp @@ -39,10 +39,8 @@ long ClampToLong(size_t value) { const tracing::TracingManagerBase* GetTracingManager( const impl::ClientSettings& settings) { - if (settings.tracing_manager) { - return settings.tracing_manager; - } - return &tracing::kDefaultTracingManager; + UASSERT(settings.tracing_manager); + return settings.tracing_manager; } } // namespace @@ -122,8 +120,10 @@ Request Client::CreateRequest() { auto idx = FindMultiIndex(easy->GetMulti()); auto wrapper = std::make_shared(std::move(easy), *this); - return Request{std::move(wrapper), statistics_[idx].CreateRequestStats(), - destination_statistics_, resolver_, plugin_pipeline_}; + return Request{ + std::move(wrapper), statistics_[idx].CreateRequestStats(), + destination_statistics_, resolver_, + plugin_pipeline_, *tracing_manager_.GetBase()}; } else { auto i = utils::RandRange(multis_.size()); auto& multi = multis_[i]; @@ -133,8 +133,10 @@ Request Client::CreateRequest() { return std::make_shared( easy_.Get()->GetBoundBlocking(*multi), *this); }).Get(); - return Request{std::move(wrapper), statistics_[i].CreateRequestStats(), - destination_statistics_, resolver_, plugin_pipeline_}; + return Request{ + std::move(wrapper), statistics_[i].CreateRequestStats(), + destination_statistics_, resolver_, + plugin_pipeline_, *tracing_manager_.GetBase()}; } catch (engine::WaitInterruptedException&) { throw clients::http::CancelException(); } catch (engine::TaskCancelledException&) { @@ -149,7 +151,6 @@ Request Client::CreateRequest() { auto urls = allowed_urls_extra_.Read(); request.SetAllowedUrlsExtra(*urls); - request.SetTracingManager(*tracing_manager_.GetBase()); request.SetHeadersPropagator(headers_propagator_); if (user_agent_) { diff --git a/core/src/clients/http/component.cpp b/core/src/clients/http/component.cpp index 1decb0059938..c639b04eea3f 100644 --- a/core/src/clients/http/component.cpp +++ b/core/src/clients/http/component.cpp @@ -30,13 +30,9 @@ clients::http::impl::ClientSettings GetClientSettings( const ComponentConfig& component_config, const ComponentContext& context) { clients::http::impl::ClientSettings settings; settings = component_config.As(); - auto* tracing_locator = - context.FindComponentOptional(); - if (tracing_locator) { - settings.tracing_manager = &tracing_locator->GetTracingManager(); - } else { - settings.tracing_manager = &tracing::kDefaultTracingManager; - } + auto& tracing_locator = + context.FindComponent(); + settings.tracing_manager = &tracing_locator.GetTracingManager(); auto* propagator_component = context.FindComponentOptional(); if (propagator_component) { diff --git a/core/src/clients/http/request.cpp b/core/src/clients/http/request.cpp index 8897cae9913b..926930866998 100644 --- a/core/src/clients/http/request.cpp +++ b/core/src/clients/http/request.cpp @@ -210,10 +210,11 @@ Request::Request(std::shared_ptr&& wrapper, std::shared_ptr&& req_stats, const std::shared_ptr& dest_stats, clients::dns::Resolver* resolver, - impl::PluginPipeline& plugin_pipeline) - : pimpl_(std::make_shared(std::move(wrapper), - std::move(req_stats), dest_stats, - resolver, plugin_pipeline)) { + impl::PluginPipeline& plugin_pipeline, + const tracing::TracingManagerBase& tracing_manager) + : pimpl_(std::make_shared( + std::move(wrapper), std::move(req_stats), dest_stats, resolver, + plugin_pipeline, tracing_manager)) { LOG_TRACE() << "Request::Request()"; // default behavior follow redirects and verify ssl pimpl_->follow_redirects(true); diff --git a/core/src/clients/http/request_state.cpp b/core/src/clients/http/request_state.cpp index ac2a6c7d3893..b851ff8d8928 100644 --- a/core/src/clients/http/request_state.cpp +++ b/core/src/clients/http/request_state.cpp @@ -162,13 +162,14 @@ RequestState::RequestState( std::shared_ptr&& wrapper, std::shared_ptr&& req_stats, const std::shared_ptr& dest_stats, - clients::dns::Resolver* resolver, impl::PluginPipeline& plugin_pipeline) + clients::dns::Resolver* resolver, impl::PluginPipeline& plugin_pipeline, + const tracing::TracingManagerBase& tracing_manager) : easy_(std::move(wrapper)), stats_(std::move(req_stats)), dest_stats_(dest_stats), original_timeout_(kDefaultTimeout), remote_timeout_(original_timeout_), - tracing_manager_{&tracing::kDefaultTracingManager}, + tracing_manager_{tracing_manager}, is_cancelled_(false), errorbuffer_(), resolver_{resolver}, diff --git a/core/src/clients/http/request_state.hpp b/core/src/clients/http/request_state.hpp index 4c29fda090f1..9be707b354d9 100644 --- a/core/src/clients/http/request_state.hpp +++ b/core/src/clients/http/request_state.hpp @@ -47,7 +47,8 @@ class RequestState : public std::enable_shared_from_this { std::shared_ptr&& req_stats, const std::shared_ptr& dest_stats, clients::dns::Resolver* resolver, - impl::PluginPipeline& plugin_pipeline); + impl::PluginPipeline& plugin_pipeline, + const tracing::TracingManagerBase& tracing_manager); ~RequestState(); using Queue = concurrent::StringStreamQueue; diff --git a/core/src/clients/http/tracing_manager_test.cpp b/core/src/clients/http/tracing_manager_test.cpp index 9e63af8fbc77..77c4e3fe7df2 100644 --- a/core/src/clients/http/tracing_manager_test.cpp +++ b/core/src/clients/http/tracing_manager_test.cpp @@ -43,6 +43,28 @@ class MockTracingManager : public tracing::TracingManagerBase { } // namespace UTEST(TracingManagerBase, TracingManagerCorrectCalls) { + MockTracingManager tracing_manager; + auto http_client_ptr = utest::CreateHttpClient(tracing_manager); + + const utest::SimpleServer http_server_final{ + clients::http::Response200WithHeader{"xxx: test"}}; + + const auto url = http_server_final.GetBaseUrl(); + auto& http_client = *http_client_ptr; + std::string data{}; + + const auto response = http_client.CreateRequest() + .post(url, data) + .timeout(std::chrono::seconds(1)) + .perform(); + + EXPECT_TRUE(response->IsOk()); + EXPECT_EQ(tracing_manager.GetCreateNewSpanCounter(), 0); + EXPECT_EQ(tracing_manager.GetFillRequestCounter(), 1); + EXPECT_EQ(tracing_manager.GetFillResponseCounter(), 0); +} + +UTEST(TracingManagerBase, TracingManagerCorrectCallsPerRequest) { auto http_client_ptr = utest::CreateHttpClient(); const utest::SimpleServer http_server_final{ @@ -53,7 +75,6 @@ UTEST(TracingManagerBase, TracingManagerCorrectCalls) { std::string data{}; MockTracingManager tracing_manager; - const auto response = http_client.CreateRequest() .post(url, data) .timeout(std::chrono::seconds(1)) diff --git a/core/src/components/common_component_list.cpp b/core/src/components/common_component_list.cpp index c7044b95ff73..8d0856e9eb15 100644 --- a/core/src/components/common_component_list.cpp +++ b/core/src/components/common_component_list.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include USERVER_NAMESPACE_BEGIN @@ -35,6 +36,7 @@ ComponentList CommonComponentList() { .Append() .Append() .Append("http-client-statistics") + .Append() .Append() .Append() .Append() diff --git a/core/src/components/common_server_component_list.cpp b/core/src/components/common_server_component_list.cpp index 49c481834c65..de2c223b155d 100644 --- a/core/src/components/common_server_component_list.cpp +++ b/core/src/components/common_server_component_list.cpp @@ -30,7 +30,6 @@ ComponentList CommonServerComponentList() { .Append() .Append() .Append() - .Append() .Append() .Append(); } diff --git a/core/src/components/component_list_test.hpp b/core/src/components/component_list_test.hpp index f05a2e6e3f72..57e9f3de37a9 100644 --- a/core/src/components/component_list_test.hpp +++ b/core/src/components/component_list_test.hpp @@ -4,13 +4,14 @@ #include #include -#include #include #include #include #include +#include + USERVER_NAMESPACE_BEGIN namespace tests { diff --git a/core/src/components/tracer.cpp b/core/src/components/tracer.cpp index faee618f47fb..45ed7b272af4 100644 --- a/core/src/components/tracer.cpp +++ b/core/src/components/tracer.cpp @@ -3,10 +3,11 @@ #include #include #include -#include #include #include +#include + USERVER_NAMESPACE_BEGIN namespace components { diff --git a/core/src/tracing/manager.cpp b/core/src/tracing/manager.cpp index 9afdd6e11d10..aa8413ddcbc4 100644 --- a/core/src/tracing/manager.cpp +++ b/core/src/tracing/manager.cpp @@ -1,18 +1,149 @@ #include +#include #include #include +#include +#include +#include USERVER_NAMESPACE_BEGIN namespace tracing { -bool DefaultTracingManager::TryFillSpanBuilderFromRequest( - const server::http::HttpRequest& request, SpanBuilder& span_builder) const { +namespace { + +constexpr std::string_view kSampledTag = "sampled"; +constexpr std::string_view kDefaultOtelTraceFlags = "00"; + +// The order matter for TryFillSpanBuilderFromRequest as it returns on first +// success +constexpr Format kAllFormatsOrdered[] = { + Format::kOpenTelemetry, + Format::kB3Alternative, + Format::kYandexTaxi, + Format::kYandex, +}; + +/// @brief Per-request data that should be available inside handlers +/// https://opentelemetry.io +struct OTelTracingHeadersInheritedData final { + std::string tracestate; + /// traceflags, 2 bytes + std::string traceflags; +}; + +/// @see TracingHeadersInheritedData for details on the contents. +engine::TaskInheritedVariable + kOTelTracingHeadersInheritedData; + +bool B3TryFillSpanBuilderFromRequest(const server::http::HttpRequest& request, + tracing::SpanBuilder& span_builder) { + namespace b3 = http::headers::b3; + const auto& trace_id = request.GetHeader(b3::kTraceId); + if (trace_id.empty()) { + return false; + } + + const auto& sampled = request.GetHeader(b3::kSampled); + if (sampled.empty()) { + return false; + } + + span_builder.SetTraceId(trace_id); + span_builder.SetParentSpanId(request.GetHeader(b3::kSpanId)); + span_builder.AddTagFrozen(std::string{kSampledTag}, sampled); + return true; +} + +template +void B3FillWithTracingContext(const tracing::Span& span, T& target) { + namespace b3 = http::headers::b3; + target.SetHeader(b3::kTraceId, span.GetTraceId()); + target.SetHeader(b3::kSpanId, span.GetSpanId()); + target.SetHeader(b3::kParentSpanId, span.GetParentId()); + + const auto& sampled = server::request::GetTaskInheritedHeader(b3::kSampled); + if (!sampled.empty()) { + target.SetHeader(b3::kSampled, sampled); + } else { + target.SetHeader(b3::kSampled, "1"); + } +} + +bool OpenTelemetryTryFillSpanBuilderFromRequest( + const server::http::HttpRequest& request, + tracing::SpanBuilder& span_builder) { + namespace opentelemetry = http::headers::opentelemetry; + const auto& traceparent = request.GetHeader(opentelemetry::kTraceParent); + if (traceparent.empty()) { + return false; + } + + auto extraction_result = + tracing::opentelemetry::ExtractTraceParentData(traceparent); + if (!extraction_result.has_value()) { + LOG_LIMITED_WARNING() << fmt::format( + "Invalid traceparent header format ({}). Skipping Opentelemetry " + "headers", + extraction_result.error()); + return false; + } + + auto data = std::move(extraction_result).value(); + + span_builder.SetTraceId(std::move(data.trace_id)); + span_builder.SetParentSpanId(std::move(data.span_id)); + if (data.trace_flags.empty()) { + data.trace_flags = std::string{kDefaultOtelTraceFlags}; + } + + const auto& tracestate = request.GetHeader(opentelemetry::kTraceState); + kOTelTracingHeadersInheritedData.Set({ + tracestate, + std::move(data.trace_flags), + }); + return true; +} + +template +void OpenTelemetryFillWithTracingContext(const tracing::Span& span, T& target) { + const auto* data = kOTelTracingHeadersInheritedData.GetOptional(); + + std::string_view traceflags = kDefaultOtelTraceFlags; + if (data) { + traceflags = data->traceflags; + } + auto traceparent_result = opentelemetry::BuildTraceParentHeader( + span.GetTraceId(), span.GetSpanId(), traceflags); + + if (!traceparent_result.has_value()) { + LOG_LIMITED_WARNING() << fmt::format( + "Cannot build opentelemetry traceparent header ({})", + traceparent_result.error()); + return; + } + + target.SetHeader(http::headers::opentelemetry::kTraceParent, + std::move(traceparent_result.value())); + if (data && !data->tracestate.empty()) { + target.SetHeader(http::headers::opentelemetry::kTraceState, + data->tracestate); + } +} + +bool YandexTaxiTryFillSpanBuilderFromRequest( + const server::http::HttpRequest& request, + tracing::SpanBuilder& span_builder) { const auto& trace_id = request.GetHeader(http::headers::kXYaTraceId); - if (!trace_id.empty()) span_builder.SetTraceId(std::move(trace_id)); + if (trace_id.empty()) { + return false; + } + + span_builder.SetTraceId(trace_id); - span_builder.SetParentSpanId(request.GetHeader(http::headers::kXYaSpanId)); + const auto& parent_span_id = request.GetHeader(http::headers::kXYaSpanId); + span_builder.SetParentSpanId(parent_span_id); const auto& parent_link = request.GetHeader(http::headers::kXYaRequestId); if (!parent_link.empty()) span_builder.SetParentLink(parent_link); @@ -20,23 +151,141 @@ bool DefaultTracingManager::TryFillSpanBuilderFromRequest( return true; } -void DefaultTracingManager::FillRequestWithTracingContext( +template +void YandexTaxiFillWithTracingContext(const tracing::Span& span, T& target) { + target.SetHeader(http::headers::kXYaRequestId, span.GetLink()); + target.SetHeader(http::headers::kXYaTraceId, span.GetTraceId()); + target.SetHeader(http::headers::kXYaSpanId, span.GetSpanId()); +} + +bool YandexTryFillSpanBuilderFromRequest( + const server::http::HttpRequest& request, + tracing::SpanBuilder& span_builder) { + const auto& trace_id = request.GetHeader(http::headers::kXRequestId); + if (trace_id.empty()) { + return false; + } + + span_builder.SetTraceId(trace_id); + return true; +} + +template +void YandexFillWithTracingContext(const tracing::Span& span, T& target) { + target.SetHeader(http::headers::kXRequestId, span.GetTraceId()); +} + +} // namespace + +Format FormatFromString(std::string_view format) { + constexpr utils::TrivialBiMap kToFormat = [](auto selector) { + return selector() + .Case("b3-alternative", Format::kB3Alternative) + .Case("opentelemetry", Format::kOpenTelemetry) + .Case("taxi", Format::kYandexTaxi) + .Case("yandex", Format::kYandex); + }; + + auto value = kToFormat.TryFind(format); + if (!value) { + throw std::runtime_error( + fmt::format("Unknown tracing format '{}' (must be one of {})", format, + kToFormat.DescribeFirst())); + } + return *value; +} + +bool TryFillSpanBuilderFromRequest(Format format, + const server::http::HttpRequest& request, + SpanBuilder& span_builder) { + switch (format) { + case Format::kYandexTaxi: + return YandexTaxiTryFillSpanBuilderFromRequest(request, span_builder); + case Format::kYandex: + return YandexTryFillSpanBuilderFromRequest(request, span_builder); + case Format::kOpenTelemetry: + return OpenTelemetryTryFillSpanBuilderFromRequest(request, span_builder); + case Format::kB3Alternative: + return B3TryFillSpanBuilderFromRequest(request, span_builder); + } + + UINVARIANT(false, "Unexpected format of tracing headers"); +} + +void FillRequestWithTracingContext( + Format format, const tracing::Span& span, + clients::http::RequestTracingEditor request) { + switch (format) { + case Format::kYandexTaxi: + YandexTaxiFillWithTracingContext(span, request); + return; + case Format::kYandex: + YandexFillWithTracingContext(span, request); + return; + case Format::kOpenTelemetry: + OpenTelemetryFillWithTracingContext(span, request); + return; + case Format::kB3Alternative: + B3FillWithTracingContext(span, request); + return; + } + + UINVARIANT(false, "Unexpected format of tracing headers"); +} + +void FillResponseWithTracingContext(Format format, const Span& span, + server::http::HttpResponse& response) { + switch (format) { + case Format::kYandexTaxi: + YandexTaxiFillWithTracingContext(span, response); + return; + case Format::kYandex: + YandexFillWithTracingContext(span, response); + return; + case Format::kOpenTelemetry: + OpenTelemetryFillWithTracingContext(span, response); + return; + case Format::kB3Alternative: + B3FillWithTracingContext(span, response); + return; + } + + UINVARIANT(false, "Unexpected format to send tracing headers"); +} + +bool GenericTracingManager::TryFillSpanBuilderFromRequest( + const server::http::HttpRequest& request, SpanBuilder& span_builder) const { + for (auto format : kAllFormatsOrdered) { + if (!(in_request_response_ & format)) { + continue; + } + + if (tracing::TryFillSpanBuilderFromRequest(format, request, span_builder)) { + return true; + } + } + return false; +} + +void GenericTracingManager::FillRequestWithTracingContext( const tracing::Span& span, clients::http::RequestTracingEditor request) const { - request.SetHeader(http::headers::kXYaRequestId, span.GetLink()); - request.SetHeader(http::headers::kXYaTraceId, span.GetTraceId()); - request.SetHeader(http::headers::kXYaSpanId, span.GetSpanId()); + for (auto format : kAllFormatsOrdered) { + if (new_request_ & format) { + tracing::FillRequestWithTracingContext(format, span, request); + } + } } -void DefaultTracingManager::FillResponseWithTracingContext( +void GenericTracingManager::FillResponseWithTracingContext( const Span& span, server::http::HttpResponse& response) const { - response.SetHeader(http::headers::kXYaRequestId, span.GetLink()); - response.SetHeader(http::headers::kXYaTraceId, span.GetTraceId()); - response.SetHeader(http::headers::kXYaSpanId, span.GetSpanId()); + for (auto format : kAllFormatsOrdered) { + if (in_request_response_ & format) { + tracing::FillResponseWithTracingContext(format, span, response); + } + } } -const DefaultTracingManager kDefaultTracingManager{}; - } // namespace tracing USERVER_NAMESPACE_END diff --git a/core/src/tracing/manager_component.cpp b/core/src/tracing/manager_component.cpp index c329c2612e37..8aa7d8cba821 100644 --- a/core/src/tracing/manager_component.cpp +++ b/core/src/tracing/manager_component.cpp @@ -10,7 +10,13 @@ USERVER_NAMESPACE_BEGIN namespace tracing { namespace { + +using FlagsFormat = utils::Flags; + +} // namespace + const TracingManagerBase& GetTracingManagerFromConfig( + const GenericTracingManager& default_manager, const components::ComponentConfig& config, const components::ComponentContext& context) { if (config.HasMember("component-name")) { @@ -19,9 +25,23 @@ const TracingManagerBase& GetTracingManagerFromConfig( return context.FindComponent(tracing_manager_name); } } - return kDefaultTracingManager; + return default_manager; +} + +FlagsFormat Parse(const yaml_config::YamlConfig& value, + formats::parse::To) { + utils::Flags format = tracing::Format{}; + + if (!value.IsArray()) { + format |= tracing::FormatFromString(value.As("taxi")); + } else { + for (const auto& f : value) { + format |= tracing::FormatFromString(f.As()); + } + } + + return format; } -} // namespace TracingManagerComponentBase::TracingManagerComponentBase( const components::ComponentConfig& config, @@ -32,7 +52,10 @@ DefaultTracingManagerLocator::DefaultTracingManagerLocator( const components::ComponentConfig& config, const components::ComponentContext& context) : components::LoggableComponentBase(config, context), - tracing_manager_(GetTracingManagerFromConfig(config, context)) {} + default_manager_(config["incomming-format"].As(), + config["new-requests-format"].As()), + tracing_manager_( + GetTracingManagerFromConfig(default_manager_, config, context)) {} const TracingManagerBase& DefaultTracingManagerLocator::GetTracingManager() const { @@ -48,6 +71,21 @@ additionalProperties: false component-name: type: string description: tracing manager component's name + incomming-format: + type: array + description: Incomming tracing data formats + items: &format_items + type: string + description: tracing formats + enum: + - b3-alternative + - yandex + - taxi + - opentelemetry + new-requests-format: + type: array + description: Send tracing data in those formats + items: *format_items )"); } diff --git a/core/src/tracing/opentracing.cpp b/core/src/tracing/opentracing_logger.cpp similarity index 92% rename from core/src/tracing/opentracing.cpp rename to core/src/tracing/opentracing_logger.cpp index f404f5c96f7f..91ec7866dc16 100644 --- a/core/src/tracing/opentracing.cpp +++ b/core/src/tracing/opentracing_logger.cpp @@ -1,4 +1,4 @@ -#include +#include #include diff --git a/core/include/userver/tracing/opentracing.hpp b/core/src/tracing/opentracing_logger.hpp similarity index 99% rename from core/include/userver/tracing/opentracing.hpp rename to core/src/tracing/opentracing_logger.hpp index 03e1d2d38b05..6c6686b09a42 100644 --- a/core/include/userver/tracing/opentracing.hpp +++ b/core/src/tracing/opentracing_logger.hpp @@ -1,4 +1,5 @@ #pragma once + #include #include diff --git a/core/src/tracing/span_opentracing.cpp b/core/src/tracing/span_opentracing.cpp index 3a50eb2791b8..2e83126cb321 100644 --- a/core/src/tracing/span_opentracing.cpp +++ b/core/src/tracing/span_opentracing.cpp @@ -2,13 +2,14 @@ #include -#include #include #include #include -#include #include +#include +#include + USERVER_NAMESPACE_BEGIN namespace tracing { diff --git a/core/src/tracing/span_test.cpp b/core/src/tracing/span_test.cpp index a8788be943e1..afcd6106a852 100644 --- a/core/src/tracing/span_test.cpp +++ b/core/src/tracing/span_test.cpp @@ -7,12 +7,13 @@ #include #include #include -#include #include #include #include #include +#include + USERVER_NAMESPACE_BEGIN class Span : public LoggingTest {}; diff --git a/core/src/tracing/tracing_benchmark.cpp b/core/src/tracing/tracing_benchmark.cpp index ff61642c3bb9..10d8de58efeb 100644 --- a/core/src/tracing/tracing_benchmark.cpp +++ b/core/src/tracing/tracing_benchmark.cpp @@ -3,7 +3,8 @@ #include #include #include -#include + +#include USERVER_NAMESPACE_BEGIN diff --git a/core/testing/include/userver/utest/http_client.hpp b/core/testing/include/userver/utest/http_client.hpp index 9a5cd7142710..40de51b2c431 100644 --- a/core/testing/include/userver/utest/http_client.hpp +++ b/core/testing/include/userver/utest/http_client.hpp @@ -12,6 +12,10 @@ namespace engine { class TaskProcessor; } +namespace tracing { +class TracingManagerBase; +} + namespace utest { std::shared_ptr CreateHttpClient(); @@ -19,6 +23,9 @@ std::shared_ptr CreateHttpClient(); std::shared_ptr CreateHttpClient( engine::TaskProcessor& fs_task_processor); +std::shared_ptr CreateHttpClient( + const tracing::TracingManagerBase& tracing_manager); + } // namespace utest USERVER_NAMESPACE_END diff --git a/core/testing/src/utest/http_client.cpp b/core/testing/src/utest/http_client.cpp index 68809cd6489b..2a04b8dd45e0 100644 --- a/core/testing/src/utest/http_client.cpp +++ b/core/testing/src/utest/http_client.cpp @@ -2,6 +2,7 @@ #include #include +#include #include USERVER_NAMESPACE_BEGIN @@ -9,17 +10,34 @@ USERVER_NAMESPACE_BEGIN namespace utest { std::shared_ptr CreateHttpClient() { - return CreateHttpClient(engine::current_task::GetTaskProcessor()); + return utest::CreateHttpClient(engine::current_task::GetTaskProcessor()); } + std::shared_ptr CreateHttpClient( engine::TaskProcessor& fs_task_processor) { + static const tracing::GenericTracingManager kDefaultTracingManager{ + tracing::Format::kYandexTaxi, tracing::Format::kYandexTaxi}; + clients::http::impl::ClientSettings static_config; static_config.io_threads = 1; + static_config.tracing_manager = &kDefaultTracingManager; + return std::make_shared( std::move(static_config), fs_task_processor, std::vector>{}); } +std::shared_ptr CreateHttpClient( + const tracing::TracingManagerBase& tracing_manager) { + clients::http::impl::ClientSettings static_config; + static_config.io_threads = 1; + static_config.tracing_manager = &tracing_manager; + + return std::make_shared( + std::move(static_config), engine::current_task::GetTaskProcessor(), + std::vector>{}); +} + } // namespace utest USERVER_NAMESPACE_END diff --git a/dynamic_config_fallbacks.yaml b/dynamic_config_fallbacks.yaml index 86771eb3bdcd..4a0ad52027d1 100644 --- a/dynamic_config_fallbacks.yaml +++ b/dynamic_config_fallbacks.yaml @@ -15,6 +15,7 @@ groups: - core/functional_tests/basic_chaos/dynamic_config_fallback.json - core/functional_tests/metrics/dynamic_config_fallback.json - core/functional_tests/uctl/dynamic_config_fallback.json + - core/functional_tests/tracing/dynamic_config_fallback.json - conan/test_package/hello_service/dynamic_config_fallback.json - core/functional_tests/cache_update/dynamic_config_fallback.json - samples/mysql_service/dynamic_config_fallback.json diff --git a/samples/hello_service/tests/test_hello.py b/samples/hello_service/tests/test_hello.py index 37601b63cd0c..712408365029 100644 --- a/samples/hello_service/tests/test_hello.py +++ b/samples/hello_service/tests/test_hello.py @@ -3,6 +3,7 @@ async def test_ping(service_client): response = await service_client.get('/hello') assert response.status == 200 assert response.content == b'Hello world!\n' + assert 'X-RequestId' not in response.headers.keys(), 'Unexpected header' # /// [Functional test] @@ -10,3 +11,4 @@ async def test_wrong_method(service_client): response = await service_client.request('KEK', '/hello') assert response.status == 400 assert response.content == b'bad request' + assert 'X-YaRequestId' not in response.headers.keys(), 'Unexpected header' diff --git a/samples/production_service/tests/test_ping.py b/samples/production_service/tests/test_ping.py index a5e7edf1aa9d..5e873be52329 100644 --- a/samples/production_service/tests/test_ping.py +++ b/samples/production_service/tests/test_ping.py @@ -1,3 +1,8 @@ async def test_ping(service_client): response = await service_client.get('/ping') assert response.status == 200 + + # Tracing headers should be present + assert 'X-YaRequestId' in response.headers.keys() + assert 'X-YaTraceId' in response.headers.keys() + assert 'X-YaSpanId' in response.headers.keys() diff --git a/scripts/docs/en/userver/logging.md b/scripts/docs/en/userver/logging.md index 5e8a3633dc0f..4d3f253411b7 100644 --- a/scripts/docs/en/userver/logging.md +++ b/scripts/docs/en/userver/logging.md @@ -137,6 +137,8 @@ thereby building a trace of requests and interactions. It can be used to identify slow query stages, bottlenecks, sequential queries, etc. +See tracing::DefaultTracingManagerLocator for more info. + ### tracing::Span When processing a request, you can create a `tracking::Span` object that measures the execution time of the current code block (technically, the time between its constructor and destructor) and stores the resulting time in the log: @@ -199,7 +201,9 @@ The HTTP client sends the current link/span_id/trace_id values in each request t When the HTTP server handles the request, it extracts data from the request headers and puts them in the Span. -Names of the headers: +Names of the headers varry depending on tracing::DefaultTracingManagerLocator +static configuration and on the chosen tracing::Fromat value. For example, +with tracing::Fromat::kYandexTaxi the following headers would be used: ``` X-YaRequestId diff --git a/universal/include/userver/http/common_headers.hpp b/universal/include/userver/http/common_headers.hpp index 2461459c49e1..2fd6565730d9 100644 --- a/universal/include/userver/http/common_headers.hpp +++ b/universal/include/userver/http/common_headers.hpp @@ -138,6 +138,21 @@ inline constexpr PredefinedHeader kXRequestId{"X-RequestId"}; inline constexpr PredefinedHeader kXBackendServer{"X-Backend-Server"}; inline constexpr PredefinedHeader kXTaxiEnvoyProxyDstVhost{ "X-Taxi-EnvoyProxy-DstVhost"}; + +/// B3 tracing Headers +namespace b3 { +inline constexpr PredefinedHeader kTraceId{"X-B3-TraceId"}; +inline constexpr PredefinedHeader kSpanId{"X-B3-SpanId"}; +inline constexpr PredefinedHeader kSampled{"X-B3-Sampled"}; +inline constexpr PredefinedHeader kParentSpanId{"X-B3-ParentSpanId"}; +} // namespace b3 + +/// OpenTelemetry tracing Headers +namespace opentelemetry { +inline constexpr PredefinedHeader kTraceParent{"traceparent"}; +inline constexpr PredefinedHeader kTraceState{"tracestate"}; +} // namespace opentelemetry + /// @} /// @name Baggage header diff --git a/universal/include/userver/http/predefined_header.hpp b/universal/include/userver/http/predefined_header.hpp index 777bd7ae462a..082a0bfc8e10 100644 --- a/universal/include/userver/http/predefined_header.hpp +++ b/universal/include/userver/http/predefined_header.hpp @@ -115,7 +115,13 @@ inline constexpr utils::TrivialBiMap kKnownHeadersLowercaseMap = .Case("x-yataxi-client-timeoutms", 30) .Case("x-yataxi-deadline-expired", 31) .Case("x-yataxi-ratelimited-by", 32) - .Case("x-yataxi-ratelimit-reason", 33); + .Case("x-yataxi-ratelimit-reason", 33) + .Case("x-b3-traceid", 34) + .Case("x-b3-spanid", 35) + .Case("x-b3-sampled", 36) + .Case("x-b3-parentspanid", 37) + .Case("traceparent", 38) + .Case("tracestate", 39); }; // We use different values for "no index" at compile and run time to simplify diff --git a/universal/include/userver/tracing/opentelemetry.hpp b/universal/include/userver/tracing/opentelemetry.hpp new file mode 100644 index 000000000000..7155b6536ca7 --- /dev/null +++ b/universal/include/userver/tracing/opentelemetry.hpp @@ -0,0 +1,28 @@ +#pragma once + +#include + +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace tracing::opentelemetry { + +struct TraceParentData { + std::string version; + std::string trace_id; + std::string span_id; + std::string trace_flags; +}; + +utils::expected ExtractTraceParentData( + std::string_view trace_parent); + +utils::expected BuildTraceParentHeader( + std::string_view trace_id, std::string_view span_id, + std::string_view trace_flags); + +} // namespace tracing::opentelemetry + +USERVER_NAMESPACE_END diff --git a/universal/include/userver/utils/expected.hpp b/universal/include/userver/utils/expected.hpp index 22b2054bced8..d3e5f5fd5e30 100644 --- a/universal/include/userver/utils/expected.hpp +++ b/universal/include/userver/utils/expected.hpp @@ -69,9 +69,14 @@ class [[nodiscard]] expected { /// @brief Return reference to the value or throws bad_expected_access /// if it's not available /// @throws utils::bad_expected_access if *this contain an unexpected value - S& value(); + S& value() &; - const S& value() const; + /// @brief Extracts the value or throws bad_expected_access + /// if it's not available + /// @throws utils::bad_expected_access if *this contain an unexpected value + S value() &&; + + const S& value() const&; /// @brief Return reference to the error value or throws bad_expected_access /// if it's not available @@ -139,7 +144,7 @@ bool expected::has_value() const noexcept { } template -S& expected::value() { +S& expected::value() & { S* result = std::get_if(&data_); if (result == nullptr) { throw bad_expected_access( @@ -149,7 +154,12 @@ S& expected::value() { } template -const S& expected::value() const { +S expected::value() && { + return std::move(value()); +} + +template +const S& expected::value() const& { const S* result = std::get_if(&data_); if (result == nullptr) { throw bad_expected_access( diff --git a/universal/include/userver/yaml_config/yaml_config.hpp b/universal/include/userver/yaml_config/yaml_config.hpp index 81d6626842aa..b8949d88eec3 100644 --- a/universal/include/userver/yaml_config/yaml_config.hpp +++ b/universal/include/userver/yaml_config/yaml_config.hpp @@ -192,7 +192,7 @@ class YamlConfig { template T YamlConfig::As() const { static_assert(formats::common::impl::kHasParse, - "There is no `Parse(const formats::yaml_config::YamlConfig&, " + "There is no `Parse(const yaml_config::YamlConfig&, " "formats::parse::To)`" "in namespace of `T` or `formats::parse`. " "Probably you forgot to include the " diff --git a/universal/src/tracing/opentelemetry.cpp b/universal/src/tracing/opentelemetry.cpp new file mode 100644 index 000000000000..f5506b3b433b --- /dev/null +++ b/universal/src/tracing/opentelemetry.cpp @@ -0,0 +1,95 @@ +#include + +#include + +#include +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace tracing::opentelemetry { + +namespace { + +constexpr std::size_t kTraceParentSize = 55; +constexpr std::size_t kVersionSize = 2; +constexpr std::size_t kTraceIdSize = 32; +constexpr std::size_t kSpanIdSize = 16; +constexpr std::size_t kTraceFlagsSize = 2; + +constexpr char kDefaultVersion[] = "00"; + +} // namespace + +utils::expected ExtractTraceParentData( + std::string_view trace_parent) { + if (trace_parent.size() != kTraceParentSize) { + return utils::unexpected("Invalid header size"); + } + + std::vector trace_fields = + utils::text::SplitIntoStringViewVector(trace_parent, "-"); + + if (trace_fields.size() != 4) { + return utils::unexpected("Invalid fields count"); + } + + for (const auto& field : trace_fields) { + if (!utils::encoding::IsHexData(field)) { + return utils::unexpected("One of the fields is not hex data"); + } + } + + const std::string_view version{trace_fields[0]}; + const std::string_view trace_id{trace_fields[1]}; + const std::string_view span_id{trace_fields[2]}; + const std::string_view trace_flags{trace_fields[3]}; + + if (version.size() != kVersionSize || trace_id.size() != kTraceIdSize || + span_id.size() != kSpanIdSize || trace_flags.size() != kTraceFlagsSize) { + return utils::unexpected("One of the fields has invalid size"); + } + + return TraceParentData{ + std::string{version}, + std::string{trace_id}, + std::string{span_id}, + std::string{trace_flags}, + }; +} + +utils::expected BuildTraceParentHeader( + std::string_view trace_id, std::string_view span_id, + std::string_view trace_flags) { + if (trace_id.size() != kTraceIdSize) { + return utils::unexpected("Invalid trace_id size"); + } + if (!utils::encoding::IsHexData(trace_id)) { + return utils::unexpected( + fmt::format("Invalid trace_id value: '{}' is not a hex", trace_id)); + } + + if (span_id.size() != kSpanIdSize) { + return utils::unexpected("Invalid span_id size"); + } + if (!utils::encoding::IsHexData(span_id)) { + return utils::unexpected( + fmt::format("Invalid span_id value: '{}' is not a hex", span_id)); + } + + if (trace_flags.size() != kTraceFlagsSize) { + return utils::unexpected("Invalid trace_flags size"); + } + if (!utils::encoding::IsHexData(trace_flags)) { + return utils::unexpected(fmt::format( + "Invalid trace_flags value: '{}' is not a hex", trace_flags)); + } + + return fmt::format("{}-{}-{}-{}", kDefaultVersion, trace_id, span_id, + trace_flags); +} + +} // namespace tracing::opentelemetry + +USERVER_NAMESPACE_END diff --git a/universal/src/tracing/opentelemetry_test.cpp b/universal/src/tracing/opentelemetry_test.cpp new file mode 100644 index 000000000000..a6b462c8c599 --- /dev/null +++ b/universal/src/tracing/opentelemetry_test.cpp @@ -0,0 +1,58 @@ +#include + +#include +#include + +#include + +USERVER_NAMESPACE_BEGIN + +namespace opentelemetry = tracing::opentelemetry; + +TEST(OpenTelemetry, TraceParentParsing) { + const std::string input = + "00-80e1afed08e019fc1110464cfa66635c-7a085853722dc6d2-01"; + + auto result = opentelemetry::ExtractTraceParentData(input); + + ASSERT_TRUE(result.has_value()); + + auto data = result.value(); + + EXPECT_EQ(data.version, "00"); + EXPECT_EQ(data.trace_id, "80e1afed08e019fc1110464cfa66635c"); + EXPECT_EQ(data.span_id, "7a085853722dc6d2"); + EXPECT_EQ(data.trace_flags, "01"); +} + +TEST(OpenTelemetry, TraceParentParsingInvalid) { + const std::vector> tests = { + //"00-80e1afed08e019fc1110464cfa66635c-7a085853722dc6d2-01"; + {"00-80e1afed08e019fc1110464cfa66635c-7a085853722dc6d2-012", + "Invalid header size"}, + {"00-80e1afed08e019fc1110464cfa66635c-7a085853722dc6d2323", + "Invalid fields count"}, + {"00-80e1afed08e019fc1110464cfa66635c-7a08585123d2-01-123", + "Invalid fields count"}, + {"020-80e1afed08e019fc1110464cfa66635c-7a08585372326d2-01", + "One of the fields is not hex data"}, + {"00-80e1afed08e019fc1110464cfa66635caa-7a0322dc6d2122-01", + "One of the fields has invalid size"}, + {"00-80e1afed08e019fc1110464cfa66635c-7a085853722dc1236-1", + "One of the fields is not hex data"}, + {"00-80e1afed08e019fc1110464cfa66635c-7a085853722dc6d232-", + "One of the fields has invalid size"}, + {"---", "Invalid header size"}, + {"-------------------------------------------------------", + "Invalid fields count"}, + {"00-80e1afed08e019fc1110464cfa66635z-7a085853722dc6d2--1", + "Invalid fields count"}}; + + for (const auto& [input, expected_error] : tests) { + auto result = opentelemetry::ExtractTraceParentData(input); + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error(), expected_error); + } +} + +USERVER_NAMESPACE_END