OpenTelemetry integration: Add support for setting OTLP headers

This is necessary for using remote OTLP endpoints provided by providers
such as Honeycomb and New Relic, which require the API key to be set
through the headers mechanism.

Adds the otel_exporter_otlp_headers / OTEL_EXPORTER_OTLP_HEADERS config
setting, which follows the regular OpenTelemetry settings format:

https://github.com/open-telemetry/opentelemetry-specification/blob/v1.20.0/specification/protocol/exporter.md

config/config.go

@@ -186,7 +186,10 @@ type ServerConfig struct {
 	FilterQueryText   string `ini:"filter_query_text"`   // none/unparsable (defaults to "unparsable")
 
 	// Configuration for OpenTelemetry trace exports
-	OtelExporterOtlpEndpoint string `ini:"otel_exporter_otlp_endpoint"` // See https://github.com/open-telemetry/opentelemetry-specification/blob/v1.20.0/specification/protocol/exporter.md
+	//
+	// See https://github.com/open-telemetry/opentelemetry-specification/blob/v1.20.0/specification/protocol/exporter.md
+	OtelExporterOtlpEndpoint string `ini:"otel_exporter_otlp_endpoint"`
+	OtelExporterOtlpHeaders  string `ini:"otel_exporter_otlp_headers"`
 
 	// HTTP proxy overrides
 	HTTPProxy  string `ini:"http_proxy"`

config/read.go

@@ -263,6 +263,9 @@ func getDefaultConfig() *ServerConfig {
 	if otelExporterOtlpEndpoint := os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT"); otelExporterOtlpEndpoint != "" {
 		config.OtelExporterOtlpEndpoint = otelExporterOtlpEndpoint
 	}
+	if otelExporterOtlpHeaders := os.Getenv("OTEL_EXPORTER_OTLP_HEADERS"); otelExporterOtlpHeaders != "" {
+		config.OtelExporterOtlpHeaders = otelExporterOtlpHeaders
+	}
 	if httpProxy := os.Getenv("HTTP_PROXY"); httpProxy != "" {
 		config.HTTPProxy = httpProxy
 	}
@@ -379,19 +382,46 @@ func CreateOTelTracingProvider(ctx context.Context, conf ServerConfig) (*sdktrac
 	ctx, cancel := context.WithTimeout(ctx, time.Second)
 	defer cancel()
 
+	var headers map[string]string
+
+	if conf.OtelExporterOtlpHeaders != "" {
+		headers = make(map[string]string)
+		for _, h := range strings.Split(conf.OtelExporterOtlpHeaders, ",") {
+			nameEscaped, valueEscaped, found := strings.Cut(h, "=")
+			if !found {
+				return nil, nil, fmt.Errorf("unsupported header setting: missing '='")
+			}
+			name, err := url.QueryUnescape(nameEscaped)
+			if err != nil {
+				return nil, nil, fmt.Errorf("unsupported header setting, could not unescape header name: %s", err)
+			}
+			value, err := url.QueryUnescape(valueEscaped)
+			if err != nil {
+				return nil, nil, fmt.Errorf("unsupported header setting, could not unescape header valie: %s", err)
+			}
+			headers[strings.TrimSpace(name)] = strings.TrimSpace(value)
+		}
+	}
+
 	switch scheme {
 	case "http", "https":
 		opts := []otlptracehttp.Option{otlptracehttp.WithEndpoint(u.Host)}
 		if scheme == "http" {
 			opts = append(opts, otlptracehttp.WithInsecure())
 		}
+		if headers != nil {
+			opts = append(opts, otlptracehttp.WithHeaders(headers))
+		}
 		traceExporter, err = otlptracehttp.New(ctx, opts...)
 		if err != nil {
 			return nil, nil, fmt.Errorf("failed to create HTTP trace exporter: %w", err)
 		}
 	case "grpc":
 		// For now we always require TLS for gRPC connections
 		opts := []otlptracegrpc.Option{otlptracegrpc.WithEndpoint(u.Host)}
+		if headers != nil {
+			opts = append(opts, otlptracegrpc.WithHeaders(headers))
+		}
 		traceExporter, err = otlptracegrpc.New(ctx, opts...)
 		if err != nil {
 			return nil, nil, fmt.Errorf("failed to create HTTP trace exporter: %w", err)