diff --git a/k8s/ex-prod-values.yaml b/k8s/ex-prod-values.yaml index aadc468cc1..d16c497b07 100644 --- a/k8s/ex-prod-values.yaml +++ b/k8s/ex-prod-values.yaml @@ -37,6 +37,5 @@ config: EX_TestEmailAddress: "test@exceptionless.io" EX_EnableArchive: "false" EX_Serilog__MinimumLevel__Default: "Warning" - EX_OTEL_EXPORTER_OTLP_ENDPOINT: http://apm.elastic-system.svc:8200 EX_Apm__EnableLogs: "true" EX_Apm__FullDetails: "true" diff --git a/k8s/ex-setup.ps1 b/k8s/ex-setup.ps1 index 74c584348d..834a3dc0b7 100644 --- a/k8s/ex-setup.ps1 +++ b/k8s/ex-setup.ps1 @@ -181,7 +181,7 @@ Write-Output "AZ_USERNAME=$AZ_USERNAME AZ_PASSWORD=$AZ_PASSWORD AZ_TENANT=$AZ_TE # renew service principal $SP_ID=$(az aks show --resource-group $RESOURCE_GROUP --name $CLUSTER --query servicePrincipalProfile.clientId -o tsv) -$SP_SECRET=$(az ad sp credential reset --name $SP_ID --years 3 --query password -o tsv) +$SP_SECRET=$(az ad sp credential reset --id $SP_ID --years 3 --query password -o tsv) # store secret in 1Password (Exceptionless Azure CI Service Principal) az aks update-credentials --resource-group $RESOURCE_GROUP --name $CLUSTER --reset-service-principal --service-principal $SP_ID --client-secret $SP_SECRET az aks get-credentials --resource-group $RESOURCE_GROUP --name $CLUSTER --overwrite-existing diff --git a/k8s/exceptionless/templates/api.yaml b/k8s/exceptionless/templates/api.yaml index 4d740caa26..25aef0c952 100644 --- a/k8s/exceptionless/templates/api.yaml +++ b/k8s/exceptionless/templates/api.yaml @@ -57,6 +57,25 @@ spec: - configMapRef: name: {{ template "exceptionless.fullname" . }}-config env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: K8S_POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: K8S_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: EX_OTEL_EXPORTER_OTLP_INSECURE + value: "true" + - name: EX_OTEL_EXPORTER_OTLP_ENDPOINT + value: $(HOST_IP):4317 + - name: EX_OTEL_RESOURCE_ATTRIBUTES + value: k8s.pod.ip=$(K8S_POD_IP),k8s.pod.uid=$(K8S_POD_UID) - name: RunJobsInProcess value: 'false' - name: EnableWebSockets diff --git a/k8s/exceptionless/templates/app.yaml b/k8s/exceptionless/templates/app.yaml index ead83f3e75..bdfff12bff 100644 --- a/k8s/exceptionless/templates/app.yaml +++ b/k8s/exceptionless/templates/app.yaml @@ -57,6 +57,25 @@ spec: - configMapRef: name: {{ template "exceptionless.fullname" . }}-config env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: K8S_POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: K8S_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: EX_OTEL_EXPORTER_OTLP_INSECURE + value: "true" + - name: EX_OTEL_EXPORTER_OTLP_ENDPOINT + value: $(HOST_IP):4317 + - name: EX_OTEL_RESOURCE_ATTRIBUTES + value: k8s.pod.ip=$(K8S_POD_IP),k8s.pod.uid=$(K8S_POD_UID) - name: RunJobsInProcess value: 'false' {{- if (empty .Values.storage.connectionString) }} diff --git a/k8s/exceptionless/templates/jobs.yaml b/k8s/exceptionless/templates/jobs.yaml index e305d5ae07..f73b03c22e 100644 --- a/k8s/exceptionless/templates/jobs.yaml +++ b/k8s/exceptionless/templates/jobs.yaml @@ -47,6 +47,26 @@ spec: envFrom: - configMapRef: name: {{ template "exceptionless.fullname" . }}-config + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: K8S_POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: K8S_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: EX_OTEL_EXPORTER_OTLP_INSECURE + value: "true" + - name: EX_OTEL_EXPORTER_OTLP_ENDPOINT + value: $(HOST_IP):4317 + - name: EX_OTEL_RESOURCE_ATTRIBUTES + value: k8s.pod.ip=$(K8S_POD_IP),k8s.pod.uid=$(K8S_POD_UID) --- apiVersion: apps/v1 @@ -98,6 +118,26 @@ spec: envFrom: - configMapRef: name: {{ template "exceptionless.fullname" . }}-config + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: K8S_POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: K8S_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: EX_OTEL_EXPORTER_OTLP_INSECURE + value: "true" + - name: EX_OTEL_EXPORTER_OTLP_ENDPOINT + value: $(HOST_IP):4317 + - name: EX_OTEL_RESOURCE_ATTRIBUTES + value: k8s.pod.ip=$(K8S_POD_IP),k8s.pod.uid=$(K8S_POD_UID) --- apiVersion: batch/v1 @@ -139,6 +179,26 @@ spec: envFrom: - configMapRef: name: {{ template "exceptionless.fullname" . }}-config + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: K8S_POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: K8S_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: EX_OTEL_EXPORTER_OTLP_INSECURE + value: "true" + - name: EX_OTEL_EXPORTER_OTLP_ENDPOINT + value: $(HOST_IP):4317 + - name: EX_OTEL_RESOURCE_ATTRIBUTES + value: k8s.pod.ip=$(K8S_POD_IP),k8s.pod.uid=$(K8S_POD_UID) restartPolicy: OnFailure --- @@ -191,6 +251,26 @@ spec: envFrom: - configMapRef: name: {{ template "exceptionless.fullname" . }}-config + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: K8S_POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: K8S_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: EX_OTEL_EXPORTER_OTLP_INSECURE + value: "true" + - name: EX_OTEL_EXPORTER_OTLP_ENDPOINT + value: $(HOST_IP):4317 + - name: EX_OTEL_RESOURCE_ATTRIBUTES + value: k8s.pod.ip=$(K8S_POD_IP),k8s.pod.uid=$(K8S_POD_UID) --- apiVersion: apps/v1 @@ -242,6 +322,26 @@ spec: envFrom: - configMapRef: name: {{ template "exceptionless.fullname" . }}-config + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: K8S_POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: K8S_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: EX_OTEL_EXPORTER_OTLP_INSECURE + value: "true" + - name: EX_OTEL_EXPORTER_OTLP_ENDPOINT + value: $(HOST_IP):4317 + - name: EX_OTEL_RESOURCE_ATTRIBUTES + value: k8s.pod.ip=$(K8S_POD_IP),k8s.pod.uid=$(K8S_POD_UID) --- apiVersion: batch/v1 @@ -277,6 +377,26 @@ spec: envFrom: - configMapRef: name: {{ template "exceptionless.fullname" . }}-config + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: K8S_POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: K8S_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: EX_OTEL_EXPORTER_OTLP_INSECURE + value: "true" + - name: EX_OTEL_EXPORTER_OTLP_ENDPOINT + value: $(HOST_IP):4317 + - name: EX_OTEL_RESOURCE_ATTRIBUTES + value: k8s.pod.ip=$(K8S_POD_IP),k8s.pod.uid=$(K8S_POD_UID) restartPolicy: OnFailure --- @@ -329,7 +449,26 @@ spec: envFrom: - configMapRef: name: {{ template "exceptionless.fullname" . }}-config - + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: K8S_POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: K8S_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: EX_OTEL_EXPORTER_OTLP_INSECURE + value: "true" + - name: EX_OTEL_EXPORTER_OTLP_ENDPOINT + value: $(HOST_IP):4317 + - name: EX_OTEL_RESOURCE_ATTRIBUTES + value: k8s.pod.ip=$(K8S_POD_IP),k8s.pod.uid=$(K8S_POD_UID) --- apiVersion: apps/v1 kind: Deployment @@ -380,7 +519,26 @@ spec: envFrom: - configMapRef: name: {{ template "exceptionless.fullname" . }}-config - + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: K8S_POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: K8S_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: EX_OTEL_EXPORTER_OTLP_INSECURE + value: "true" + - name: EX_OTEL_EXPORTER_OTLP_ENDPOINT + value: $(HOST_IP):4317 + - name: EX_OTEL_RESOURCE_ATTRIBUTES + value: k8s.pod.ip=$(K8S_POD_IP),k8s.pod.uid=$(K8S_POD_UID) --- apiVersion: batch/v1 kind: CronJob @@ -421,6 +579,26 @@ spec: envFrom: - configMapRef: name: {{ template "exceptionless.fullname" . }}-config + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: K8S_POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: K8S_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: EX_OTEL_EXPORTER_OTLP_INSECURE + value: "true" + - name: EX_OTEL_EXPORTER_OTLP_ENDPOINT + value: $(HOST_IP):4317 + - name: EX_OTEL_RESOURCE_ATTRIBUTES + value: k8s.pod.ip=$(K8S_POD_IP),k8s.pod.uid=$(K8S_POD_UID) restartPolicy: OnFailure --- @@ -463,6 +641,26 @@ spec: envFrom: - configMapRef: name: {{ template "exceptionless.fullname" . }}-config + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: K8S_POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: K8S_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: EX_OTEL_EXPORTER_OTLP_INSECURE + value: "true" + - name: EX_OTEL_EXPORTER_OTLP_ENDPOINT + value: $(HOST_IP):4317 + - name: EX_OTEL_RESOURCE_ATTRIBUTES + value: k8s.pod.ip=$(K8S_POD_IP),k8s.pod.uid=$(K8S_POD_UID) restartPolicy: OnFailure --- @@ -505,6 +703,26 @@ spec: envFrom: - configMapRef: name: {{ template "exceptionless.fullname" . }}-config + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: K8S_POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: K8S_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: EX_OTEL_EXPORTER_OTLP_INSECURE + value: "true" + - name: EX_OTEL_EXPORTER_OTLP_ENDPOINT + value: $(HOST_IP):4317 + - name: EX_OTEL_RESOURCE_ATTRIBUTES + value: k8s.pod.ip=$(K8S_POD_IP),k8s.pod.uid=$(K8S_POD_UID) restartPolicy: OnFailure --- @@ -557,7 +775,26 @@ spec: envFrom: - configMapRef: name: {{ template "exceptionless.fullname" . }}-config - + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: K8S_POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: K8S_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: EX_OTEL_EXPORTER_OTLP_INSECURE + value: "true" + - name: EX_OTEL_EXPORTER_OTLP_ENDPOINT + value: $(HOST_IP):4317 + - name: EX_OTEL_RESOURCE_ATTRIBUTES + value: k8s.pod.ip=$(K8S_POD_IP),k8s.pod.uid=$(K8S_POD_UID) --- apiVersion: apps/v1 kind: Deployment @@ -608,7 +845,26 @@ spec: envFrom: - configMapRef: name: {{ template "exceptionless.fullname" . }}-config - + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: K8S_POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: K8S_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: EX_OTEL_EXPORTER_OTLP_INSECURE + value: "true" + - name: EX_OTEL_EXPORTER_OTLP_ENDPOINT + value: $(HOST_IP):4317 + - name: EX_OTEL_RESOURCE_ATTRIBUTES + value: k8s.pod.ip=$(K8S_POD_IP),k8s.pod.uid=$(K8S_POD_UID) --- apiVersion: apps/v1 kind: Deployment @@ -659,6 +915,26 @@ spec: envFrom: - configMapRef: name: {{ template "exceptionless.fullname" . }}-config + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: K8S_POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: K8S_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: EX_OTEL_EXPORTER_OTLP_INSECURE + value: "true" + - name: EX_OTEL_EXPORTER_OTLP_ENDPOINT + value: $(HOST_IP):4317 + - name: EX_OTEL_RESOURCE_ATTRIBUTES + value: k8s.pod.ip=$(K8S_POD_IP),k8s.pod.uid=$(K8S_POD_UID) {{- if (empty .Values.storage.connectionString) }} volumeMounts: - mountPath: "/app/storage" @@ -719,3 +995,23 @@ spec: envFrom: - configMapRef: name: {{ template "exceptionless.fullname" . }}-config + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: K8S_POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: K8S_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: EX_OTEL_EXPORTER_OTLP_INSECURE + value: "true" + - name: EX_OTEL_EXPORTER_OTLP_ENDPOINT + value: $(HOST_IP):4317 + - name: EX_OTEL_RESOURCE_ATTRIBUTES + value: k8s.pod.ip=$(K8S_POD_IP),k8s.pod.uid=$(K8S_POD_UID) diff --git a/src/Exceptionless.Web/ApmExtensions.cs b/src/Exceptionless.Web/ApmExtensions.cs index 5b7b6f3ad8..5712bd8d25 100644 --- a/src/Exceptionless.Web/ApmExtensions.cs +++ b/src/Exceptionless.Web/ApmExtensions.cs @@ -32,7 +32,6 @@ public static IHostBuilder AddApm(this IHostBuilder builder, ApmConfig config) services.AddOpenTelemetry().WithTracing(b => { - b.AddProcessor(); b.SetResourceBuilder(resourceBuilder); b.AddAspNetCoreInstrumentation(o => @@ -151,7 +150,7 @@ public class ApmConfig public ApmConfig(IConfigurationRoot config, string processName, string? serviceVersion, bool enableRedis) { - EnableExporter = !string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("OTEL_EXPORTER_OTLP_ENDPOINT")); + EnableExporter = !string.IsNullOrWhiteSpace(config.GetValue("OTEL_EXPORTER_OTLP_ENDPOINT")); _apmConfig = config.GetSection("Apm"); processName = processName.StartsWith('-') ? processName : "-" + processName; @@ -259,131 +258,3 @@ public class FilteredOtlpExporterOptions : OtlpExporterOptions { public Func? Filter { get; set; } } - -public class ElasticCompatibilityProcessor : BaseProcessor -{ - private readonly AsyncLocal _currentTransactionId = new(); - public const string TransactionIdTagName = "transaction.id"; - - public override void OnEnd(Activity activity) - { - if (activity.Parent == null) - _currentTransactionId.Value = activity.SpanId; - - if (_currentTransactionId.Value.HasValue) - activity.SetTag(TransactionIdTagName, _currentTransactionId.Value.Value.ToString()); - - if (activity.Kind == ActivityKind.Server) - { - string? httpScheme = null; - string? httpTarget = null; - string? urlScheme = null; - string? urlPath = null; - string? urlQuery = null; - string? netHostName = null; - int? netHostPort = null; - string? serverAddress = null; - int? serverPort = null; - - foreach (var tag in activity.TagObjects) - { - if (tag.Key == TraceSemanticConventions.HttpScheme) - httpScheme = ProcessStringAttribute(tag); - - if (tag.Key == TraceSemanticConventions.HttpTarget) - httpTarget = ProcessStringAttribute(tag); - - if (tag.Key == TraceSemanticConventions.UrlScheme) - urlScheme = ProcessStringAttribute(tag); - - if (tag.Key == TraceSemanticConventions.UrlPath) - urlPath = ProcessStringAttribute(tag); - - if (tag.Key == TraceSemanticConventions.UrlQuery) - urlQuery = ProcessStringAttribute(tag); - - if (tag.Key == TraceSemanticConventions.NetHostName) - netHostName = ProcessStringAttribute(tag); - - if (tag.Key == TraceSemanticConventions.ServerAddress) - serverAddress = ProcessStringAttribute(tag); - - if (tag.Key == TraceSemanticConventions.NetHostPort) - netHostPort = ProcessIntAttribute(tag); - - if (tag.Key == TraceSemanticConventions.ServerPort) - serverPort = ProcessIntAttribute(tag); - } - - // Set the older semantic convention attributes - if (httpScheme is null && urlScheme is not null) - SetStringAttribute(TraceSemanticConventions.HttpScheme, urlScheme); - - if (httpTarget is null && urlPath is not null) - { - var target = urlPath; - - if (urlQuery is not null) - target += $"?{urlQuery}"; - - SetStringAttribute(TraceSemanticConventions.HttpTarget, target); - } - - if (netHostName is null && serverAddress is not null) - SetStringAttribute(TraceSemanticConventions.NetHostName, serverAddress); - - if (netHostPort is null && serverPort is not null) - SetIntAttribute(TraceSemanticConventions.NetHostPort, serverPort.Value); - } - - string? ProcessStringAttribute(KeyValuePair tag) - { - if (tag.Value is string value) - { - return value; - } - - return null; - } - - int? ProcessIntAttribute(KeyValuePair tag) - { - if (tag.Value is int value) - { - return value; - } - - return null; - } - - void SetStringAttribute(string attributeName, string value) - { - activity.SetTag(attributeName, value); - } - - void SetIntAttribute(string attributeName, int value) - { - activity.SetTag(attributeName, value); - } - } -} - -internal static class TraceSemanticConventions -{ - // HTTP - public const string HttpScheme = "http.scheme"; - public const string HttpTarget = "http.target"; - - // NET - public const string NetHostName = "net.host.name"; - public const string NetHostPort = "net.host.port"; - - // SERVER - public const string ServerAddress = "server.address"; - public const string ServerPort = "server.port"; - - // URL - public const string UrlPath = "url.path"; - public const string UrlQuery = "url.query"; - public const string UrlScheme = "url.scheme"; -}