From beab342e9ed3e474a18571523a767e2ca8f4d796 Mon Sep 17 00:00:00 2001 From: Aaron van Meerten Date: Wed, 8 May 2024 09:21:32 -0500 Subject: [PATCH 1/3] task: enable connect proxy for JVB to autoscaler --- .../templates/jitsi_meet_jvb.nomad.tpl | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/nomad/jitsi_packs/packs/jitsi_meet_jvb/templates/jitsi_meet_jvb.nomad.tpl b/nomad/jitsi_packs/packs/jitsi_meet_jvb/templates/jitsi_meet_jvb.nomad.tpl index 32d20b48a..76d9814f2 100644 --- a/nomad/jitsi_packs/packs/jitsi_meet_jvb/templates/jitsi_meet_jvb.nomad.tpl +++ b/nomad/jitsi_packs/packs/jitsi_meet_jvb/templates/jitsi_meet_jvb.nomad.tpl @@ -33,6 +33,7 @@ job [[ template "job_name" . ]] { count = 1 network { + mode = "bridge" # This requests a dynamic port named "http". This will # be something like "46283", but we refer to it via the # label "http". @@ -66,6 +67,17 @@ job [[ template "job_name" . ]] { port = "http" + connect { + sidecar_service { + proxy { + upstreams { + destination_name = "autoscaler" + local_bind_port = 2223 + } + } + } + } + check { name = "health" type = "http" @@ -201,7 +213,8 @@ EOF JVB_INSTANCE_ID = "${NOMAD_SHORT_ALLOC_ID}" LOCAL_ADDRESS = "${attr.unique.network.ip-address}" AUTOSCALER_SIDECAR_PORT = "6000" - AUTOSCALER_URL = "https://${meta.cloud_name}-autoscaler.jitsi.net" + AUTOSCALER_URL = "http://localhost:2223" +# AUTOSCALER_URL = "https://${meta.cloud_name}-autoscaler.jitsi.net" AUTOSCALER_SIDECAR_KEY_FILE = "/secrets/asap.key" AUTOSCALER_SIDECAR_REGION = "${meta.cloud_region}" AUTOSCALER_SIDECAR_GROUP_NAME = "${NOMAD_META_group}" From 77bc0943a901ce271c4e8fda071c9e596a935b7f Mon Sep 17 00:00:00 2001 From: Aaron van Meerten Date: Thu, 9 May 2024 10:26:57 -0500 Subject: [PATCH 2/3] include metrics for JVB envoy --- .../jitsi_meet_jvb/templates/jitsi_meet_jvb.nomad.tpl | 8 ++++++++ nomad/telegraf.hcl | 9 +++++++++ 2 files changed, 17 insertions(+) diff --git a/nomad/jitsi_packs/packs/jitsi_meet_jvb/templates/jitsi_meet_jvb.nomad.tpl b/nomad/jitsi_packs/packs/jitsi_meet_jvb/templates/jitsi_meet_jvb.nomad.tpl index 76d9814f2..ea8cf103d 100644 --- a/nomad/jitsi_packs/packs/jitsi_meet_jvb/templates/jitsi_meet_jvb.nomad.tpl +++ b/nomad/jitsi_packs/packs/jitsi_meet_jvb/templates/jitsi_meet_jvb.nomad.tpl @@ -44,6 +44,9 @@ job [[ template "job_name" . ]] { port "colibri" { to = 9090 } + port "metrics_envoy" { + to = 9102 + } } service { @@ -63,6 +66,7 @@ job [[ template "job_name" . ]] { public_ip = "${meta.public_ip}" group = "${NOMAD_META_group}" jvb_pool_mode = "[[ or (env "CONFIG_jvb_pool_mode") "shard" ]]" + metrics_port_envoy = "${NOMAD_HOST_PORT_metrics_envoy}" } port = "http" @@ -70,6 +74,10 @@ job [[ template "job_name" . ]] { connect { sidecar_service { proxy { + config { + # Expose metrics for prometheus (envoy) + envoy_prometheus_bind_addr = "0.0.0.0:9102" + } upstreams { destination_name = "autoscaler" local_bind_port = 2223 diff --git a/nomad/telegraf.hcl b/nomad/telegraf.hcl index fe793a2ae..e0aff5b65 100644 --- a/nomad/telegraf.hcl +++ b/nomad/telegraf.hcl @@ -328,6 +328,15 @@ EOF host = "{{"{{"}}.Node}}" shard-role = "docker-dhmirror" role = "docker-dhmirror" + [[inputs.prometheus.consul.query]] + name = "jvb-sidecar-proxy" + tag = "ip-{{ env "attr.unique.network.ip-address" }}" + url = 'http://{{"{{"}}if ne .ServiceAddress ""}}{{"{{"}}.ServiceAddress}}{{"{{"}}else}}{{"{{"}}.Address}}{{"{{"}}end}}:{{"{{"}}with .ServiceMeta.metrics_port_envoy}}{{"{{"}}.}}{{"{{"}}else}}{{"{{"}}.ServicePort}}{{"{{"}}end}}/metrics' + [inputs.prometheus.consul.query.tags] + host = "{{"{{"}}.Node}}" + shard-role = "jvb-envoy" + role = "jvb-envoy" + [[inputs.prometheus]] namepass = [ From 4ffa19c27ef84e6b63ae1ac219e49196eb99ee7c Mon Sep 17 00:00:00 2001 From: Aaron van Meerten Date: Thu, 9 May 2024 10:36:20 -0500 Subject: [PATCH 3/3] comment out JVB envoy stats --- nomad/telegraf.hcl | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/nomad/telegraf.hcl b/nomad/telegraf.hcl index e0aff5b65..e1b034724 100644 --- a/nomad/telegraf.hcl +++ b/nomad/telegraf.hcl @@ -328,14 +328,15 @@ EOF host = "{{"{{"}}.Node}}" shard-role = "docker-dhmirror" role = "docker-dhmirror" - [[inputs.prometheus.consul.query]] - name = "jvb-sidecar-proxy" - tag = "ip-{{ env "attr.unique.network.ip-address" }}" - url = 'http://{{"{{"}}if ne .ServiceAddress ""}}{{"{{"}}.ServiceAddress}}{{"{{"}}else}}{{"{{"}}.Address}}{{"{{"}}end}}:{{"{{"}}with .ServiceMeta.metrics_port_envoy}}{{"{{"}}.}}{{"{{"}}else}}{{"{{"}}.ServicePort}}{{"{{"}}end}}/metrics' - [inputs.prometheus.consul.query.tags] - host = "{{"{{"}}.Node}}" - shard-role = "jvb-envoy" - role = "jvb-envoy" +# uncomment for consul connect envoy sidecar metrics (~1700 per sidecar) +# [[inputs.prometheus.consul.query]] +# name = "jvb-sidecar-proxy" +# tag = "ip-{{ env "attr.unique.network.ip-address" }}" +# url = 'http://{{"{{"}}if ne .ServiceAddress ""}}{{"{{"}}.ServiceAddress}}{{"{{"}}else}}{{"{{"}}.Address}}{{"{{"}}end}}:{{"{{"}}with .ServiceMeta.metrics_port_envoy}}{{"{{"}}.}}{{"{{"}}else}}{{"{{"}}.ServicePort}}{{"{{"}}end}}/metrics' +# [inputs.prometheus.consul.query.tags] +# host = "{{"{{"}}.Node}}" +# shard-role = "jvb-envoy" +# role = "jvb-envoy" [[inputs.prometheus]]