enable ipv6 on some nodes, bump images

ansible/inventories/devnet-12/group_vars/all/images.yaml

@@ -2,7 +2,7 @@ default_ethereum_client_images:
 ### Consensus layer clients
   lighthouse: sigp/lighthouse:v4.6.0
   lodestar: chainsafe/lodestar:v1.15.0-rc.0
-  nimbus: statusim/nimbus-eth2:multiarch-v24.1.2
+  nimbus: statusim/nimbus-eth2:multiarch-v24.2.0
   prysm: gcr.io/prysmaticlabs/prysm/beacon-chain:v4.2.1-rc.1
   prysm_validator: gcr.io/prysmaticlabs/prysm/validator:v4.2.1-rc.1
   teku: consensys/teku:24.1.1
@@ -12,8 +12,8 @@ default_ethereum_client_images:
   erigon: thorax/erigon:v2.57.1
   geth: ethereum/client-go:v1.13.11
   ethereumjs: ethpandaops/ethereumjs:stable-3981bca
-  nethermind: nethermindeth/nethermind:a55635d
-  reth: ghcr.io/paradigmxyz/reth:v0.1.0-alpha.15
+  nethermind: nethermindeth/nethermind:1.25.4-dc1db75
+  reth: ethpandaops/reth:main-ce46419
 
 
 default_tooling_images:

ansible/inventories/devnet-12/host_vars/lighthouse-geth-1.yaml

@@ -1,3 +1,18 @@
+geth_container_command:
+  - --datadir=/data
+  - --port={{ geth_ports_p2p }}
+  - --http
+  - --http.addr=0.0.0.0
+  - --http.port={{ geth_ports_http_rpc }}
+  - --authrpc.addr=0.0.0.0
+  - --authrpc.port={{ geth_ports_engine }}
+  - --authrpc.vhosts=*
+  - --authrpc.jwtsecret=/execution-auth.jwt
+  - --nat=extip:{{ geth_announced_ip }}
+  - --metrics
+  - --metrics.port={{ geth_ports_metrics }}
+  - --metrics.addr=0.0.0.0
+
 geth_container_command_extra_args:
   - --http.api=eth,net,web3,debug,admin,txpool
   - --http.vhosts=*
@@ -7,6 +22,29 @@ geth_container_command_extra_args:
   - --ethstats={{ inventory_hostname }}:{{ ethstats_secret }}@{{ ethstats_url }}
   - --metrics.expensive
 
+lighthouse_container_command:
+  - lighthouse
+  - beacon_node
+  - --datadir=/data
+  - --disable-upnp
+  - --disable-enr-auto-update
+  - --enr-address={{ lighthouse_announced_ip }}
+  - --enr-tcp-port={{ lighthouse_ports_p2p_tcp }}
+  - --enr-udp-port={{ lighthouse_ports_p2p_udp }}
+  - --listen-address=0.0.0.0
+  - --listen-address=::0
+  - --port={{ lighthouse_ports_p2p_tcp }}
+  - --discovery-port={{ lighthouse_ports_p2p_udp }}
+  - --http
+  - --http-address=0.0.0.0
+  - --http-port={{ lighthouse_ports_http_beacon }}
+  - --execution-jwt=/execution-auth.jwt
+  - --execution-endpoint={{ lighthouse_execution_engine_endpoint }}
+  - --metrics
+  - --metrics-address=0.0.0.0
+  - --metrics-allow-origin=*
+  - --metrics-port={{ lighthouse_ports_metrics }}
+
 lighthouse_container_command_extra_args:
   - --testnet-dir=/network-config
   - --boot-nodes={{ ethereum_cl_bootnodes | join(',') }}
@@ -18,3 +56,17 @@ lighthouse_container_command_extra_args:
   - --slasher-history-length=256
   - --slasher-max-db-size=16
   - --checkpoint-sync-url={{ checkpoint_sync_url }}
+  - --port6={{ lighthouse_ipv6_port }}
+  - --enr-udp6-port={{ lighthouse_ipv6_port }}
+  - --enr-tcp6-port={{ lighthouse_ipv6_port }}
+ethereum_node_docker_network_name: shared6
+lighthouse_ports_quic: 9001
+lighthouse_ipv6_port: 9002
+lighthouse_container_ports:
+  - "127.0.0.1:{{ lighthouse_ports_http_beacon }}:{{ lighthouse_ports_http_beacon }}"
+  - "{{ lighthouse_ports_p2p_tcp }}:{{ lighthouse_ports_p2p_tcp }}"
+  - "{{ lighthouse_ports_p2p_udp }}:{{ lighthouse_ports_p2p_udp }}/udp"
+  - "{{ lighthouse_ports_quic }}:{{ lighthouse_ports_quic }}"
+  - "{{ lighthouse_ports_quic }}:{{ lighthouse_ports_quic }}/udp"
+  - "[::]:{{ lighthouse_ipv6_port }}:{{ lighthouse_ipv6_port }}"
+  - "[::]:{{ lighthouse_ipv6_port }}:{{ lighthouse_ipv6_port }}/udp"

ansible/inventories/devnet-12/host_vars/lodestar-geth-1.yaml

@@ -0,0 +1,42 @@
+lodestar_container_command:
+  - beacon
+  - --dataDir=/data
+  - --discv5
+  - --listenAddress=0.0.0.0
+  - --port={{ lodestar_ports_p2p_tcp }}
+  - --enr.ip={{ lodestar_announced_ip }}
+  - --enr.tcp={{ lodestar_ports_p2p_tcp }}
+  - --enr.udp={{ lodestar_ports_p2p_udp }}
+  - --rest
+  - --rest.address=0.0.0.0
+  - --rest.port={{ lodestar_ports_http_beacon }}
+  - --jwt-secret=/execution-auth.jwt
+  - --execution.urls={{ lodestar_execution_engine_endpoint }}
+  - --metrics
+  - --metrics.address=0.0.0.0
+  - --metrics.port={{ lodestar_ports_metrics }}
+
+lodestar_container_command_extra_args:
+  - --paramsFile=/network-config/config.yaml
+  - --genesisStateFile=/network-config/genesis.ssz
+  - --bootnodes={{ ethereum_cl_bootnodes | join(',') }}
+  - --rest.namespace="*"
+  - --network.connectToDiscv5Bootnodes
+  - --nat=true
+  - --listenAddress6=::0
+  - --port6={{ lodestar_ports_p2p_udp +1}}
+  - --discoveryPort6={{ lodestar_ports_p2p_udp +1 }}
+  - --enr.tcp6={{ lodestar_ports_p2p_tcp +1 }}
+  - --enr.udp6={{ lodestar_ports_p2p_udp +1 }}
+  - --checkpointSyncUrl=https://{{ secret_nginx_shared_basic_auth.name }}:{{ secret_nginx_shared_basic_auth.password }}@bn.lodestar-nethermind-1.dencun-devnet-12.ethpandaops.io
+
+ethereum_node_docker_network_name: shared7
+#docker network create --ipv6 --subnet 2a03:b0c0:3:d0::1a7f:9000/124 shared7
+lodestar_container_ports:
+  - "127.0.0.1:{{ lodestar_ports_http_beacon }}:{{ lodestar_ports_http_beacon }}"
+  - "{{ lodestar_ports_p2p_tcp }}:{{ lodestar_ports_p2p_tcp }}"
+  - "{{ lodestar_ports_p2p_udp }}:{{ lodestar_ports_p2p_udp }}/udp"
+  - "[::]:{{ lodestar_ports_p2p_tcp+1 }}:{{ lodestar_ports_p2p_tcp+1 }}"
+  - "[::]:{{ lodestar_ports_p2p_udp+1 }}:{{ lodestar_ports_p2p_udp+1 }}/udp"
+lodestar_checkpoint_sync_enabled: true
+checkpoint_sync_enabled: true

ansible/inventories/devnet-12/host_vars/nimbus-erigon-1-arm.yaml

@@ -1,3 +1,3 @@
 default_ethereum_client_images:
-  nimbus: statusim/nimbus-eth2:multiarch-v24.1.2
+  nimbus: statusim/nimbus-eth2:multiarch-v24.2.0
   erigon: thorax/erigon:2.57.1-arm64

ansible/inventories/devnet-12/inventory.ini

@@ -3,93 +3,89 @@ localhost
 [all:vars]
 ethereum_network_name=dencun-devnet-12
 
-[vouch]
-vouch validator_start=2705 validator_end=2855
-prysm-geth-1 validator_start=2700 validator_end=2705
-
 [bootnode]
-bootnode-1 ansible_host=64.226.104.183 cloud=digitalocean cloud_region=fra1
+bootnode-1 ansible_host=64.226.104.183 ipv6= cloud=digitalocean cloud_region=fra1
 
 [lighthouse_besu]
-lighthouse-besu-1 ansible_host=134.209.145.195 cloud=digitalocean cloud_region=blr1 validator_start=200 validator_end=225
+lighthouse-besu-1 ansible_host=134.209.145.195 ipv6= cloud=digitalocean cloud_region=blr1 validator_start=200 validator_end=225
 
 [lighthouse_erigon]
-lighthouse-erigon-1 ansible_host=137.184.11.255 cloud=digitalocean cloud_region=sfo3 validator_start=225 validator_end=250
+lighthouse-erigon-1 ansible_host=137.184.11.255 ipv6= cloud=digitalocean cloud_region=sfo3 validator_start=225 validator_end=250
 
 [lighthouse_geth]
-lighthouse-geth-1 ansible_host=164.90.222.95 cloud=digitalocean cloud_region=fra1 validator_start=0 validator_end=100
+lighthouse-geth-1 ansible_host=64.227.126.157 ipv6=2a03:b0c0:3:d0::1432:5001 cloud=digitalocean cloud_region=fra1 validator_start=0 validator_end=100
 
 [lighthouse_nethermind]
-lighthouse-nethermind-1 ansible_host=206.189.191.180 cloud=digitalocean cloud_region=nyc1 validator_start=100 validator_end=200
+lighthouse-nethermind-1 ansible_host=206.189.191.180 ipv6= cloud=digitalocean cloud_region=nyc1 validator_start=100 validator_end=200
 
 [lighthouse_reth]
-lighthouse-reth-1 ansible_host=170.64.200.223 cloud=digitalocean cloud_region=syd1 validator_start=1290 validator_end=1450
+lighthouse-reth-1 ansible_host=170.64.200.223 ipv6= cloud=digitalocean cloud_region=syd1 validator_start=1290 validator_end=1450
 
 [lodestar_besu]
-lodestar-besu-1 ansible_host=137.184.11.204 cloud=digitalocean cloud_region=sfo3 validator_start=720 validator_end=745
+lodestar-besu-1 ansible_host=137.184.11.204 ipv6= cloud=digitalocean cloud_region=sfo3 validator_start=720 validator_end=745
 
 [lodestar_erigon]
-lodestar-erigon-1 ansible_host=170.64.162.71 cloud=digitalocean cloud_region=syd1 validator_start=745 validator_end=770
+lodestar-erigon-1 ansible_host=170.64.162.71 ipv6= cloud=digitalocean cloud_region=syd1 validator_start=745 validator_end=770
 
 [lodestar_ethereumjs]
-lodestar-ethereumjs-1 ansible_host=104.248.116.60 cloud=digitalocean cloud_region=nyc1 validator_start=770 validator_end=780
+lodestar-ethereumjs-1 ansible_host=104.248.116.60 ipv6= cloud=digitalocean cloud_region=nyc1 validator_start=770 validator_end=780
 
 [lodestar_geth]
-lodestar-geth-1 ansible_host=68.183.214.202 cloud=digitalocean cloud_region=fra1 validator_start=510 validator_end=620
+lodestar-geth-1 ansible_host=139.59.128.222 ipv6=2a03:b0c0:3:d0::1a7f:9001 cloud=digitalocean cloud_region=fra1 validator_start=510 validator_end=620
 
 [lodestar_nethermind]
-lodestar-nethermind-1 ansible_host=134.209.145.199 cloud=digitalocean cloud_region=blr1 validator_start=620 validator_end=720
+lodestar-nethermind-1 ansible_host=134.209.145.199 ipv6= cloud=digitalocean cloud_region=blr1 validator_start=620 validator_end=720
 
 [lodestar_reth]
-lodestar-reth-1 ansible_host=159.65.122.255 cloud=digitalocean cloud_region=fra1 validator_start=1450 validator_end=1600
+lodestar-reth-1 ansible_host=159.65.122.255 ipv6= cloud=digitalocean cloud_region=fra1 validator_start=1450 validator_end=1600
 
 [mev_relay]
-mev-relay-1 ansible_host=159.223.24.145 cloud=digitalocean cloud_region=fra1
+mev-relay-1 ansible_host=159.223.24.145 ipv6= cloud=digitalocean cloud_region=fra1
 
 [nimbus_besu]
-nimbus-besu-1 ansible_host=68.183.85.180 cloud=digitalocean cloud_region=blr1 validator_start=1240 validator_end=1265
+nimbus-besu-1 ansible_host=68.183.85.180 ipv6= cloud=digitalocean cloud_region=blr1 validator_start=1240 validator_end=1265
 
 [nimbus_erigon]
-nimbus-erigon-1 ansible_host=146.190.161.212 cloud=digitalocean cloud_region=sfo3 validator_start=1265 validator_end=1290
+nimbus-erigon-1 ansible_host=146.190.161.212 ipv6= cloud=digitalocean cloud_region=sfo3 validator_start=1265 validator_end=1290
 
 [nimbus_geth]
-nimbus-geth-1 ansible_host=64.227.24.82 cloud=digitalocean cloud_region=nyc1 validator_start=1030 validator_end=1140
+nimbus-geth-1 ansible_host=64.227.24.82 ipv6= cloud=digitalocean cloud_region=nyc1 validator_start=1030 validator_end=1140
 
 [nimbus_nethermind]
-nimbus-nethermind-1 ansible_host=164.90.214.14 cloud=digitalocean cloud_region=fra1 validator_start=1140 validator_end=1240
+nimbus-nethermind-1 ansible_host=164.90.214.14 ipv6= cloud=digitalocean cloud_region=fra1 validator_start=1140 validator_end=1240
 
 [nimbus_reth]
-nimbus-reth-1 ansible_host=137.184.45.27 cloud=digitalocean cloud_region=sfo3 validator_start=1700 validator_end=1800
+nimbus-reth-1 ansible_host=137.184.45.27 ipv6= cloud=digitalocean cloud_region=sfo3 validator_start=1700 validator_end=1800
 
 [prysm_besu]
-prysm-besu-1 ansible_host=167.172.180.72 cloud=digitalocean cloud_region=fra1 validator_start=3050 validator_end=3150
+prysm-besu-1 ansible_host=167.172.180.72 ipv6= cloud=digitalocean cloud_region=fra1 validator_start=3050 validator_end=3150
 
 [prysm_erigon]
-prysm-erigon-1 ansible_host=157.245.101.239 cloud=digitalocean cloud_region=blr1 validator_start=3150 validator_end=3250
+prysm-erigon-1 ansible_host=157.245.101.239 ipv6= cloud=digitalocean cloud_region=blr1 validator_start=3150 validator_end=3250
 
 [prysm_geth]
-prysm-geth-1 ansible_host=170.64.220.204 cloud=digitalocean cloud_region=syd1 validator_start=2855 validator_end=2950
+prysm-geth-1 ansible_host=170.64.220.204 ipv6= cloud=digitalocean cloud_region=syd1 validator_start=2855 validator_end=2950
 
 [prysm_nethermind]
-prysm-nethermind-1 ansible_host=146.190.208.58 cloud=digitalocean cloud_region=nyc1 validator_start=2950 validator_end=3050
+prysm-nethermind-1 ansible_host=146.190.208.58 ipv6= cloud=digitalocean cloud_region=nyc1 validator_start=2950 validator_end=3050
 
 [prysm_reth]
-prysm-reth-1 ansible_host=159.89.95.105 cloud=digitalocean cloud_region=nyc1 validator_start=3250 validator_end=3350
+prysm-reth-1 ansible_host=159.89.95.105 ipv6= cloud=digitalocean cloud_region=nyc1 validator_start=3250 validator_end=3350
 
 [teku_besu]
-teku-besu-1 ansible_host=128.199.3.253 cloud=digitalocean cloud_region=sfo3 validator_start=980 validator_end=1005
+teku-besu-1 ansible_host=128.199.3.253 ipv6= cloud=digitalocean cloud_region=sfo3 validator_start=980 validator_end=1005
 
 [teku_erigon]
-teku-erigon-1 ansible_host=170.64.161.86 cloud=digitalocean cloud_region=syd1 validator_start=1005 validator_end=1030
+teku-erigon-1 ansible_host=170.64.161.86 ipv6= cloud=digitalocean cloud_region=syd1 validator_start=1005 validator_end=1030
 
 [teku_geth]
-teku-geth-1 ansible_host=164.90.218.160 cloud=digitalocean cloud_region=fra1 validator_start=780 validator_end=880
+teku-geth-1 ansible_host=164.90.218.160 ipv6= cloud=digitalocean cloud_region=fra1 validator_start=780 validator_end=880
 
 [teku_nethermind]
-teku-nethermind-1 ansible_host=134.209.145.105 cloud=digitalocean cloud_region=blr1 validator_start=880 validator_end=980
+teku-nethermind-1 ansible_host=134.209.145.105 ipv6= cloud=digitalocean cloud_region=blr1 validator_start=880 validator_end=980
 
 [teku_reth]
-teku-reth-1 ansible_host=165.232.186.105 cloud=digitalocean cloud_region=blr1 validator_start=1600 validator_end=1700
+teku-reth-1 ansible_host=165.232.186.105 ipv6= cloud=digitalocean cloud_region=blr1 validator_start=1600 validator_end=1700
 
 
 # Consensus client groups

ansible/requirements.yaml

@@ -14,6 +14,7 @@ roles:
 
 collections:
   - name: ansible.posix
+  - name: ansible.utils
   - name: community.sops
     version: 1.6.0
   - name: ethpandaops.general

terraform/devnet-12/ansible_inventory.tmpl

@@ -7,7 +7,7 @@ ethereum_network_name=${ethereum_network_name}
 [${replace(gid, "-", "_")}]
 %{ for key, host in hosts ~}
 %{ if host.group == gid ~}
-${host.hostname} ansible_host=${host.ip} cloud=${host.cloud} cloud_region=${host.region} %{ if max(host.validator_start, host.validator_end) != 0 }validator_start=${host.validator_start} validator_end=${host.validator_end}%{ endif }
+${host.hostname} ansible_host=${host.ip} ipv6=${host.ipv6} cloud=${host.cloud} cloud_region=${host.region} %{ if max(host.validator_start, host.validator_end) != 0 }validator_start=${host.validator_start} validator_end=${host.validator_end}%{ endif }
 %{ endif ~}
 %{ endfor ~}
 

terraform/devnet-12/main.tf

@@ -137,6 +137,7 @@ locals {
             tags   = "group_name:${vm_group.name},val_start:${vm_group.validator_start + (i * (vm_group.validator_end - vm_group.validator_start) / vm_group.count)},val_end:${min(vm_group.validator_start + ((i + 1) * (vm_group.validator_end - vm_group.validator_start) / vm_group.count), vm_group.validator_end)}"
             region = try(vm_group.location, local.digitalocean_default_region)
             size   = try(vm_group.size, local.digitalocean_default_size)
+            ipv6   = try(vm_group.ipv6, false)
           }
 
         }
@@ -256,12 +257,12 @@ resource "digitalocean_firewall" "main" {
   // Consensus layer p2p port
   inbound_rule {
     protocol         = "tcp"
-    port_range       = "9000-9001"
+    port_range       = "9000-9002"
     source_addresses = ["0.0.0.0/0", "::/0"]
   }
   inbound_rule {
     protocol         = "udp"
-    port_range       = "9000-9001"
+    port_range       = "9000-9002"
     source_addresses = ["0.0.0.0/0", "::/0"]
   }
 
@@ -315,6 +316,18 @@ resource "cloudflare_record" "server_record" {
   ttl     = 120
 }
 
+resource "cloudflare_record" "server_record6" {
+  for_each = {
+    for vm in local.digitalocean_vms : "${vm.id}" => vm if vm.ipv6
+  }
+  zone_id = data.cloudflare_zone.default.id
+  name    = "${each.value.name}.${var.ethereum_network}"
+  type    = "AAAA"
+  value   = digitalocean_droplet.main[each.value.id].ipv6_address
+  proxied = false
+  ttl     = 120
+}
+
 resource "cloudflare_record" "server_record_rpc" {
   for_each = {
     for vm in local.digitalocean_vms : "${vm.id}" => vm
@@ -327,6 +340,18 @@ resource "cloudflare_record" "server_record_rpc" {
   ttl     = 120
 }
 
+resource "cloudflare_record" "server_record_rpc6" {
+  for_each = {
+    for vm in local.digitalocean_vms : "${vm.id}" => vm if vm.ipv6
+  }
+  zone_id = data.cloudflare_zone.default.id
+  name    = "rpc.${each.value.name}.${var.ethereum_network}"
+  type    = "AAAA"
+  value   = digitalocean_droplet.main[each.value.id].ipv6_address
+  proxied = false
+  ttl     = 120
+}
+
 resource "cloudflare_record" "server_record_beacon" {
   for_each = {
     for vm in local.digitalocean_vms : "${vm.id}" => vm
@@ -339,6 +364,18 @@ resource "cloudflare_record" "server_record_beacon" {
   ttl     = 120
 }
 
+resource "cloudflare_record" "server_record_beacon6" {
+  for_each = {
+    for vm in local.digitalocean_vms : "${vm.id}" => vm if vm.ipv6
+  }
+  zone_id = data.cloudflare_zone.default.id
+  name    = "bn.${each.value.name}.${var.ethereum_network}"
+  type    = "AAAA"
+  value   = digitalocean_droplet.main[each.value.id].ipv6_address
+  proxied = false
+  ttl     = 120
+}
+
 ////////////////////////////////////////////////////////////////////////////////////////
 //                          GENERATED FILES AND OUTPUTS
 ////////////////////////////////////////////////////////////////////////////////////////
@@ -354,6 +391,7 @@ resource "local_file" "ansible_inventory" {
         {
           for key, server in digitalocean_droplet.main : "do.${key}" => {
             ip              = "${server.ipv4_address}"
+            ipv6            = try(server.ipv6_address, "none")
             group           = try(split(":", tolist(server.tags)[2])[1], "unknown")
             validator_start = try(split(":", tolist(server.tags)[4])[1], 0)
             validator_end   = try(split(":", tolist(server.tags)[3])[1], 0) # if the tag is not a number it will be 0 - e.g no validator keys

terraform/devnet-12/nodes.tf

@@ -28,6 +28,7 @@ variable "lighthouse_geth" {
     validator_start = 0
     validator_end   = 100
     location        = "fra1"
+    ipv6            = true
   }
 }
 
@@ -128,6 +129,7 @@ variable "lodestar_geth" {
     validator_start = 510
     validator_end   = 620
     location        = "fra1"
+    ipv6            = true
   }
 }