diff --git a/.gitignore b/.gitignore index b71f162fd7..bb27de2141 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,7 @@ *.iml bin *.o +*.sln tmp terraform.tfvars terraform.tfstate* diff --git a/build/terraform/e2e/gke-autopilot/module.tf b/build/terraform/e2e/gke-autopilot/module.tf index da24c3cd90..4aee19b828 100644 --- a/build/terraform/e2e/gke-autopilot/module.tf +++ b/build/terraform/e2e/gke-autopilot/module.tf @@ -50,4 +50,4 @@ module "gke_cluster" { } udpFirewall = false // firewall is created at the project module level -} \ No newline at end of file +} diff --git a/build/terraform/e2e/gke-standard/module.tf b/build/terraform/e2e/gke-standard/module.tf index 906a1a71df..930b1d9c37 100644 --- a/build/terraform/e2e/gke-standard/module.tf +++ b/build/terraform/e2e/gke-standard/module.tf @@ -64,4 +64,4 @@ module "gke_cluster" { } udpFirewall = false // firewall is created at the project module level -} \ No newline at end of file +} diff --git a/build/terraform/e2e/state-bucket/main.tf b/build/terraform/e2e/state-bucket/main.tf index 74d4171321..18fbe2065c 100644 --- a/build/terraform/e2e/state-bucket/main.tf +++ b/build/terraform/e2e/state-bucket/main.tf @@ -22,7 +22,7 @@ // tfstate, delete your local .terraform and .tfstate files. You may need to run // `sudo chown -R yourusername .` to be able to delete them. Then navigate to this directory and run // `terraform init`. Pull in the tfstate file from gcloud with -// `terraform import google_storage_bucket.default agones-images-e2e-infra-bucket-tfstate`. +// `terraform import google_storage_bucket.default ""-e2e-infra-bucket-tfstate`. // # GCS bucket for holding the Terraform state of the e2e Terraform config. diff --git a/build/terraform/upgrade/gke-autopilot/module.tf b/build/terraform/upgrade/gke-autopilot/module.tf new file mode 100644 index 0000000000..95f82f1fa7 --- /dev/null +++ b/build/terraform/upgrade/gke-autopilot/module.tf @@ -0,0 +1,53 @@ +// Copyright 2024 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + +// Run: +// terraform apply -var project="" + +terraform { + required_version = ">= 1.0.0" + required_providers { + google = { + source = "hashicorp/google" + version = "~> 4.25.0" + } + helm = { + source = "hashicorp/helm" + version = "~> 2.3" + } + } +} + +variable "project" {} +variable "kubernetesVersion" {} +variable "location" {} +variable "releaseChannel" {} + +module "gke_cluster" { + source = "../../../../install/terraform/modules/gke-autopilot" + + cluster = { + "name" = format("gke-autopilot-upgrade-test-cluster-%s", replace(var.kubernetesVersion, ".", "-")) + "project" = var.project + "location" = var.location + "releaseChannel" = var.releaseChannel + "kubernetesVersion" = var.kubernetesVersion + "deletionProtection" = false + "maintenanceExclusionStartTime" = timestamp() + "maintenanceExclusionEndTime" = timeadd(timestamp(), "2640h") # 110 days + } + + udpFirewall = false // firewall is created at the project module level +} diff --git a/build/terraform/upgrade/gke-standard/module.tf b/build/terraform/upgrade/gke-standard/module.tf new file mode 100644 index 0000000000..78e457ab2f --- /dev/null +++ b/build/terraform/upgrade/gke-standard/module.tf @@ -0,0 +1,67 @@ +// Copyright 2024 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + +// Run: +// terraform apply -var project="" + +terraform { + required_version = ">= 1.0.0" + required_providers { + google = { + source = "hashicorp/google" + version = "~> 4.25.0" + } + helm = { + source = "hashicorp/helm" + version = "~> 2.3" + } + } +} + +variable "project" {} +variable "kubernetesVersion" {} +variable "location" {} +variable "releaseChannel" {} + +variable "machineType" { + default = "e2-standard-4" +} + +variable "initialNodeCount" { + default = 4 +} + +variable "overrideName" { + default = "" +} + +module "gke_cluster" { + source = "../../../../install/terraform/modules/gke" + + cluster = { + "name" = var.overrideName != "" ? var.overrideName : format("standard-upgrade-test-cluster-%s", replace(var.kubernetesVersion, ".", "-")) + "location" = var.location + "releaseChannel" = var.releaseChannel + "machineType" = var.machineType + "initialNodeCount" = var.initialNodeCount + "enableImageStreaming" = true + "project" = var.project + "kubernetesVersion" = var.kubernetesVersion + "maintenanceExclusionStartTime" = timestamp() + "maintenanceExclusionEndTime" = timeadd(timestamp(), "2640h") # 110 days + } + + udpFirewall = false // firewall is created at the project module level +} diff --git a/build/terraform/upgrade/module.tf b/build/terraform/upgrade/module.tf new file mode 100644 index 0000000000..6fb0bc4232 --- /dev/null +++ b/build/terraform/upgrade/module.tf @@ -0,0 +1,109 @@ +// Copyright 2024 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + +// Run: +// terraform init -backend-config="bucket=-upgrade-infra-bucket-tfstate" -backend-config="prefix=terraform/state" +// terraform apply -var project="" + +terraform { + required_version = ">= 1.0.0" + required_providers { + google = { + source = "hashicorp/google" + version = "~> 4.25.0" + } + helm = { + source = "hashicorp/helm" + version = "~> 2.3" + } + } + backend "gcs" { + } +} + +variable "project" {} +variable "kubernetes_versions" { + description = "Create upgrade test clusters with these k8s versions in these regions" + type = map(list(string)) + default = { + "1.28" = ["us-west1", "RAPID"] + "1.29" = ["europe-west1", "RAPID"] + "1.30" = ["asia-east1", "RAPID"] + // "1.31" = ["us-east1", "RAPID"] + // + // Before merge: When adding Kubernetes version 1.{N}, first uncomment the line above, extending + // the infrastructure to 4 versions temporarily. Come back to these instructions after the + // update PR merges. + // + // After merge: After the Kubernetes update PR merges, and all active PRs are updated: + // + // * Move the 1.{N-3} line to the bottom and comment it out + // * Change the (commented out) 1.{N-3} to 1.{N+1} + // * You should now have 3 versions uncommented (versions 1.{N-2} .. 1.{N}), + // and 1.{N+1} commented out for the next update. The new, commented out 1.{N+1} + // should be using the region of the previous 1.{N-3} - this region will become + // unused. + // + // Rationale: We cycle the regions us-east1 -> us-west1 -> europe-west1 -> asia-east1 -> us-east1 + // as versions are added, using 4 regions so that the PR adding 1.{N} is in a unique region to + // 1.{N-3} .. 1.{N-1}, meaning versions never need to share a region in CI. + } +} + +module "gke_standard_cluster" { + for_each = var.kubernetes_versions + source = "./gke-standard" + project = var.project + kubernetesVersion = each.key + location = each.value[0] + releaseChannel = each.value[1] +} + +module "gke_autopilot_cluster" { + for_each = var.kubernetes_versions + source = "./gke-autopilot" + project = var.project + kubernetesVersion = each.key + location = each.value[0] + releaseChannel = each.value[1] +} + +resource "google_compute_firewall" "udp" { + name = "gke-game-server-firewall" + project = var.project + network = "default" + + allow { + protocol = "udp" + ports = ["7000-8000"] + } + + target_tags = ["game-server"] + source_ranges = ["0.0.0.0/0"] +} + +resource "google_compute_firewall" "tcp" { + name = "gke-game-server-firewall-tcp" + project = var.project + network = "default" + + allow { + protocol = "tcp" + ports = ["7000-8000"] + } + + target_tags = ["game-server"] + source_ranges = ["0.0.0.0/0"] +} diff --git a/build/terraform/upgrade/state-bucket/main.tf b/build/terraform/upgrade/state-bucket/main.tf new file mode 100644 index 0000000000..0234e30418 --- /dev/null +++ b/build/terraform/upgrade/state-bucket/main.tf @@ -0,0 +1,43 @@ +// Copyright 2024 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + +// Run: +// terraform apply -var project="" + +// GCS bucket for holding the Terraform state of the upgrade test Terraform config. + +terraform { + required_version = ">= 1.0.0" + required_providers { + google = { + source = "hashicorp/google" + version = "~> 4.25.0" + } + } +} + +variable "project" {} + +resource "google_storage_bucket" "default" { + project = var.project + name = "${var.project}-upgrade-infra-bucket-tfstate" + force_destroy = false + uniform_bucket_level_access = true + location = "US" + storage_class = "STANDARD" + versioning { + enabled = true + } +} diff --git a/cmd/allocator/main.go b/cmd/allocator/main.go index 4c6caea7df..b73f654fc5 100644 --- a/cmd/allocator/main.go +++ b/cmd/allocator/main.go @@ -26,6 +26,16 @@ import ( "sync" "time" + "agones.dev/agones/pkg" + "agones.dev/agones/pkg/allocation/converters" + pb "agones.dev/agones/pkg/allocation/go" + allocationv1 "agones.dev/agones/pkg/apis/allocation/v1" + "agones.dev/agones/pkg/client/clientset/versioned" + "agones.dev/agones/pkg/client/informers/externalversions" + "agones.dev/agones/pkg/gameserverallocations" + "agones.dev/agones/pkg/gameservers" + "agones.dev/agones/pkg/metrics" + "agones.dev/agones/pkg/util/fswatch" "github.com/heptiolabs/healthcheck" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -45,15 +55,6 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" - "agones.dev/agones/pkg" - "agones.dev/agones/pkg/allocation/converters" - pb "agones.dev/agones/pkg/allocation/go" - allocationv1 "agones.dev/agones/pkg/apis/allocation/v1" - "agones.dev/agones/pkg/client/clientset/versioned" - "agones.dev/agones/pkg/client/informers/externalversions" - "agones.dev/agones/pkg/gameserverallocations" - "agones.dev/agones/pkg/gameservers" - "agones.dev/agones/pkg/util/fswatch" "agones.dev/agones/pkg/util/httpserver" "agones.dev/agones/pkg/util/runtime" "agones.dev/agones/pkg/util/signals" @@ -218,9 +219,19 @@ func main() { logger.WithField("grpc-port", conf.GRPCPort).WithField("http-port", conf.HTTPPort).Fatal("Must specify a valid gRPC port or an HTTP port for the allocator service") } healthserver := &httpserver.Server{Logger: logger} - health, closer := setupMetricsRecorder(conf, healthserver) + var health healthcheck.Handler + + metricsConf := metrics.Config{ + Stackdriver: conf.Stackdriver, + PrometheusMetrics: conf.PrometheusMetrics, + GCPProjectID: conf.GCPProjectID, + StackdriverLabels: conf.StackdriverLabels, + } + health, closer := metrics.SetupMetrics(metricsConf, healthserver) defer closer() + metrics.SetReportingPeriod(conf.PrometheusMetrics, conf.Stackdriver) + kubeClient, agonesClient, err := getClients(conf) if err != nil { logger.WithError(err).Fatal("could not create clients") diff --git a/cmd/allocator/metrics.go b/cmd/allocator/metrics.go index 7ce8ed2110..95421627fd 100644 --- a/cmd/allocator/metrics.go +++ b/cmd/allocator/metrics.go @@ -14,10 +14,6 @@ package main import ( - "agones.dev/agones/pkg/metrics" - "agones.dev/agones/pkg/util/httpserver" - "github.com/heptiolabs/healthcheck" - prom "github.com/prometheus/client_golang/prometheus" "go.opencensus.io/plugin/ocgrpc" "go.opencensus.io/stats/view" ) @@ -31,32 +27,3 @@ func registerMetricViews() { logger.WithError(err).Error("could not register view") } } - -func setupMetricsRecorder(conf config, healthserver *httpserver.Server) (health healthcheck.Handler, closer func()) { - health = healthcheck.NewHandler() - closer = func() {} - - // Stackdriver metrics - if conf.Stackdriver { - sd, err := metrics.RegisterStackdriverExporter(conf.GCPProjectID, conf.StackdriverLabels) - if err != nil { - logger.WithError(err).Fatal("Could not register stackdriver exporter") - } - // It is imperative to invoke flush before your main function exits - closer = func() { sd.Flush() } - } - - // Prometheus metrics - if conf.PrometheusMetrics { - registry := prom.NewRegistry() - metricHandler, err := metrics.RegisterPrometheusExporter(registry) - if err != nil { - logger.WithError(err).Fatal("Could not register prometheus exporter") - } - healthserver.Handle("/metrics", metricHandler) - health = healthcheck.NewMetricsHandler(registry, "agones") - } - - metrics.SetReportingPeriod(conf.PrometheusMetrics, conf.Stackdriver) - return -} diff --git a/cmd/controller/main.go b/cmd/controller/main.go index 360aa8294c..41171d1902 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -25,12 +25,23 @@ import ( "strings" "time" + "agones.dev/agones/pkg" agonesv1 "agones.dev/agones/pkg/apis/agones/v1" + "agones.dev/agones/pkg/client/clientset/versioned" + "agones.dev/agones/pkg/client/informers/externalversions" + "agones.dev/agones/pkg/cloudproduct" + "agones.dev/agones/pkg/fleetautoscalers" + "agones.dev/agones/pkg/fleets" + "agones.dev/agones/pkg/gameservers" + "agones.dev/agones/pkg/gameserversets" + "agones.dev/agones/pkg/metrics" "agones.dev/agones/pkg/portallocator" + "agones.dev/agones/pkg/util/httpserver" + "agones.dev/agones/pkg/util/runtime" + "agones.dev/agones/pkg/util/signals" "github.com/google/uuid" "github.com/heptiolabs/healthcheck" "github.com/pkg/errors" - prom "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" "github.com/spf13/pflag" "github.com/spf13/viper" @@ -43,19 +54,6 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/leaderelection" "k8s.io/client-go/tools/leaderelection/resourcelock" - - "agones.dev/agones/pkg" - "agones.dev/agones/pkg/client/clientset/versioned" - "agones.dev/agones/pkg/client/informers/externalversions" - "agones.dev/agones/pkg/cloudproduct" - "agones.dev/agones/pkg/fleetautoscalers" - "agones.dev/agones/pkg/fleets" - "agones.dev/agones/pkg/gameservers" - "agones.dev/agones/pkg/gameserversets" - "agones.dev/agones/pkg/metrics" - "agones.dev/agones/pkg/util/httpserver" - "agones.dev/agones/pkg/util/runtime" - "agones.dev/agones/pkg/util/signals" ) const ( @@ -180,28 +178,15 @@ func main() { var rs []runner var health healthcheck.Handler - // Stackdriver metrics - if ctlConf.Stackdriver { - sd, err := metrics.RegisterStackdriverExporter(ctlConf.GCPProjectID, ctlConf.StackdriverLabels) - if err != nil { - logger.WithError(err).Fatal("Could not register stackdriver exporter") - } - // It is imperative to invoke flush before your main function exits - defer sd.Flush() + metricsConf := metrics.Config{ + Stackdriver: ctlConf.Stackdriver, + PrometheusMetrics: ctlConf.PrometheusMetrics, + GCPProjectID: ctlConf.GCPProjectID, + StackdriverLabels: ctlConf.StackdriverLabels, } - // Prometheus metrics - if ctlConf.PrometheusMetrics { - registry := prom.NewRegistry() - metricHandler, err := metrics.RegisterPrometheusExporter(registry) - if err != nil { - logger.WithError(err).Fatal("Could not register prometheus exporter") - } - server.Handle("/metrics", metricHandler) - health = healthcheck.NewMetricsHandler(registry, "agones") - } else { - health = healthcheck.NewHandler() - } + health, closer := metrics.SetupMetrics(metricsConf, server) + defer closer() // If we are using Prometheus only exporter we can make reporting more often, // every 1 seconds, if we are using Stackdriver we would use 60 seconds reporting period, diff --git a/cmd/controller/pprof.go b/cmd/controller/pprof.go index a79e170e1b..f42733bd96 100644 --- a/cmd/controller/pprof.go +++ b/cmd/controller/pprof.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build profile // +build profile package main diff --git a/cmd/extensions/main.go b/cmd/extensions/main.go index f6cb453536..61882de2d6 100644 --- a/cmd/extensions/main.go +++ b/cmd/extensions/main.go @@ -23,16 +23,6 @@ import ( "strings" "time" - "github.com/heptiolabs/healthcheck" - "github.com/pkg/errors" - prom "github.com/prometheus/client_golang/prometheus" - "github.com/sirupsen/logrus" - "github.com/spf13/pflag" - "github.com/spf13/viper" - "gopkg.in/natefinch/lumberjack.v2" - "k8s.io/client-go/informers" - "k8s.io/client-go/kubernetes" - "agones.dev/agones/pkg" "agones.dev/agones/pkg/client/clientset/versioned" "agones.dev/agones/pkg/client/informers/externalversions" @@ -49,6 +39,14 @@ import ( "agones.dev/agones/pkg/util/runtime" "agones.dev/agones/pkg/util/signals" "agones.dev/agones/pkg/util/webhooks" + "github.com/heptiolabs/healthcheck" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "github.com/spf13/pflag" + "github.com/spf13/viper" + "gopkg.in/natefinch/lumberjack.v2" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" ) const ( @@ -158,29 +156,15 @@ func main() { } var health healthcheck.Handler - // Stackdriver metrics - if ctlConf.Stackdriver { - sd, err := metrics.RegisterStackdriverExporter(ctlConf.GCPProjectID, ctlConf.StackdriverLabels) - if err != nil { - logger.WithError(err).Fatal("Could not register stackdriver exporter") - } - // It is imperative to invoke flush before your main function exits - defer sd.Flush() + metricsConf := metrics.Config{ + Stackdriver: ctlConf.Stackdriver, + PrometheusMetrics: ctlConf.PrometheusMetrics, + GCPProjectID: ctlConf.GCPProjectID, + StackdriverLabels: ctlConf.StackdriverLabels, } - // Prometheus metrics - if ctlConf.PrometheusMetrics { - registry := prom.NewRegistry() - metricHandler, err := metrics.RegisterPrometheusExporter(registry) - if err != nil { - logger.WithError(err).Fatal("Could not register prometheus exporter in extensions") - } - server.Handle("/metrics", metricHandler) - health = healthcheck.NewMetricsHandler(registry, "agones") - } else { - logger.Info("Not registaring prometheus metrics") - health = healthcheck.NewHandler() - } + health, closer := metrics.SetupMetrics(metricsConf, server) + defer closer() podReady = true health.AddReadinessCheck("agones-extensions", func() error { diff --git a/cmd/extensions/pprof.go b/cmd/extensions/pprof.go index 6c33f94d90..6c1bec83c8 100644 --- a/cmd/extensions/pprof.go +++ b/cmd/extensions/pprof.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build profile // +build profile package main diff --git a/pkg/metrics/exporter.go b/pkg/metrics/exporter.go index fdacaed5e5..a29cb3da53 100644 --- a/pkg/metrics/exporter.go +++ b/pkg/metrics/exporter.go @@ -19,9 +19,11 @@ import ( "os" "time" + "agones.dev/agones/pkg/util/httpserver" "cloud.google.com/go/compute/metadata" "contrib.go.opencensus.io/exporter/prometheus" "contrib.go.opencensus.io/exporter/stackdriver" + "github.com/heptiolabs/healthcheck" "github.com/pkg/errors" prom "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/collectors" @@ -29,6 +31,14 @@ import ( "google.golang.org/genproto/googleapis/api/monitoredres" ) +// Config holds configuration for metrics reporting +type Config struct { + GCPProjectID string + StackdriverLabels string + Stackdriver bool + PrometheusMetrics bool +} + // RegisterPrometheusExporter register a prometheus exporter to OpenCensus with a given prometheus metric registry. // It will automatically add go runtime and process metrics using default prometheus collectors. // The function return an http.handler that you can use to expose the prometheus endpoint. @@ -119,3 +129,33 @@ func getMonitoredResource(projectID string) (*monitoredres.MonitoredResource, er }, }, nil } + +// SetupMetrics initializes metrics reporting with the provided configuration +func SetupMetrics(conf Config, server *httpserver.Server) (healthcheck.Handler, func()) { + var health healthcheck.Handler + var closer = func() {} + + // Stackriver Metrics + if conf.Stackdriver { + sd, err := RegisterStackdriverExporter(conf.GCPProjectID, conf.StackdriverLabels) + if err != nil { + logger.WithError(err).Fatal("Could not register Stackdriver exporter") + } + closer = func() { sd.Flush() } + } + + // Prometheus Metrics + if conf.PrometheusMetrics { + registry := prom.NewRegistry() + metricHandler, err := RegisterPrometheusExporter(registry) + if err != nil { + logger.WithError(err).Fatal("Could not register Prometheus exporter") + } + server.Handle("/metrics", metricHandler) + health = healthcheck.NewMetricsHandler(registry, "agones") + } else { + health = healthcheck.NewHandler() + } + + return health, closer +} diff --git a/site/content/en/docs/Getting Started/create-gameserver.md b/site/content/en/docs/Getting Started/create-gameserver.md index bf894f1bf1..79ff772a0d 100644 --- a/site/content/en/docs/Getting Started/create-gameserver.md +++ b/site/content/en/docs/Getting Started/create-gameserver.md @@ -78,7 +78,7 @@ Kind: GameServer Metadata: Creation Timestamp: 2019-02-27T15:06:20Z Finalizers: - agones.dev + agones.dev/controller Generate Name: simple-game-server- Generation: 1 Resource Version: 30377 diff --git a/site/content/en/docs/Guides/metrics.md b/site/content/en/docs/Guides/metrics.md index c1f9a806e0..3fcd799730 100644 --- a/site/content/en/docs/Guides/metrics.md +++ b/site/content/en/docs/Guides/metrics.md @@ -65,7 +65,7 @@ Follow the [Google Cloud Monitoring installation steps](#google-cloud-monitoring | agones_fleet_autoscalers_limited | The fleet autoscaler is outside the limits set by MinReplicas and MaxReplicas. | gauge | | agones_gameservers_node_count | The distribution of gameservers per node | histogram | | agones_nodes_count | The count of nodes empty and with gameservers | gauge | -| agones_gameservers_state_duration | The distribution of gameserver state duration in seconds. Note: this metric could have some missing samples by design. Do not use the `_total` counter as the real value for state changes. | histogram | +| agones_gameserver_state_duration | The distribution of gameserver state duration in seconds. Note: this metric could have some missing samples by design. Do not use the `_total` counter as the real value for state changes. | histogram | | agones_k8s_client_http_request_total | The total of HTTP requests to the Kubernetes API by status code | counter | | agones_k8s_client_http_request_duration_seconds | The distribution of HTTP requests latencies to the Kubernetes API by status code | histogram | | agones_k8s_client_cache_list_total | The total number of list operations for client-go caches | counter | diff --git a/site/content/en/docs/Installation/Creating Cluster/eks.md b/site/content/en/docs/Installation/Creating Cluster/eks.md index 7aaa826a7e..8c67d8f442 100644 --- a/site/content/en/docs/Installation/Creating Cluster/eks.md +++ b/site/content/en/docs/Installation/Creating Cluster/eks.md @@ -41,6 +41,12 @@ For Agones to work correctly, we need to allow UDP traffic to pass through to ou * Select **Inbound Rules** * **Edit Rules** to add a new **Custom UDP Rule** with a 7000-8000 port range and an appropriate **Source** CIDR range (`0.0.0.0/0` allows all traffic) +## Use with custom CNI plugins + +To ensure compatibility with custom CNI plugins (such as Calico or [Cilium](https://docs.cilium.io/en/stable/installation/k8s-install-helm/)), configure extensions application to use `hostNetwork` for proper functionality via the `agones.extensions.hostNetwork` Agones [Helm]({{< relref "../Install Agones/helm.md" >}}) variable. Also make sure to set the `agones.extensions.http.port` and the `agones.extensions.webhooks.port` values as well. +This requirement arises due to the control plane's inability to reach in-cluster pod endpoints when the data plane operates on a separate network layer ([Github Issue](https://github.com/aws/containers-roadmap/issues/2227)). +Reconfiguring the kube-apiserver is not possible as it is an AWS managed component. + ## Next Steps -- Continue to [Install Agones]({{< relref "../Install Agones/_index.md" >}}). +* Continue to [Install Agones]({{< relref "../Install Agones/_index.md" >}}). diff --git a/site/content/en/docs/Installation/Install Agones/helm.md b/site/content/en/docs/Installation/Install Agones/helm.md index 63417e6db5..497d7a382b 100644 --- a/site/content/en/docs/Installation/Install Agones/helm.md +++ b/site/content/en/docs/Installation/Install Agones/helm.md @@ -293,7 +293,9 @@ The following tables lists the configurable parameters of the Agones chart and t | Parameter | Description | Default | |----------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------| +|`agones.extensions.hostNetwork` | Determines if the Agones extensions should operate in hostNetwork mode. If running in hostNetwork mode, you should change `agones.extensions.http.port` and `agones.extensions.webhooks.port` to an available port. | `false` | | `agones.extensions.http.port` | Port to use for liveness probe service and metrics | `8080` | +|`agones.extensions.webhooks.port` | Port to use for webhook service | `8081` | | `agones.extensions.healthCheck.initialDelaySeconds` | Initial delay before performing the first probe (in seconds) | `3` | | `agones.extensions.healthCheck.periodSeconds` | Seconds between every liveness probe (in seconds) | `3` | | `agones.extensions.healthCheck.failureThreshold` | Number of times before giving up (in seconds) | `3` | @@ -328,14 +330,6 @@ The following tables lists the configurable parameters of the Agones chart and t ### GameServers -{{% feature expiryVersion="1.40.0" %}} -| Parameter | Description | Default | -|----------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------| -| `gameservers.namespaces` | a list of namespaces you are planning to use to deploy game servers | `["default"]` | -| `gameservers.minPort` | Minimum port to use for dynamic port allocation | `7000` | -| `gameservers.maxPort` | Maximum port to use for dynamic port allocation | `8000` | -| `gameservers.podPreserveUnknownFields` | Disable [field pruning][pruning] and schema validation on the Pod template for a [GameServer][gameserver] definition | `false` | -{{% /feature %}} | Parameter | Description | Default | |----------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------| | `gameservers.namespaces` | a list of namespaces you are planning to use to deploy game servers | `["default"]` | diff --git a/test/sdk/go/Dockerfile b/test/sdk/go/Dockerfile new file mode 100644 index 0000000000..fe0ca478cd --- /dev/null +++ b/test/sdk/go/Dockerfile @@ -0,0 +1,40 @@ +# Copyright 2024 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build the Go image from source +FROM golang:1.22.6 AS build-stage + +WORKDIR /agones.dev + +COPY *.go ./ + +# Note: because this installs the most recently released version of Agones, this will need to be +# built and pushed post-release. +# TODO: Add to post-release check list. +RUN go mod init agones.dev/agones/test/sdk/go/sdk-client-test +RUN go mod tidy +RUN go mod download + +RUN CGO_ENABLED=0 GOOS=linux go build -o /sdk-client-test + +# Copy the above binary into a lean image +FROM gcr.io/distroless/static-debian12:nonroot AS build-release-stage + +WORKDIR / + +COPY --from=build-stage /sdk-client-test /sdk-client-test + +USER nonroot:nonroot + +ENTRYPOINT ["/sdk-client-test"] diff --git a/test/sdk/go/Makefile b/test/sdk/go/Makefile new file mode 100644 index 0000000000..3c3e60551b --- /dev/null +++ b/test/sdk/go/Makefile @@ -0,0 +1,51 @@ +# Copyright 2024 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Makefile for building a test game-server +# + +# __ __ _ _ _ +# \ \ / /_ _ _ __(_) __ _| |__ | | ___ ___ +# \ \ / / _` | '__| |/ _` | '_ \| |/ _ \ __| +# \ V / (_| | | | | (_| | |_) | | __\__ \ +# \_/ \__,_|_| |_|\__,_|_.__/|_|\___|___/ +# + +REGISTRY ?= +mkfile_path := $(abspath $(lastword $(MAKEFILE_LIST))) +project_path := $(dir $(mkfile_path)) +root_path = $(realpath $(project_path)/) +# Because go mod init in the Dockerfile installs the most recently released version of Agones, this +# will need to be built and pushed post-release. During DEV it will be built at DEV - 1. +release_version = 1.43.0 +server_tag := $(REGISTRY)/sdk-client-test:$(release_version) + +# _____ _ +# |_ _|_ _ _ __ __ _ ___| |_ ___ +# | |/ _` | '__/ _` |/ _ \ __/ __| +# | | (_| | | | (_| | __/ |_\__ \ +# |_|\__,_|_| \__, |\___|\__|___/ +# |___/ + +# Build a docker image for the server, and tag it +build: + cd $(root_path) && docker build -f $(project_path)Dockerfile --tag=$(server_tag) . + +push: build + docker push $(server_tag) + +# build and push the sdk-client-test image with specified tag +cloud-build: + cd $(root_path) && gcloud builds submit --config=cloudbuild.yaml diff --git a/test/sdk/go/cloudbuild.yaml b/test/sdk/go/cloudbuild.yaml new file mode 100644 index 0000000000..55faf63609 --- /dev/null +++ b/test/sdk/go/cloudbuild.yaml @@ -0,0 +1,38 @@ +--- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +steps: + # + # Creates the initial make + docker build platform + # + - name: ubuntu + script: | + echo 'FROM gcr.io/cloud-builders/docker:24.0.6\nRUN apt-get install make\nENTRYPOINT [\"/usr/bin/make\"]' > Dockerfile.build + - name: gcr.io/cloud-builders/docker:24.0.6 + id: build-make-docker + entrypoint: docker + args: [build, -f, Dockerfile.build, -t, make-docker, .] + + # build and push sdk-client-test image to Google Artifact Registry + - name: make-docker + id: push + dir: . + env: ['REGISTRY=${_REGISTRY}'] + script: | + make push +options: + dynamic_substitutions: true +substitutions: + _REGISTRY: us-docker.pkg.dev/${PROJECT_ID}/ci +timeout: 1800s diff --git a/test/sdk/go/sdk-client-test.go b/test/sdk/go/sdk-client-test.go index a5c022e9f8..699c382f7f 100644 --- a/test/sdk/go/sdk-client-test.go +++ b/test/sdk/go/sdk-client-test.go @@ -105,6 +105,9 @@ func main() { testLists(sdk) } + // Delay before shutdown to prevent Game Servers from churning too quickly on a running cluster + time.Sleep(8 * time.Second) + err = sdk.Shutdown() if err != nil { log.Fatalf("Could not shutdown GameServer: %s", err) diff --git a/test/upgrade/Dockerfile b/test/upgrade/Dockerfile index d7157ea7e6..52aa6a5e3f 100644 --- a/test/upgrade/Dockerfile +++ b/test/upgrade/Dockerfile @@ -15,6 +15,7 @@ FROM gcr.io/cloud-builders/gcloud AS builder RUN apt-get update && \ + apt-get install -y curl && \ apt-get clean WORKDIR /usr/local @@ -39,11 +40,12 @@ FROM golang:1.22.6 AS build-stage WORKDIR /agones.dev -COPY go.mod go.sum ./ -RUN go mod download - COPY *.go ./ +RUN go mod init agones.dev/agones/test/upgrade/testContainer +RUN go mod tidy +RUN go mod download + RUN CGO_ENABLED=0 GOOS=linux go build -o /upgrade-test # Copy the above binary into a lean image diff --git a/test/upgrade/Makefile b/test/upgrade/Makefile index e181b6b04c..4b6bfe5a0a 100644 --- a/test/upgrade/Makefile +++ b/test/upgrade/Makefile @@ -27,8 +27,8 @@ REGISTRY ?= mkfile_path := $(abspath $(lastword $(MAKEFILE_LIST))) project_path := $(dir $(mkfile_path)) root_path = $(realpath $(project_path)/) -version := 0.1 -server_tag := $(REGISTRY)/upgrade-test-controller:$(version) +dev_version = 1.44.0-dev +server_tag := $(REGISTRY)/upgrade-test-controller:$(dev_version) # _____ _ # |_ _|_ _ _ __ __ _ ___| |_ ___ diff --git a/test/upgrade/gameserverTemplate.yaml b/test/upgrade/gameserverTemplate.yaml new file mode 100644 index 0000000000..407ed218aa --- /dev/null +++ b/test/upgrade/gameserverTemplate.yaml @@ -0,0 +1,66 @@ +# Copyright 2024 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: gameserver-template +data: + gameserver.yaml: | + --- + apiVersion: agones.dev/v1 + kind: GameServer + metadata: + generateName: sdk-client-test- + labels: + agonesVersion: {{ .AgonesVersion }} + app: sdk-client-test + spec: + ports: + - name: default + portPolicy: Dynamic + containerPort: 7654 + sdkServer: + logLevel: Debug + {{ if .CountsAndLists }} + counters: + rooms: + count: 1 + capacity: 10 + lists: + players: + capacity: 100 + values: + - test0 + - test1 + - test2 + {{ end }} + template: + metadata: + labels: + agonesVersion: {{ .AgonesVersion }} + spec: + containers: + - name: sdk-client-test + image: "{{ .Registry }}:{{ .AgonesVersion }}" + imagePullPolicy: Always + resources: + requests: + memory: 64Mi + cpu: 20m + limits: + memory: 64Mi + cpu: 20m + serviceAccountName: agones-sa diff --git a/test/upgrade/go.mod b/test/upgrade/go.mod deleted file mode 100644 index 713631e28f..0000000000 --- a/test/upgrade/go.mod +++ /dev/null @@ -1,51 +0,0 @@ -module agones.dev/agones/test/upgrade/testContainer - -go 1.22.0 - -toolchain go1.22.6 - -require ( - k8s.io/apimachinery v0.31.0 - k8s.io/client-go v0.31.0 -) - -require ( - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/emicklei/go-restful/v3 v3.11.0 // indirect - github.com/fxamacker/cbor/v2 v2.7.0 // indirect - github.com/go-logr/logr v1.4.2 // indirect - github.com/go-openapi/jsonpointer v0.19.6 // indirect - github.com/go-openapi/jsonreference v0.20.2 // indirect - github.com/go-openapi/swag v0.22.4 // indirect - github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.4 // indirect - github.com/google/gnostic-models v0.6.8 // indirect - github.com/google/go-cmp v0.6.0 // indirect - github.com/google/gofuzz v1.2.0 // indirect - github.com/google/uuid v1.6.0 // indirect - github.com/josharian/intern v1.0.0 // indirect - github.com/json-iterator/go v1.1.12 // indirect - github.com/mailru/easyjson v0.7.7 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/onsi/gomega v1.33.1 // indirect - github.com/x448/float16 v0.8.4 // indirect - golang.org/x/net v0.26.0 // indirect - golang.org/x/oauth2 v0.21.0 // indirect - golang.org/x/sys v0.21.0 // indirect - golang.org/x/term v0.21.0 // indirect - golang.org/x/text v0.16.0 // indirect - golang.org/x/time v0.3.0 // indirect - google.golang.org/protobuf v1.34.2 // indirect - gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/api v0.31.0 // indirect - k8s.io/klog/v2 v2.130.1 // indirect - k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect - k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect - sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect - sigs.k8s.io/yaml v1.4.0 // indirect -) diff --git a/test/upgrade/go.sum b/test/upgrade/go.sum deleted file mode 100644 index 0cde55df07..0000000000 --- a/test/upgrade/go.sum +++ /dev/null @@ -1,152 +0,0 @@ -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= -github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= -github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= -github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= -github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= -github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= -github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= -github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= -github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= -github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= -github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af h1:kmjWCqn2qkEml422C2Rrd27c3VGxi6a/6HNq8QmHRKM= -github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= -github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA= -github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= -github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= -github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= -github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= -golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= -golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= -golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= -golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= -gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.31.0 h1:b9LiSjR2ym/SzTOlfMHm1tr7/21aD7fSkqgD/CVJBCo= -k8s.io/api v0.31.0/go.mod h1:0YiFF+JfFxMM6+1hQei8FY8M7s1Mth+z/q7eF1aJkTE= -k8s.io/apimachinery v0.31.0 h1:m9jOiSr3FoSSL5WO9bjm1n6B9KROYYgNZOb4tyZ1lBc= -k8s.io/apimachinery v0.31.0/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= -k8s.io/client-go v0.31.0 h1:QqEJzNjbN2Yv1H79SsS+SWnXkBgVu4Pj3CJQgbx0gI8= -k8s.io/client-go v0.31.0/go.mod h1:Y9wvC76g4fLjmU0BA+rV+h2cncoadjvjjkkIGoTLcGU= -k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= -k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= -k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= -k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= -k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= -sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= -sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= -sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= -sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/test/upgrade/main.go b/test/upgrade/main.go index 3bd7391c7f..6ce62924ae 100644 --- a/test/upgrade/main.go +++ b/test/upgrade/main.go @@ -19,55 +19,73 @@ import ( "context" "encoding/json" "fmt" + "html/template" "log" "os" "os/exec" "regexp" + "strconv" "strings" "time" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + "k8s.io/client-go/tools/cache" ) const ( - // HELM_CMD is the command line invocation of helm - HELM_CMD = "helm" - // KUBECTL_CMD is the command line invocation of kubectl - KUBECTL_CMD = "kubectl" - // IMAGE_PULL_POLICY sets the Agones Helm configuration to always pull the image - IMAGE_PULL_POLICY = "Always" - // SIDECAR_PULL_POLICY sets the Agones Helm configuration to always pull the SDK image - SIDECAR_PULL_POLICY = "true" - // LOG_LEVEL sets the Agones Helm configuration log level - LOG_LEVEL = "debug" - // HELM_CHART is the helm chart for the public Agones releases - HELM_CHART = "agones/agones" - // AGONES_REGISTRY is the public registry for Agones releases - AGONES_REGISTRY = "us-docker.pkg.dev/agones-images/release" + // HelmCmd is the command line invocation of helm + HelmCmd = "helm" + // KubectlCmd is the command line invocation of kubectl + KubectlCmd = "kubectl" + // ImagePullPolicy sets the Agones Helm configuration to always pull the image + ImagePullPolicy = "Always" + // SidecarPullPolicy sets the Agones Helm configuration to always pull the SDK image + SidecarPullPolicy = "true" + // LogLevel sets the Agones Helm configuration log level + LogLevel = "debug" + // HelmChart is the helm chart for the public Agones releases + HelmChart = "agones/agones" + // AgonesRegistry is the public registry for Agones releases + AgonesRegistry = "us-docker.pkg.dev/agones-images/release" + // TestRegistry is the public registry for upgrade test container files + TestRegistry = "us-docker.pkg.dev/agones-images/ci/sdk-client-test" ) var ( - // TODO: Get the build version of dev (i.e. 1.44.0-dev-b765f49) - // DEV is the current development version of Agones - DEV = os.Getenv("DEV") - // POD_NAME the name of the pod this container is running in - POD_NAME = os.Getenv("POD_NAME") - // POD_NAMESPACE the name of the pod namespace this container is running in - POD_NAMESPACE = os.Getenv("POD_NAMESPACE") - // VERSION_MAPPINGS are the valid Kubernetes, Agones, and Feature Gate version configurations - VERSION_MAPPINGS = os.Getenv("version-mappings.json") + // Dev is the current development version of Agones + Dev = os.Getenv("Dev") + // ReleaseVersion is the latest released version of Agones (DEV - 1). + ReleaseVersion = os.Getenv("ReleaseVersion") + // PodName the name of the pod this container is running in + PodName = os.Getenv("PodName") + // PodNamespace the name of the pod namespace this container is running in + PodNamespace = os.Getenv("PodNamespace") + // VersionMappings are the valid Kubernetes, Agones, and Feature Gate version configurations + VersionMappings = os.Getenv("version-mappings.json") ) func main() { ctx := context.Background() + cfg, err := rest.InClusterConfig() + if err != nil { + log.Fatal("Could not create in cluster config", cfg) + } + + kubeClient, err := kubernetes.NewForConfig(cfg) + if err != nil { + log.Fatal("Could not create the kubernetes api clientset", err) + } - validConfigs := ConfigTestSetup(ctx) + validConfigs := configTestSetup(ctx, kubeClient) + go watchGameServerPods(kubeClient, make(chan struct{}), make(map[string]podLog), len(validConfigs)*2) addAgonesRepo() runConfigWalker(ctx, validConfigs) + cleanUpResources() } type versionMappings struct { @@ -81,8 +99,21 @@ type featureGates struct { } type configTest struct { - agonesVersion string - featureGates string + agonesVersion string + featureGates string + gameServerPath string +} + +// CountsAndLists can be removed from the template once CountsAndLists is GA in all tested versions +type gameServerTemplate struct { + AgonesVersion string + Registry string + CountsAndLists bool +} + +type podLog struct { + SdkVersion string + GameServerVersion string } type helmStatuses []struct { @@ -96,14 +127,14 @@ type helmStatuses []struct { } // Determine test scenario to run -func ConfigTestSetup(ctx context.Context) []*configTest { +func configTestSetup(ctx context.Context, kubeClient *kubernetes.Clientset) []*configTest { versionMap := versionMappings{} // Find the Kubernetes version of the node that this test is running on. - k8sVersion := findK8sVersion(ctx) + k8sVersion := findK8sVersion(ctx, kubeClient) // Get the mappings of valid Kubernetes, Agones, and Feature Gate versions from the configmap. - err := json.Unmarshal([]byte(VERSION_MAPPINGS), &versionMap) + err := json.Unmarshal([]byte(VersionMappings), &versionMap) if err != nil { log.Fatal("Could not Unmarshal", err) } @@ -112,34 +143,49 @@ func ConfigTestSetup(ctx context.Context) []*configTest { configTests := []*configTest{} for _, agonesVersion := range versionMap.K8sToAgonesVersions[k8sVersion] { ct := configTest{} + // TODO: create different valid config based off of available feature gates. + // containsCountsAndLists will need to be updated to return true for when CountsAndLists=true. + countsAndLists := containsCountsAndLists(agonesVersion) ct.agonesVersion = agonesVersion - if agonesVersion == "DEV" { - ct.agonesVersion = DEV + if agonesVersion == "Dev" { + ct.agonesVersion = Dev + // Game server container cannot be created at DEV version due to go.mod only able to access + // published Agones versions. Use N-1 for DEV. + ct.gameServerPath = createGameServerFile(ReleaseVersion, countsAndLists) + } else { + ct.gameServerPath = createGameServerFile(agonesVersion, countsAndLists) } - // TODO: create different valid config based off of available feature gates configTests = append(configTests, &ct) } return configTests } -// Finds the Kubernetes version of the Kubelet on the node that the current pod is running on. -// The Kubelet version is the same version as the node. -func findK8sVersion(ctx context.Context) string { - cfg, err := rest.InClusterConfig() - if err != nil { - log.Fatal("Could not create in cluster config", cfg) +// containsCountsAndLists returns true if the agonesVersion >= 1.41.0 when the CountsAndLists +// feature entered Beta (on by default) +func containsCountsAndLists(agonesVersion string) bool { + if agonesVersion == "Dev" { + return true } - - kubeClient, err := kubernetes.NewForConfig(cfg) + r := regexp.MustCompile(`\d+\.\d+`) + strVersion := r.FindString(agonesVersion) + floatVersion, err := strconv.ParseFloat(strVersion, 64) if err != nil { - log.Fatal("Could not create the kubernetes api clientset", err) + log.Fatalf("Could not convert agonesVersion %s to float: %s", agonesVersion, err) } + if floatVersion > 1.40 { + return true + } + return false +} +// Finds the Kubernetes version of the Kubelet on the node that the current pod is running on. +// The Kubelet version is the same version as the node. +func findK8sVersion(ctx context.Context, kubeClient *kubernetes.Clientset) string { // Wait to get pod and node as these may take a while to start on a new Autopilot cluster. var pod *v1.Pod - err = wait.PollUntilContextTimeout(ctx, 5*time.Second, 7*time.Minute, true, func(ctx context.Context) (done bool, err error) { - pod, err = kubeClient.CoreV1().Pods(POD_NAMESPACE).Get(ctx, POD_NAME, metav1.GetOptions{}) + err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 7*time.Minute, true, func(ctx context.Context) (done bool, err error) { + pod, err = kubeClient.CoreV1().Pods(PodNamespace).Get(ctx, PodName, metav1.GetOptions{}) if err != nil { return false, nil } @@ -166,10 +212,7 @@ func findK8sVersion(ctx context.Context) string { // Finds the major.min version. I.e. k8sVersion 1.30 from gkeVersion v1.30.2-gke.1587003 gkeVersion := node.Status.NodeInfo.KubeletVersion log.Println("KubeletVersion", gkeVersion) - r, err := regexp.Compile("\\d*\\.\\d*") - if err != nil { - log.Fatal("Could not compile regex: ", err) - } + r := regexp.MustCompile(`\d+\.\d+`) return r.FindString(gkeVersion) } @@ -196,7 +239,7 @@ func addAgonesRepo() { {"repo", "update"}} for _, args := range installArgs { - _, err := runExecCommand(HELM_CMD, args...) + _, err := runExecCommand(HelmCmd, args...) if err != nil { log.Fatalf("Could not add Agones helm repo: %s", err) } @@ -208,6 +251,8 @@ func installAgonesRelease(version, registry, featureGates, imagePullPolicy, side // TODO: Include feature gates. (Current issue with Helm and string formatting of the feature gates.) // "--set agones.featureGates=%s "+ + // Remove this print line which is here to prevent linter complaining about featureGates not used. + log.Printf("Agones Version %s, FeatureGates %s", version, featureGates) helmString := fmt.Sprintf( "upgrade --install --atomic --wait --timeout=10m --namespace=agones-system --create-namespace --version %s "+ @@ -226,7 +271,7 @@ func installAgonesRelease(version, registry, featureGates, imagePullPolicy, side helmArgs := strings.Split(helmString, " ") - _, err := runExecCommand(HELM_CMD, helmArgs...) + _, err := runExecCommand(HelmCmd, helmArgs...) if err != nil { return err } @@ -235,16 +280,17 @@ func installAgonesRelease(version, registry, featureGates, imagePullPolicy, side } func runConfigWalker(ctx context.Context, validConfigs []*configTest) { + cancelCtx, cancel := context.WithCancel(ctx) + for _, config := range validConfigs { - registry := AGONES_REGISTRY - chart := HELM_CHART - if config.agonesVersion == DEV { + registry := AgonesRegistry + chart := HelmChart + if config.agonesVersion == Dev { // TODO: Update to templated value for registry and chart for Dev build continue } - - err := installAgonesRelease(config.agonesVersion, registry, config.featureGates, IMAGE_PULL_POLICY, - SIDECAR_PULL_POLICY, LOG_LEVEL, chart) + err := installAgonesRelease(config.agonesVersion, registry, config.featureGates, ImagePullPolicy, + SidecarPullPolicy, LogLevel, chart) if err != nil { log.Printf("installAgonesRelease err: %s", err) } @@ -258,21 +304,27 @@ func runConfigWalker(ctx context.Context, validConfigs []*configTest) { } return false, nil }) - if err != nil || helmStatus != "deployed" { log.Fatalf("PollUntilContextTimeout timed out while attempting upgrade to Agones version %s. Helm Status %s", config.agonesVersion, helmStatus) } + + go createGameServers(cancelCtx, config.gameServerPath) + // Allow some soak time at the Agones version before next upgrade + time.Sleep(1 * time.Minute) } + cancel() + // TODO: Replace sleep with wait for the existing healthy Game Servers finish naturally by reaching their shutdown phase. + time.Sleep(30 * time.Second) } // checkHelmStatus returns the status of the Helm release at a specified agonesVersion if it exists. func checkHelmStatus(agonesVersion string) string { helmStatus := helmStatuses{} checkStatus := []string{"list", "-a", "-nagones-system", "-ojson"} - out, err := runExecCommand(HELM_CMD, checkStatus...) + out, err := runExecCommand(HelmCmd, checkStatus...) if err != nil { - log.Fatalf("Could not run command %s %s, err: %s", KUBECTL_CMD, checkStatus, err) + log.Fatalf("Could not run command %s %s, err: %s", KubectlCmd, checkStatus, err) } err = json.Unmarshal(out, &helmStatus) @@ -287,3 +339,132 @@ func checkHelmStatus(agonesVersion string) string { } return "" } + +// Creates a gameserver yaml file from the mounted gameserver.yaml template. The name of the new +// gameserver yaml is based on the Agones version, i.e. gs1440.yaml for Agones version 1.44.0 +func createGameServerFile(agonesVersion string, countsAndLists bool) string { + gsTmpl := gameServerTemplate{Registry: TestRegistry, AgonesVersion: agonesVersion, CountsAndLists: countsAndLists} + + gsTemplate, err := template.ParseFiles("gameserver.yaml") + if err != nil { + log.Fatal("Could not ParseFiles template gameserver.yaml", err) + } + + // Must use /tmp since the container is a non-root user. + gsPath := strings.ReplaceAll("/tmp/gs"+agonesVersion, ".", "") + gsPath += ".yaml" + // Check if the file already exists + if _, err := os.Stat(gsPath); err == nil { + return gsPath + } + gsFile, err := os.Create(gsPath) + if err != nil { + log.Fatal("Could not create file ", err) + } + + err = gsTemplate.Execute(gsFile, gsTmpl) + if err != nil { + if fErr := gsFile.Close(); fErr != nil { + log.Printf("Could not close game server file %s, err: %s", gsPath, fErr) + } + log.Fatal("Could not Execute template gameserver.yaml ", err) + } + if err = gsFile.Close(); err != nil { + log.Printf("Could not close game server file %s, err: %s", gsPath, err) + } + + return gsPath +} + +// Create a game server every five seconds until the context is cancelled. The game server container +// be the same binary version as the game server file. The SDK version is always the same as the +// version of the Agones controller that created it. The Game Server shuts itself down after the +// tests have run as part of the `sdk-client-test` logic. +func createGameServers(ctx context.Context, gsPath string) { + args := []string{"create", "-f", gsPath} + ticker := time.NewTicker(5 * time.Second) + + for { + select { + case <-ctx.Done(): + ticker.Stop() + return + case <-ticker.C: + _, err := runExecCommand(KubectlCmd, args...) + // TODO: Do not ignore error if unable to create due to something other than cluster scale up + if err != nil { + log.Printf("Could not create Gameserver %s: %s", gsPath, err) + } + } + } +} + +// watchGameServerPods watches all game server pods for CrashLoopBackOff. Errors if the number of +// CrashLoopBackOff backoff pods exceeds the number of acceptedFailures. +func watchGameServerPods(kubeClient *kubernetes.Clientset, stopCh chan struct{}, failedPods map[string]podLog, acceptedFailures int) { + // Filter by label agones.dev/role=gameserver to only game server pods + labelOptions := informers.WithTweakListOptions(func(opts *metav1.ListOptions) { + opts.LabelSelector = "agones.dev/role=gameserver" + }) + kubeInformerFactory := informers.NewSharedInformerFactoryWithOptions(kubeClient, 5*time.Second, + informers.WithNamespace("default"), labelOptions) + podInformer := kubeInformerFactory.Core().V1().Pods().Informer() + + _, err := podInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + UpdateFunc: func(_, newObj interface{}) { + newPod := newObj.(*v1.Pod) + for _, cs := range newPod.Status.ContainerStatuses { + if cs.Name != "sdk-client-test" || cs.State.Waiting == nil || cs.State.Waiting.Reason != "CrashLoopBackOff" { + continue + } + gsVersion := newPod.Labels["agonesVersion"] + sdkVersion := newPod.Annotations["agones.dev/sdk-version"] + log.Printf("%s for pod: %s with game server binary version %s, and SDK version %s", cs.State.Waiting.Reason, newPod.Name, gsVersion, sdkVersion) + // Put failed pods into the map until it reaches capacity. + failedPods[newPod.Name] = podLog{GameServerVersion: gsVersion, SdkVersion: sdkVersion} + if len(failedPods) > acceptedFailures { + log.Fatalf("Too many Game Server pods in CrashLoopBackOff: %v", failedPods) + } + } + }, + }) + if err != nil { + log.Fatal("Not able to create AddEventHandler", err) + } + + go podInformer.Run(stopCh) + if !cache.WaitForCacheSync(stopCh, podInformer.HasSynced) { + log.Fatal("Timed out waiting for caches to sync") + } +} + +// Deletes any remaining Game Servers, Uninstalls Agones, and Deletes agones-system namespace. +func cleanUpResources() { + args := []string{"delete", "gs", "-l", "app=sdk-client-test"} + _, err := runExecCommand(KubectlCmd, args...) + if err != nil { + log.Println("Could not delete game servers", err) + } + + args = []string{"uninstall", "agones", "-n", "agones-system"} + _, err = runExecCommand(HelmCmd, args...) + if err != nil { + log.Println("Could not Helm uninstall Agones", err) + } + + // Apiservice v1.allocation.agones.dev, which is part of Service agones-system/agones-controller-service, + // does not always get cleaned up on Helm uninstall, and needs to be deleted (if it exists) before + // the agones-system namespace can be removed. + // Ignore the error, because an "error" means Helm already uninstall the apiservice. + args = []string{"delete", "apiservice", "v1.allocation.agones.dev"} + out, err := runExecCommand(KubectlCmd, args...) + if err == nil { + fmt.Println(string(out)) + } + + args = []string{"delete", "ns", "agones-system"} + _, err = runExecCommand(KubectlCmd, args...) + if err != nil { + log.Println("Could not delete agones-system namespace", err) + } +} diff --git a/test/upgrade/permissions.yaml b/test/upgrade/permissions.yaml index 2de3ae82f1..1f4a96005b 100644 --- a/test/upgrade/permissions.yaml +++ b/test/upgrade/permissions.yaml @@ -83,47 +83,47 @@ description: "This priority class should be used for upgrade-test-runner pods on apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: namespace-creator + name: namespace-manager rules: - apiGroups: [""] resources: ["namespaces"] - verbs: ["get", "list", "watch", "create"] + verbs: ["get", "list", "watch", "create", "delete"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: create-namespaces + name: manage-namespaces subjects: - kind: ServiceAccount name: agones-sa namespace: default roleRef: kind: ClusterRole - name: namespace-creator + name: namespace-manager apiGroup: rbac.authorization.k8s.io --- -# Helm needs to be able to create secrets +# Helm needs to be able to perform CRUD operations on secrets apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: namespace: agones-system - name: secret-creator + name: secret-manager rules: - apiGroups: [""] resources: ["secrets"] - verbs: ["get", "watch", "list", "create", "patch", "update"] + verbs: ["get", "watch", "list", "create", "patch", "update", "delete"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: create-secrets + name: manage-secrets subjects: - kind: ServiceAccount name: agones-sa namespace: default roleRef: kind: ClusterRole - name: secret-creator + name: secret-manager apiGroup: rbac.authorization.k8s.io --- # Helm needs to be able to create priorityclasses @@ -376,7 +376,7 @@ rules: verbs: ["patch"] - apiGroups: ["agones.dev"] resources: ["fleets"] - verbs: ["get", "list", "update", "watch"] + verbs: ["create", "delete", "get", "list", "update", "watch"] - apiGroups: ["agones.dev"] resources: ["fleets/status", "gameserversets/status"] verbs: ["update"] @@ -439,6 +439,9 @@ subjects: - kind: User name: system:serviceaccount:default:agones-sa apiGroup: rbac.authorization.k8s.io + - kind: ServiceAccount + name: agones-sa + namespace: default roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -488,6 +491,9 @@ subjects: - kind: User name: system:serviceaccount:default:agones-sa apiGroup: rbac.authorization.k8s.io + - kind: ServiceAccount + name: agones-sa + namespace: default roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole diff --git a/test/upgrade/upgradeTest.yaml b/test/upgrade/upgradeTest.yaml index 91910e0a4e..ebb301953c 100644 --- a/test/upgrade/upgradeTest.yaml +++ b/test/upgrade/upgradeTest.yaml @@ -26,21 +26,29 @@ spec: spec: containers: - name: upgrade-test-controller - # TODO(igooch): Update image name to use a templated value - image: $(REGISTRY)/upgrade-test-controller:0.1 + # TODO: Update image name to use a templated value for current Dev version + image: us-docker.pkg.dev/agones-images/ci/upgrade-test-controller:1.44.0-dev imagePullPolicy: Always env: - - name: POD_NAME + - name: PodName valueFrom: fieldRef: fieldPath: metadata.name - - name: POD_NAMESPACE + - name: PodNamespace valueFrom: fieldRef: fieldPath: metadata.namespace envFrom: - configMapRef: name: version-map + volumeMounts: + - name: gs-volume + mountPath: gameserver.yaml + subPath: "gameserver.yaml" + volumes: + - name: gs-volume + configMap: + name: gameserver-template priorityClassName: high-priority restartPolicy: Never serviceAccountName: agones-sa diff --git a/test/upgrade/versionMap.yaml b/test/upgrade/versionMap.yaml index 67bc14d601..b0a7499de0 100644 --- a/test/upgrade/versionMap.yaml +++ b/test/upgrade/versionMap.yaml @@ -18,17 +18,16 @@ kind: ConfigMap metadata: name: version-map data: - DEV: 1.44.0-dev + Dev: "1.44.0-dev" + ReleaseVersion: "1.43.0" version-mappings.json: | { "k8sToAgonesVersions": { "1.25": [ - "1.33.0", "1.34.0", "1.35.0" ], "1.26": [ - "1.33.0", "1.34.0", "1.35.0", "1.36.0", @@ -56,25 +55,21 @@ data: "1.41.0", "1.42.0", "1.43.0", - "DEV" + "Dev" ], "1.29": [ "1.40.0", "1.41.0", "1.42.0", "1.43.0", - "DEV" + "Dev" ], "1.30": [ "1.43.0", - "DEV" + "Dev" ] }, "agonesVersionFeatureGates": { - "1.33.0": { - "alphaGates": ["PlayerAllocationFilter", "PlayerTracking"], - "betaGates": [] - }, "1.34.0": { "alphaGates": ["PlayerAllocationFilter", "PlayerTracking"], "betaGates": [] @@ -115,7 +110,7 @@ data: "alphaGates": ["GKEAutopilotExtendedDurationPods", "PlayerAllocationFilter", "PlayerTracking", "PortPolicyNone", "PortRanges", "RollingUpdateFix"], "betaGates": ["AutopilotPassthroughPort", "CountsAndLists", "DisableResyncOnSDKServer"] }, - "DEV": { + "Dev": { "alphaGates": ["GKEAutopilotExtendedDurationPods", "PlayerAllocationFilter", "PlayerTracking", "PortPolicyNone", "PortRanges", "RollingUpdateFix", "ScheduledAutoscaler"], "betaGates": ["AutopilotPassthroughPort", "CountsAndLists", "DisableResyncOnSDKServer"] }