From 2bb3de07c6368b3f71a829f8eef9361b94f7b103 Mon Sep 17 00:00:00 2001 From: Dimitris Gkanatsios Date: Tue, 30 Nov 2021 17:51:24 -0800 Subject: [PATCH 1/4] fixing controller issues --- docs/development.md | 7 ++-- installfiles/operator.yaml | 2 +- installfiles/operator_with_monitoring.yaml | 2 +- installfiles/operator_with_security.yaml | 2 +- ...operator_with_security_and_monitoring.yaml | 2 +- .../api/v1alpha1/gameserverbuild_types.go | 1 + .../mps.playfab.com_gameserverbuilds.yaml | 2 + operator/config/manager/kustomization.yaml | 2 +- operator/config/manager/manager.yaml | 2 +- .../controllers/gameserverbuild_controller.go | 39 ++++++++----------- operator/main.go | 12 +++--- 11 files changed, 36 insertions(+), 37 deletions(-) diff --git a/docs/development.md b/docs/development.md index 5128a6bb..820d5f93 100644 --- a/docs/development.md +++ b/docs/development.md @@ -84,9 +84,10 @@ make deletekindcluster && make builddockerlocal && make createkindcluster && mak ## Run controller locally -export INIT_CONTAINER_TAG=... -export SIDECAR_TAG=... -cd operator && THUNDERNETES_SIDECAR_IMAGE=ghcr.io/playfab/thundernetes-sidecar-go:${SIDECAR_TAG} THUNDERNETES_INIT_CONTAINER_IMAGE=ghcr.io/playfab/thundernetes-initcontainer:${INIT_CONTAINER_TAG} go run main.go +```bash +cd operator +THUNDERNETES_SIDECAR_IMAGE=ghcr.io/playfab/thundernetes-sidecar-go:0.1.0 THUNDERNETES_INIT_CONTAINER_IMAGE=ghcr.io/playfab/thundernetes-initcontainer:0.1.0 go run main.go +``` ## [ADVANCED] Install thundernetes via cloning this repository diff --git a/installfiles/operator.yaml b/installfiles/operator.yaml index 06ed2926..12f3c991 100755 --- a/installfiles/operator.yaml +++ b/installfiles/operator.yaml @@ -8080,7 +8080,7 @@ spec: value: ghcr.io/playfab/thundernetes-sidecar-go:0.1.0 - name: THUNDERNETES_INIT_CONTAINER_IMAGE value: ghcr.io/playfab/thundernetes-initcontainer:0.1.0 - - name: POD_NAMESPACE + - name: TLS_SECRET_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace diff --git a/installfiles/operator_with_monitoring.yaml b/installfiles/operator_with_monitoring.yaml index 13d7822f..d7d8369c 100644 --- a/installfiles/operator_with_monitoring.yaml +++ b/installfiles/operator_with_monitoring.yaml @@ -8080,7 +8080,7 @@ spec: value: ghcr.io/playfab/thundernetes-sidecar-go:0.1.0 - name: THUNDERNETES_INIT_CONTAINER_IMAGE value: ghcr.io/playfab/thundernetes-initcontainer:0.1.0 - - name: POD_NAMESPACE + - name: TLS_SECRET_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace diff --git a/installfiles/operator_with_security.yaml b/installfiles/operator_with_security.yaml index 68860244..36a6bbda 100644 --- a/installfiles/operator_with_security.yaml +++ b/installfiles/operator_with_security.yaml @@ -8080,7 +8080,7 @@ spec: value: ghcr.io/playfab/thundernetes-sidecar-go:0.1.0 - name: THUNDERNETES_INIT_CONTAINER_IMAGE value: ghcr.io/playfab/thundernetes-initcontainer:0.1.0 - - name: POD_NAMESPACE + - name: TLS_SECRET_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace diff --git a/installfiles/operator_with_security_and_monitoring.yaml b/installfiles/operator_with_security_and_monitoring.yaml index bfd5f125..e9297c70 100644 --- a/installfiles/operator_with_security_and_monitoring.yaml +++ b/installfiles/operator_with_security_and_monitoring.yaml @@ -8080,7 +8080,7 @@ spec: value: ghcr.io/playfab/thundernetes-sidecar-go:0.1.0 - name: THUNDERNETES_INIT_CONTAINER_IMAGE value: ghcr.io/playfab/thundernetes-initcontainer:0.1.0 - - name: POD_NAMESPACE + - name: TLS_SECRET_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace diff --git a/operator/api/v1alpha1/gameserverbuild_types.go b/operator/api/v1alpha1/gameserverbuild_types.go index 82dcc8d9..c2edc557 100644 --- a/operator/api/v1alpha1/gameserverbuild_types.go +++ b/operator/api/v1alpha1/gameserverbuild_types.go @@ -73,6 +73,7 @@ type GameServerBuildSpec struct { type GameServerBuildStatus struct { // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster // Important: Run "make" to regenerate code after modifying this file + CurrentPending int `json:"currentPending,omitempty"` CurrentInitializing int `json:"currentInitializing"` CurrentStandingBy int `json:"currentStandingBy"` CurrentStandingByReadyDesired string `json:"currentStandingByReadyDesired"` diff --git a/operator/config/crd/bases/mps.playfab.com_gameserverbuilds.yaml b/operator/config/crd/bases/mps.playfab.com_gameserverbuilds.yaml index f8e226aa..ec947427 100644 --- a/operator/config/crd/bases/mps.playfab.com_gameserverbuilds.yaml +++ b/operator/config/crd/bases/mps.playfab.com_gameserverbuilds.yaml @@ -6104,6 +6104,8 @@ spec: currentActive: type: integer currentInitializing: + type: integer + currentPending: description: 'INSERT ADDITIONAL STATUS FIELD - define observed state of cluster Important: Run "make" to regenerate code after modifying this file' diff --git a/operator/config/manager/kustomization.yaml b/operator/config/manager/kustomization.yaml index e75e8d61..1ad8e90f 100755 --- a/operator/config/manager/kustomization.yaml +++ b/operator/config/manager/kustomization.yaml @@ -11,4 +11,4 @@ kind: Kustomization images: - name: controller newName: thundernetes-operator - newTag: ae80381 + newTag: b460a56 diff --git a/operator/config/manager/manager.yaml b/operator/config/manager/manager.yaml index f30e4cab..8633e843 100644 --- a/operator/config/manager/manager.yaml +++ b/operator/config/manager/manager.yaml @@ -38,7 +38,7 @@ spec: value: ${IMAGE_NAME_SIDECAR}:${SIDECAR_TAG} - name: THUNDERNETES_INIT_CONTAINER_IMAGE value: ${IMAGE_NAME_INIT_CONTAINER}:${INIT_CONTAINER_TAG} - - name: POD_NAMESPACE + - name: TLS_SECRET_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace diff --git a/operator/controllers/gameserverbuild_controller.go b/operator/controllers/gameserverbuild_controller.go index 7dd56b6a..b70d421d 100644 --- a/operator/controllers/gameserverbuild_controller.go +++ b/operator/controllers/gameserverbuild_controller.go @@ -21,6 +21,7 @@ import ( "fmt" "time" + mpsv1alpha1 "github.com/playfab/thundernetes/operator/api/v1alpha1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -32,8 +33,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/log" - mpsv1alpha1 "github.com/playfab/thundernetes/operator/api/v1alpha1" - hm "github.com/cornelk/hashmap" ) @@ -120,11 +119,13 @@ func (r *GameServerBuildReconciler) Reconcile(ctx context.Context, req ctrl.Requ } // calculate counts by state so we can update .status accordingly - var activeCount, standingByCount, crashesCount, initializingCount int + var activeCount, standingByCount, crashesCount, initializingCount, pendingCount int for i := 0; i < len(gameServers.Items); i++ { gs := gameServers.Items[i] - if gs.Status.State == "" || gs.Status.State == mpsv1alpha1.GameServerStateInitializing { + if gs.Status.State == "" { + pendingCount++ + } else if gs.Status.State == mpsv1alpha1.GameServerStateInitializing { initializingCount++ } else if gs.Status.State == mpsv1alpha1.GameServerStateStandingBy { standingByCount++ @@ -148,20 +149,13 @@ func (r *GameServerBuildReconciler) Reconcile(ctx context.Context, req ctrl.Requ } } - // if at least one gameServer doesn't have a State, this means that it's initializing - // update the gameServerBuild status and exit the reconcile loop - // once this gameServer gets a State, the reconcile loop will be re-triggered again - if initializingCount > 0 { - return r.updateStatus(ctx, &gsb, initializingCount, standingByCount, activeCount, crashesCount) - } - // user has decreased standingBy numbers if standingByCount > gsb.Spec.StandingBy { deletedCount := 0 for i := 0; i < standingByCount-gsb.Spec.StandingBy; i++ { gs := gameServers.Items[i] - // we're deleting only standingBy servers - if gs.Status.State == "" || gs.Status.State == mpsv1alpha1.GameServerStateStandingBy { + // we're deleting only standingBy/initializing servers + if gs.Status.State == "" || gs.Status.State == mpsv1alpha1.GameServerStateInitializing || gs.Status.State == mpsv1alpha1.GameServerStateStandingBy { if err := r.Delete(ctx, &gs); err != nil { return ctrl.Result{}, err } @@ -185,7 +179,7 @@ func (r *GameServerBuildReconciler) Reconcile(ctx context.Context, req ctrl.Requ for i := 0; i <= standingByCount+activeCount-gsb.Spec.Max; i++ { gs := gameServers.Items[i] // we're deleting only standingBy or initializing servers - if gs.Status.State == "" || gs.Status.State == mpsv1alpha1.GameServerStateStandingBy { + if gs.Status.State == "" || gs.Status.State == mpsv1alpha1.GameServerStateInitializing || gs.Status.State == mpsv1alpha1.GameServerStateStandingBy { if err := r.Delete(ctx, &gs); err != nil { return ctrl.Result{}, err } @@ -202,7 +196,8 @@ func (r *GameServerBuildReconciler) Reconcile(ctx context.Context, req ctrl.Requ } // we are in need of standingBy servers, so we're creating them here - for i := 0; i < gsb.Spec.StandingBy-standingByCount && i+standingByCount+activeCount < gsb.Spec.Max; i++ { + for i := 0; i < gsb.Spec.StandingBy-pendingCount-initializingCount-standingByCount && + i+pendingCount+initializingCount+standingByCount+activeCount < gsb.Spec.Max; i++ { newgs, err := NewGameServerForGameServerBuild(&gsb, r.PortRegistry) if err != nil { return ctrl.Result{}, err @@ -216,16 +211,18 @@ func (r *GameServerBuildReconciler) Reconcile(ctx context.Context, req ctrl.Requ r.Recorder.Eventf(&gsb, corev1.EventTypeNormal, "Creating", "Creating GameServer %s", newgs.Name) } - return r.updateStatus(ctx, &gsb, initializingCount, standingByCount, activeCount, crashesCount) + return r.updateStatus(ctx, &gsb, pendingCount, initializingCount, standingByCount, activeCount, crashesCount) } -func (r *GameServerBuildReconciler) updateStatus(ctx context.Context, gsb *mpsv1alpha1.GameServerBuild, initializingCount, standingByCount, activeCount, crashesCount int) (ctrl.Result, error) { +func (r *GameServerBuildReconciler) updateStatus(ctx context.Context, gsb *mpsv1alpha1.GameServerBuild, pendingCount, initializingCount, standingByCount, activeCount, crashesCount int) (ctrl.Result, error) { // update GameServerBuild status only if one of the fields has changed - if gsb.Status.CurrentInitializing != initializingCount || + if gsb.Status.CurrentPending != pendingCount || + gsb.Status.CurrentInitializing != initializingCount || gsb.Status.CurrentActive != activeCount || gsb.Status.CurrentStandingBy != standingByCount || crashesCount > 0 { + gsb.Status.CurrentPending = pendingCount gsb.Status.CurrentInitializing = initializingCount gsb.Status.CurrentActive = activeCount gsb.Status.CurrentStandingBy = standingByCount @@ -323,9 +320,8 @@ func (r *GameServerBuildReconciler) gameServersUnderDeletionWereDeleted(ctx cont // so it's safe to remove the GameServerBuild entry from the map gameServersUnderDeletion.Del(gsb.Name) return true, nil - } else { - return false, nil } + return false, nil } return true, nil } @@ -354,9 +350,8 @@ func (r *GameServerBuildReconciler) gameServersUnderCreationWereCreated(ctx cont // so it's safe to remove the GameServerBuild entry from the map gameServersUnderCreation.Del(gsb.Name) return true, nil - } else { - return false, nil } + return false, nil } return true, nil } diff --git a/operator/main.go b/operator/main.go index ed160b09..9476c46d 100644 --- a/operator/main.go +++ b/operator/main.go @@ -98,15 +98,15 @@ func main() { os.Exit(1) } - namespace := os.Getenv("POD_NAMESPACE") - if namespace == "" { - setupLog.Error(err, "unable to start live API client") - os.Exit(1) - } var crt, key []byte apiServiceSecurity := os.Getenv("API_SERVICE_SECURITY") if apiServiceSecurity == "usetls" { + namespace := os.Getenv("TLS_SECRET_NAMESPACE") + if namespace == "" { + setupLog.Error(err, "unable to get TLS_SECRET_NAMESPACE env variable") + os.Exit(1) + } crt, key, err = getTlsSecret(k8sClient, namespace) if err != nil { setupLog.Error(err, "unable to get TLS secret") @@ -142,7 +142,7 @@ func main() { err = http.NewApiServer(mgr, crt, key) if err != nil { - setupLog.Error(err, "unable to create HTTP API Server", "API Server", "HTTP API Server") + setupLog.Error(err, "unable to create HTTP allocation API Server", "Allocation API Server", "HTTP Allocation API Server") os.Exit(1) } From a35b72d7bf44e6b38428eeb032f12a010b405648 Mon Sep 17 00:00:00 2001 From: Dimitris-Ilias Gkanatsios Date: Tue, 30 Nov 2021 19:55:21 -0800 Subject: [PATCH 2/4] updating sample version --- docs/development.md | 1 + samples/netcore/sample-nodeaffinity.yaml | 2 +- samples/netcore/sample.yaml | 2 +- samples/netcore/sample_second_node_pool.yaml | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/development.md b/docs/development.md index 820d5f93..6907856a 100644 --- a/docs/development.md +++ b/docs/development.md @@ -6,6 +6,7 @@ - Run `make create-install-files` - Push and merge - Run the GitHub Actions workflow [here](https://github.com/PlayFab/thundernetes/actions/workflows/publish.yml) +- Replace the image on the [netcore-sample YAML files](../samples/netcore) ## Generate install files diff --git a/samples/netcore/sample-nodeaffinity.yaml b/samples/netcore/sample-nodeaffinity.yaml index b5a0f7ea..44e9a2e1 100644 --- a/samples/netcore/sample-nodeaffinity.yaml +++ b/samples/netcore/sample-nodeaffinity.yaml @@ -18,7 +18,7 @@ spec: portName: gameport # must be the same as the port name described below podSpec: containers: - - image: ghcr.io/playfab/thundernetes-netcore-sample:0.0.1.2 + - image: ghcr.io/playfab/thundernetes-netcore-sample:0.1.0 name: thundernetes-sample-netcore ports: - containerPort: 80 # your game server port diff --git a/samples/netcore/sample.yaml b/samples/netcore/sample.yaml index db689fdd..f03ad772 100644 --- a/samples/netcore/sample.yaml +++ b/samples/netcore/sample.yaml @@ -18,7 +18,7 @@ spec: portName: gameport # must be the same as the port name described below podSpec: containers: - - image: ghcr.io/playfab/thundernetes-netcore-sample:0.0.1.2 + - image: ghcr.io/playfab/thundernetes-netcore-sample:0.1.0 name: thundernetes-sample-netcore ports: - containerPort: 80 # your game server port diff --git a/samples/netcore/sample_second_node_pool.yaml b/samples/netcore/sample_second_node_pool.yaml index 7ffde12c..7831be2c 100644 --- a/samples/netcore/sample_second_node_pool.yaml +++ b/samples/netcore/sample_second_node_pool.yaml @@ -18,7 +18,7 @@ spec: portName: gameport # must be the same as the port name described below podSpec: containers: - - image: ghcr.io/playfab/thundernetes-netcore-sample:0.0.1.2 + - image: ghcr.io/playfab/thundernetes-netcore-sample:0.1.0 name: thundernetes-sample-netcore ports: - containerPort: 80 # your game server port From a10194ba4dba082482d11bb0ea73b23f7112c442 Mon Sep 17 00:00:00 2001 From: Dimitris Gkanatsios Date: Wed, 1 Dec 2021 16:46:25 -0800 Subject: [PATCH 3/4] adding pendingCount metric --- operator/config/manager/kustomization.yaml | 28 +++++++++---------- .../controllers/gameserverbuild_controller.go | 25 ++++++++++------- operator/controllers/metrics.go | 1 + 3 files changed, 30 insertions(+), 24 deletions(-) diff --git a/operator/config/manager/kustomization.yaml b/operator/config/manager/kustomization.yaml index 1ad8e90f..4238adff 100755 --- a/operator/config/manager/kustomization.yaml +++ b/operator/config/manager/kustomization.yaml @@ -1,14 +1,14 @@ -resources: -- manager.yaml -generatorOptions: - disableNameSuffixHash: true -configMapGenerator: -- files: - - controller_manager_config.yaml - name: manager-config -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -images: -- name: controller - newName: thundernetes-operator - newTag: b460a56 +resources: +- manager.yaml +generatorOptions: + disableNameSuffixHash: true +configMapGenerator: +- files: + - controller_manager_config.yaml + name: manager-config +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +images: +- name: controller + newName: thundernetes-operator + newTag: b460a56 diff --git a/operator/controllers/gameserverbuild_controller.go b/operator/controllers/gameserverbuild_controller.go index b70d421d..b66b90ca 100644 --- a/operator/controllers/gameserverbuild_controller.go +++ b/operator/controllers/gameserverbuild_controller.go @@ -19,6 +19,7 @@ package controllers import ( "context" "fmt" + "sort" "time" mpsv1alpha1 "github.com/playfab/thundernetes/operator/api/v1alpha1" @@ -149,12 +150,18 @@ func (r *GameServerBuildReconciler) Reconcile(ctx context.Context, req ctrl.Requ } } + // we are sorting GameServers from newest to oldest, since newest have more chances of being in an initializing or pending state + // prioritizing deletion of newest GameServers, if this is needed + sort.SliceStable(gameServers.Items, func(i, j int) bool { + return gameServers.Items[i].GetCreationTimestamp().After(gameServers.Items[j].GetCreationTimestamp().Time) + }) + // user has decreased standingBy numbers if standingByCount > gsb.Spec.StandingBy { deletedCount := 0 for i := 0; i < standingByCount-gsb.Spec.StandingBy; i++ { gs := gameServers.Items[i] - // we're deleting only standingBy/initializing servers + // we're deleting only initializing/pending/standingBy servers, never touching active if gs.Status.State == "" || gs.Status.State == mpsv1alpha1.GameServerStateInitializing || gs.Status.State == mpsv1alpha1.GameServerStateStandingBy { if err := r.Delete(ctx, &gs); err != nil { return ctrl.Result{}, err @@ -166,8 +173,8 @@ func (r *GameServerBuildReconciler) Reconcile(ctx context.Context, req ctrl.Requ } } if deletedCount != standingByCount-gsb.Spec.StandingBy { - log.Info("User modified .Spec.StandingBy - No standingBy servers left to delete") - r.Recorder.Eventf(&gsb, corev1.EventTypeNormal, "User modified .Spec.StandingBy - No standingBy servers left to delete", "Tried to delete %d GameServers but deleted only %d", standingByCount-gsb.Spec.StandingBy, deletedCount) + log.Info("User modified .Spec.StandingBy - No non-active servers left to delete") + r.Recorder.Eventf(&gsb, corev1.EventTypeNormal, "User modified .Spec.StandingBy - No non-active servers left to delete", "Tried to delete %d GameServers but deleted only %d", standingByCount-gsb.Spec.StandingBy, deletedCount) } } @@ -195,9 +202,9 @@ func (r *GameServerBuildReconciler) Reconcile(ctx context.Context, req ctrl.Requ } } + nonActiveGameServersCount := standingByCount + initializingCount + pendingCount // we are in need of standingBy servers, so we're creating them here - for i := 0; i < gsb.Spec.StandingBy-pendingCount-initializingCount-standingByCount && - i+pendingCount+initializingCount+standingByCount+activeCount < gsb.Spec.Max; i++ { + for i := 0; i < gsb.Spec.StandingBy-nonActiveGameServersCount && i+nonActiveGameServersCount+activeCount < gsb.Spec.Max; i++ { newgs, err := NewGameServerForGameServerBuild(&gsb, r.PortRegistry) if err != nil { return ctrl.Result{}, err @@ -229,15 +236,12 @@ func (r *GameServerBuildReconciler) updateStatus(ctx context.Context, gsb *mpsv1 gsb.Status.CrashesCount = gsb.Status.CrashesCount + crashesCount gsb.Status.CurrentStandingByReadyDesired = fmt.Sprintf("%d/%d", standingByCount, gsb.Spec.StandingBy) - var health mpsv1alpha1.GameServerBuildHealth if gsb.Status.CrashesCount >= gsb.Spec.CrashesToMarkUnhealthy { - health = mpsv1alpha1.BuildUnhealthy + gsb.Status.Health = mpsv1alpha1.BuildUnhealthy } else { - health = mpsv1alpha1.BuildHealthy + gsb.Status.Health = mpsv1alpha1.BuildHealthy } - gsb.Status.Health = health - if err := r.Status().Update(ctx, gsb); err != nil { if apierrors.IsConflict(err) { return ctrl.Result{Requeue: true}, nil @@ -247,6 +251,7 @@ func (r *GameServerBuildReconciler) updateStatus(ctx context.Context, gsb *mpsv1 } } + CurrentGameServerGauge.WithLabelValues(gsb.Name, PendingServerStatus).Set(float64(pendingCount)) CurrentGameServerGauge.WithLabelValues(gsb.Name, InitializingServerStatus).Set(float64(initializingCount)) CurrentGameServerGauge.WithLabelValues(gsb.Name, StandingByServerStatus).Set(float64(standingByCount)) CurrentGameServerGauge.WithLabelValues(gsb.Name, ActiveServerStatus).Set(float64(activeCount)) diff --git a/operator/controllers/metrics.go b/operator/controllers/metrics.go index 783ba40e..388d8922 100644 --- a/operator/controllers/metrics.go +++ b/operator/controllers/metrics.go @@ -10,6 +10,7 @@ const ( ActiveServerStatus = "active" StandingByServerStatus = "standingby" InitializingServerStatus = "initializing" + PendingServerStatus = "pending" ) var ( From cd647423ffc898abf658edc70a30465612496940 Mon Sep 17 00:00:00 2001 From: Dimitris-Ilias Gkanatsios Date: Wed, 1 Dec 2021 20:52:19 -0800 Subject: [PATCH 4/4] adding localhost to sidecar http server URL --- sidecar-go/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sidecar-go/main.go b/sidecar-go/main.go index 4de099b5..a20b2420 100644 --- a/sidecar-go/main.go +++ b/sidecar-go/main.go @@ -33,5 +33,5 @@ func main() { http.HandleFunc("/v1/sessionHosts/", sm.heartbeatHandler) - http.ListenAndServe(fmt.Sprintf(":%d", SidecarPort), nil) + http.ListenAndServe(fmt.Sprintf("localhost:%d", SidecarPort), nil) }