From aebc3f13afe06999e8f2ad6dee5370cb7eedd427 Mon Sep 17 00:00:00 2001 From: ttyS3 Date: Sun, 30 Jan 2022 22:36:40 +0800 Subject: [PATCH 1/6] chore: add health check endpoint docs: update document about health check fix: fix up Sqlite3 ping. current ping will success even if the db file is missing fix: do not expose privacy information in output field --- .../doc/installation/on-kubernetes.en-us.md | 44 ++++++ .../doc/installation/on-kubernetes.zh-tw.md | 44 ++++++ modules/cache/cache.go | 38 +++-- routers/web/healthcheck/check.go | 145 ++++++++++++++++++ routers/web/web.go | 4 + 5 files changed, 263 insertions(+), 12 deletions(-) create mode 100644 routers/web/healthcheck/check.go diff --git a/docs/content/doc/installation/on-kubernetes.en-us.md b/docs/content/doc/installation/on-kubernetes.en-us.md index 9fe869254c1d..abfbdf16798b 100644 --- a/docs/content/doc/installation/on-kubernetes.en-us.md +++ b/docs/content/doc/installation/on-kubernetes.en-us.md @@ -25,3 +25,47 @@ helm install gitea gitea-charts/gitea ``` If you would like to customize your install, which includes kubernetes ingress, please refer to the complete [Gitea helm chart configuration details](https://gitea.com/gitea/helm-chart/) + +## Health check endpoint + +Gitea comes with a health check endpoint `/api/healthz`, you can configure it in kubernetes like this: + +```yaml + livenessProbe: + httpGet: + path: /api/healthz + port: http + initialDelaySeconds: 200 + timeoutSeconds: 5 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 10 +``` + +a successful health check response will respond with http code `200`, here's example: + +``` +HTTP/1.1 200 OK + + +{ + "status": "pass", + "description": "Gitea: Git with a cup of tea", + "checks": { + "cache:ping": [ + { + "status": "pass", + "time": "2022-02-19T09:16:08Z" + } + ], + "database:ping": [ + { + "status": "pass", + "time": "2022-02-19T09:16:08Z" + } + ] + } +} +``` + +for more information, please reference to kubernetes documentation [Define a liveness HTTP request](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#define-a-liveness-http-request) diff --git a/docs/content/doc/installation/on-kubernetes.zh-tw.md b/docs/content/doc/installation/on-kubernetes.zh-tw.md index 9add5c4ee1cf..5ea412aa000d 100644 --- a/docs/content/doc/installation/on-kubernetes.zh-tw.md +++ b/docs/content/doc/installation/on-kubernetes.zh-tw.md @@ -25,3 +25,47 @@ helm install gitea gitea-charts/gitea ``` 若您想自訂安裝（包括使用 kubernetes ingress），請前往完整的 [Gitea helm chart configuration details](https://gitea.com/gitea/helm-chart/) + +##運行狀況檢查終端節點 + +Gitea 附帶了一個運行狀況檢查端點 `/api/healthz`，你可以像這樣在 kubernetes 中配置它: + +```yaml + livenessProbe: + httpGet: + path: /api/healthz + port: http + initialDelaySeconds: 200 + timeoutSeconds: 5 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 10 +``` + +成功的運行狀況檢查回應將使用 HTTP 代碼 `200` 進行回應，下面是示例: + +``` +HTTP/1.1 200 OK + + +{ + "status": "pass", + "description": "Gitea: Git with a cup of tea", + "checks": { + "cache:ping": [ + { + "status": "pass", + "time": "2022-02-19T09:16:08Z" + } + ], + "database:ping": [ + { + "status": "pass", + "time": "2022-02-19T09:16:08Z" + } + ] + } +} +``` + +有關更多信息，請參考kubernetes文檔[定義一個存活態 HTTP請求接口]（https://kubernetes.io/zh/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/） diff --git a/modules/cache/cache.go b/modules/cache/cache.go index 0198f8da7375..db1d4b07971e 100644 --- a/modules/cache/cache.go +++ b/modules/cache/cache.go @@ -5,6 +5,7 @@ package cache import ( + "errors" "fmt" "strconv" @@ -34,25 +35,38 @@ func NewContext() error { if conn, err = newCache(setting.CacheService.Cache); err != nil { return err } - const testKey = "__gitea_cache_test" - const testVal = "test-value" - if err = conn.Put(testKey, testVal, 10); err != nil { + err = Ping() + if err != nil { return err } - val := conn.Get(testKey) - if valStr, ok := val.(string); !ok || valStr != testVal { - // If the cache is full, the Get may not read the expected value stored by Put. - // Since we have checked that Put can success, so we just show a warning here, do not return an error to panic. - log.Warn("cache (adapter:%s, config:%s) doesn't seem to work correctly, set test value '%v' but get '%v'", - setting.CacheService.Cache.Adapter, setting.CacheService.Cache.Conn, - testVal, val, - ) - } } return err } +// Ping checks if the cache service works or not, it not, it returns an error +func Ping() error { + if conn == nil { + return errors.New("cache not available") + } + var err error + const testKey = "__gitea_cache_test" + const testVal = "test-value" + if err = conn.Put(testKey, testVal, 10); err != nil { + return err + } + val := conn.Get(testKey) + if valStr, ok := val.(string); !ok || valStr != testVal { + // If the cache is full, the Get may not read the expected value stored by Put. + // Since we have checked that Put can success, so we just show a warning here, do not return an error to panic. + log.Warn("cache (adapter:%s, config:%s) doesn't seem to work correctly, set test value '%v' but get '%v'", + setting.CacheService.Cache.Adapter, setting.CacheService.Cache.Conn, + testVal, val, + ) + } + return nil +} + // GetCache returns the currently configured cache func GetCache() mc.Cache { return conn diff --git a/routers/web/healthcheck/check.go b/routers/web/healthcheck/check.go new file mode 100644 index 000000000000..0db2074e63c6 --- /dev/null +++ b/routers/web/healthcheck/check.go @@ -0,0 +1,145 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package healthcheck + +import ( + "net/http" + "os" + "time" + + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/modules/cache" + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" +) + +type status string + +const ( + // pass healthy (acceptable aliases: "ok" to support Node's Terminus and "up" for Java's SpringBoot) + // fail unhealthy (acceptable aliases: "error" to support Node's Terminus and "down" for Java's SpringBoot), and + // warn healthy, with some concerns. + // + // ref https://datatracker.ietf.org/doc/html/draft-inadarei-api-health-check#section-3.1 + // status: (required) indicates whether the service status is acceptable + // or not. API publishers SHOULD use following values for the field: + // The value of the status field is case-insensitive and is tightly + // related with the HTTP response code returned by the health endpoint. + // For "pass" status, HTTP response code in the 2xx-3xx range MUST be + // used. For "fail" status, HTTP response code in the 4xx-5xx range + // MUST be used. In case of the "warn" status, endpoints MUST return + // HTTP status in the 2xx-3xx range, and additional information SHOULD + // be provided, utilizing optional fields of the response. + pass status = "pass" + fail status = "fail" + warn status = "warn" +) + +func (s status) ToHTTPStatus() int { + if s == pass || s == warn { + return http.StatusOK + } + return http.StatusFailedDependency +} + +type checks map[string][]componentStatus + +// response is the data returned by the health endpoint, which will be marshaled to JSON format +type response struct { + Status status `json:"status"` + Description string `json:"description"` // a human-friendly description of the service + Checks checks `json:"checks"` // The Checks Object +} + +// componentStatus presents one status of a single check object +// an object that provides detailed health statuses of additional downstream systems and endpoints +// which can affect the overall health of the main API. +type componentStatus struct { + Status status `json:"status"` + Time string `json:"time"` // the date-time, in ISO8601 format + Output string `json:"output,omitempty"` // this field SHOULD be omitted for "pass" state. +} + +// HealthChecker implements health check +type HealthChecker struct{} + +// Check is the health check API handler +func (h *HealthChecker) Check(w http.ResponseWriter, r *http.Request) { + rsp := response{ + Status: pass, + Description: "Gitea: Git with a cup of tea", + Checks: make(checks), + } + + statuses := make([]status, 0) + statuses = append(statuses, h.database(rsp.Checks)) + statuses = append(statuses, h.cache(rsp.Checks)) + + for _, s := range statuses { + if s != pass { + rsp.Status = fail + break + } + } + + data, _ := json.MarshalIndent(rsp, "", " ") + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(rsp.Status.ToHTTPStatus()) + _, _ = w.Write(data) +} + +// database checks gitea database status +func (h *HealthChecker) database(checks checks) status { + st := componentStatus{} + if err := db.GetEngine(db.DefaultContext).Ping(); err != nil { + st.Status = fail + st.Time = getCheckTime() + log.Error("database ping failed with error: %v", err) + } else { + st.Status = pass + st.Time = getCheckTime() + } + + if setting.Database.UseSQLite3 && st.Status == pass { + if !setting.EnableSQLite3 { + st.Status = fail + st.Time = getCheckTime() + log.Error("SQLite3 health check failed with error: %v", "this Gitea binary is built without SQLite3 enabled") + } else { + if _, err := os.Stat(setting.Database.Path); err != nil { + st.Status = fail + st.Time = getCheckTime() + log.Error("SQLite3 file exists check failed with error: %v", err) + } + } + } + + checks["database:ping"] = []componentStatus{st} + return st.Status +} + +// cache checks gitea cache status +func (h *HealthChecker) cache(checks checks) status { + if !setting.CacheService.Enabled { + return pass + } + + st := componentStatus{} + if err := cache.Ping(); err != nil { + st.Status = fail + st.Time = getCheckTime() + log.Error("cache ping failed with error: %v", err) + } else { + st.Status = pass + st.Time = getCheckTime() + } + checks["cache:ping"] = []componentStatus{st} + return st.Status +} + +func getCheckTime() string { + return time.Now().UTC().Format(time.RFC3339) +} diff --git a/routers/web/web.go b/routers/web/web.go index d8c197fb967e..44e6d73629fa 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -29,6 +29,7 @@ import ( "code.gitea.io/gitea/routers/web/dev" "code.gitea.io/gitea/routers/web/events" "code.gitea.io/gitea/routers/web/explore" + "code.gitea.io/gitea/routers/web/healthcheck" "code.gitea.io/gitea/routers/web/org" "code.gitea.io/gitea/routers/web/repo" "code.gitea.io/gitea/routers/web/user" @@ -150,6 +151,9 @@ func Routes(sessioner func(http.Handler) http.Handler) *web.Route { rw.WriteHeader(200) }) + hc := &healthcheck.HealthChecker{} + routes.Get("/api/healthz", hc.Check) + // Removed: toolbox.Toolboxer middleware will provide debug information which seems unnecessary common = append(common, context.Contexter()) From e8b26b874b95119869138c3fb77fa4f73b6f9317 Mon Sep 17 00:00:00 2001 From: ttyS3 Date: Wed, 23 Feb 2022 08:23:01 +0800 Subject: [PATCH 2/6] refactor: remove HealthChecker struct --- routers/web/healthcheck/check.go | 13 +++++-------- routers/web/web.go | 3 +-- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/routers/web/healthcheck/check.go b/routers/web/healthcheck/check.go index 0db2074e63c6..e0952d92ce0d 100644 --- a/routers/web/healthcheck/check.go +++ b/routers/web/healthcheck/check.go @@ -63,11 +63,8 @@ type componentStatus struct { Output string `json:"output,omitempty"` // this field SHOULD be omitted for "pass" state. } -// HealthChecker implements health check -type HealthChecker struct{} - // Check is the health check API handler -func (h *HealthChecker) Check(w http.ResponseWriter, r *http.Request) { +func Check(w http.ResponseWriter, r *http.Request) { rsp := response{ Status: pass, Description: "Gitea: Git with a cup of tea", @@ -75,8 +72,8 @@ func (h *HealthChecker) Check(w http.ResponseWriter, r *http.Request) { } statuses := make([]status, 0) - statuses = append(statuses, h.database(rsp.Checks)) - statuses = append(statuses, h.cache(rsp.Checks)) + statuses = append(statuses, checkDatabase(rsp.Checks)) + statuses = append(statuses, checkCache(rsp.Checks)) for _, s := range statuses { if s != pass { @@ -92,7 +89,7 @@ func (h *HealthChecker) Check(w http.ResponseWriter, r *http.Request) { } // database checks gitea database status -func (h *HealthChecker) database(checks checks) status { +func checkDatabase(checks checks) status { st := componentStatus{} if err := db.GetEngine(db.DefaultContext).Ping(); err != nil { st.Status = fail @@ -122,7 +119,7 @@ func (h *HealthChecker) database(checks checks) status { } // cache checks gitea cache status -func (h *HealthChecker) cache(checks checks) status { +func checkCache(checks checks) status { if !setting.CacheService.Enabled { return pass } diff --git a/routers/web/web.go b/routers/web/web.go index 44e6d73629fa..3fe0c469978c 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -151,8 +151,7 @@ func Routes(sessioner func(http.Handler) http.Handler) *web.Route { rw.WriteHeader(200) }) - hc := &healthcheck.HealthChecker{} - routes.Get("/api/healthz", hc.Check) + routes.Get("/api/healthz", healthcheck.Check) // Removed: toolbox.Toolboxer middleware will provide debug information which seems unnecessary common = append(common, context.Contexter()) From 1515a840843778a6a2b0767000dda37d6bd354a5 Mon Sep 17 00:00:00 2001 From: Marcos de Oliveira Date: Tue, 12 Apr 2022 12:46:24 -0300 Subject: [PATCH 3/6] Added `/api/healthz` to install routes. This was needed for using /api/healthz endpoint in Docker healthchecks, otherwise, Docker would never become healthy if using healthz endpoint and users would not be able to complete the installation of Gitea. --- routers/install/routes.go | 2 ++ routers/web/healthcheck/check.go | 17 +++++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/routers/install/routes.go b/routers/install/routes.go index ef96e99628ef..41e4d0fcebcb 100644 --- a/routers/install/routes.go +++ b/routers/install/routes.go @@ -16,6 +16,7 @@ import ( "code.gitea.io/gitea/modules/web" "code.gitea.io/gitea/modules/web/middleware" "code.gitea.io/gitea/routers/common" + "code.gitea.io/gitea/routers/web/healthcheck" "code.gitea.io/gitea/services/forms" "gitea.com/go-chi/session" @@ -106,6 +107,7 @@ func Routes() *web.Route { r.Use(Init) r.Get("/", Install) r.Post("/", web.Bind(forms.InstallForm{}), SubmitInstall) + r.Get("/api/healthz", healthcheck.CheckInstall) r.NotFound(web.Wrap(installNotFound)) return r diff --git a/routers/web/healthcheck/check.go b/routers/web/healthcheck/check.go index e0952d92ce0d..129de866a3eb 100644 --- a/routers/web/healthcheck/check.go +++ b/routers/web/healthcheck/check.go @@ -50,8 +50,8 @@ type checks map[string][]componentStatus // response is the data returned by the health endpoint, which will be marshaled to JSON format type response struct { Status status `json:"status"` - Description string `json:"description"` // a human-friendly description of the service - Checks checks `json:"checks"` // The Checks Object + Description string `json:"description"` // a human-friendly description of the service + Checks checks `json:"checks,omitempty"` // The Checks Object, should be omitted on installation route } // componentStatus presents one status of a single check object @@ -88,6 +88,19 @@ func Check(w http.ResponseWriter, r *http.Request) { _, _ = w.Write(data) } +// CheckInstall always return pass. Should only be used in Install routes +func CheckInstall(w http.ResponseWriter, r *http.Request) { + rsp := response{ + Status: pass, + Description: "Gitea: Installation stage", + } + + data, _ := json.MarshalIndent(rsp, "", " ") + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(rsp.Status.ToHTTPStatus()) + _, _ = w.Write(data) +} + // database checks gitea database status func checkDatabase(checks checks) status { st := componentStatus{} From ddb09410f74b9ae82f3c6e40346aeadd9e555a50 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Wed, 4 May 2022 11:56:26 +0800 Subject: [PATCH 4/6] Update modules/cache/cache.go --- modules/cache/cache.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/cache/cache.go b/modules/cache/cache.go index db1d4b07971e..fd32aa153b01 100644 --- a/modules/cache/cache.go +++ b/modules/cache/cache.go @@ -35,8 +35,7 @@ func NewContext() error { if conn, err = newCache(setting.CacheService.Cache); err != nil { return err } - err = Ping() - if err != nil { + if err = Ping(); err != nil { return err } } From 0e988e5d0d1b7cf45a8ae8027366dc4681a2f382 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Wed, 4 May 2022 12:10:37 +0800 Subject: [PATCH 5/6] fine tune --- routers/install/routes.go | 2 +- routers/web/healthcheck/check.go | 22 +++++----------------- routers/web/web.go | 2 +- 3 files changed, 7 insertions(+), 19 deletions(-) diff --git a/routers/install/routes.go b/routers/install/routes.go index 41e4d0fcebcb..e77081afe0a8 100644 --- a/routers/install/routes.go +++ b/routers/install/routes.go @@ -107,7 +107,7 @@ func Routes() *web.Route { r.Use(Init) r.Get("/", Install) r.Post("/", web.Bind(forms.InstallForm{}), SubmitInstall) - r.Get("/api/healthz", healthcheck.CheckInstall) + r.Get("/api/healthz", healthcheck.Check) r.NotFound(web.Wrap(installNotFound)) return r diff --git a/routers/web/healthcheck/check.go b/routers/web/healthcheck/check.go index 129de866a3eb..481f05c0da7e 100644 --- a/routers/web/healthcheck/check.go +++ b/routers/web/healthcheck/check.go @@ -67,14 +67,15 @@ type componentStatus struct { func Check(w http.ResponseWriter, r *http.Request) { rsp := response{ Status: pass, - Description: "Gitea: Git with a cup of tea", + Description: setting.AppName, Checks: make(checks), } statuses := make([]status, 0) - statuses = append(statuses, checkDatabase(rsp.Checks)) - statuses = append(statuses, checkCache(rsp.Checks)) - + if setting.InstallLock { + statuses = append(statuses, checkDatabase(rsp.Checks)) + statuses = append(statuses, checkCache(rsp.Checks)) + } for _, s := range statuses { if s != pass { rsp.Status = fail @@ -88,19 +89,6 @@ func Check(w http.ResponseWriter, r *http.Request) { _, _ = w.Write(data) } -// CheckInstall always return pass. Should only be used in Install routes -func CheckInstall(w http.ResponseWriter, r *http.Request) { - rsp := response{ - Status: pass, - Description: "Gitea: Installation stage", - } - - data, _ := json.MarshalIndent(rsp, "", " ") - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(rsp.Status.ToHTTPStatus()) - _, _ = w.Write(data) -} - // database checks gitea database status func checkDatabase(checks checks) status { st := componentStatus{} diff --git a/routers/web/web.go b/routers/web/web.go index e40807faa4cf..dcaad3d2bde1 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -31,8 +31,8 @@ import ( "code.gitea.io/gitea/routers/web/events" "code.gitea.io/gitea/routers/web/explore" "code.gitea.io/gitea/routers/web/feed" - "code.gitea.io/gitea/routers/web/misc" "code.gitea.io/gitea/routers/web/healthcheck" + "code.gitea.io/gitea/routers/web/misc" "code.gitea.io/gitea/routers/web/org" "code.gitea.io/gitea/routers/web/repo" "code.gitea.io/gitea/routers/web/user" From 7ad0aa3481f0ba25406c71faaa825dc2eed99bbc Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Wed, 4 May 2022 12:56:05 +0800 Subject: [PATCH 6/6] Remove unnecessary test code. Now there are 2 routes for installation (and maybe more in future) --- routers/install/routes_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/routers/install/routes_test.go b/routers/install/routes_test.go index 35a66c1c4742..29003c3841be 100644 --- a/routers/install/routes_test.go +++ b/routers/install/routes_test.go @@ -13,7 +13,6 @@ import ( func TestRoutes(t *testing.T) { routes := Routes() assert.NotNil(t, routes) - assert.Len(t, routes.R.Routes(), 1) assert.EqualValues(t, "/", routes.R.Routes()[0].Pattern) assert.Nil(t, routes.R.Routes()[0].SubRoutes) assert.Len(t, routes.R.Routes()[0].Handlers, 2)