From 2cd46ad69070b3a7197aa708ae200ee9bf907ac1 Mon Sep 17 00:00:00 2001 From: Bo-Yi Wu Date: Tue, 13 Jul 2021 23:58:47 +0800 Subject: [PATCH] chore: Add metric and status package. (#585) --- gorush/global.go | 3 -- gorush/main_test.go | 7 +++- gorush/notification_apns.go | 5 +-- gorush/notification_apns_test.go | 5 +-- gorush/notification_fcm.go | 9 ++--- gorush/notification_hms.go | 5 +-- gorush/server.go | 38 +++++++++++++++++++- gorush/worker.go | 3 +- main.go | 11 +++--- {gorush => metric}/metrics.go | 34 ++++++++++++------ metric/metrics_test.go | 15 ++++++++ {gorush => status}/status.go | 59 +++++++++---------------------- {gorush => status}/status_test.go | 42 +++++++++++++++++----- storage/memory/memory.go | 2 +- 14 files changed, 154 insertions(+), 84 deletions(-) rename {gorush => metric}/metrics.go (79%) create mode 100644 metric/metrics_test.go rename {gorush => status}/status.go (54%) rename {gorush => status}/status_test.go (86%) diff --git a/gorush/global.go b/gorush/global.go index 8728d60..1f51c43 100644 --- a/gorush/global.go +++ b/gorush/global.go @@ -2,7 +2,6 @@ package gorush import ( "github.com/appleboy/gorush/config" - "github.com/appleboy/gorush/storage" "github.com/appleboy/go-fcm" "github.com/msalihkarakasli/go-hms-push/push/core" @@ -20,8 +19,6 @@ var ( FCMClient *fcm.Client // HMSClient is Huawei push client HMSClient *core.HMSClient - // StatStorage implements the storage interface - StatStorage storage.Storage // MaxConcurrentIOSPushes pool to limit the number of concurrent iOS pushes MaxConcurrentIOSPushes chan struct{} ) diff --git a/gorush/main_test.go b/gorush/main_test.go index 2948b4e..81e9cd0 100644 --- a/gorush/main_test.go +++ b/gorush/main_test.go @@ -8,6 +8,7 @@ import ( "github.com/appleboy/gorush/config" "github.com/appleboy/gorush/logx" + "github.com/appleboy/gorush/status" ) func TestMain(m *testing.M) { @@ -21,12 +22,16 @@ func TestMain(m *testing.M) { log.Fatal(err) } + if err := status.InitAppStatus(PushConf); err != nil { + log.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) wg := &sync.WaitGroup{} wg.Add(int(PushConf.Core.WorkerNum)) InitWorkers(ctx, wg, PushConf.Core.WorkerNum, PushConf.Core.QueueNum) - if err := InitAppStatus(); err != nil { + if err := status.InitAppStatus(PushConf); err != nil { log.Fatal(err) } diff --git a/gorush/notification_apns.go b/gorush/notification_apns.go index 9b2ce07..1952ce1 100644 --- a/gorush/notification_apns.go +++ b/gorush/notification_apns.go @@ -13,6 +13,7 @@ import ( "github.com/appleboy/gorush/core" "github.com/appleboy/gorush/logx" + "github.com/appleboy/gorush/status" "github.com/mitchellh/mapstructure" "github.com/sideshow/apns2" @@ -428,7 +429,7 @@ Retry: }(logx.LogError, createLogPushEntry(core.FailedPush, token, req, err), PushConf.Core.FeedbackURL, PushConf.Core.FeedbackTimeout) } - StatStorage.AddIosError(1) + status.StatStorage.AddIosError(1) // We should retry only "retryable" statuses. More info about response: // https://developer.apple.com/documentation/usernotifications/setting_up_a_remote_notification_server/handling_notification_responses_from_apns if res != nil && res.StatusCode >= http.StatusInternalServerError { @@ -438,7 +439,7 @@ Retry: if res != nil && res.Sent() { logPush(core.SucceededPush, token, req, nil) - StatStorage.AddIosSuccess(1) + status.StatStorage.AddIosSuccess(1) } // free push slot <-MaxConcurrentIOSPushes diff --git a/gorush/notification_apns_test.go b/gorush/notification_apns_test.go index ff2e371..eee596b 100644 --- a/gorush/notification_apns_test.go +++ b/gorush/notification_apns_test.go @@ -12,6 +12,7 @@ import ( "github.com/appleboy/gorush/config" "github.com/appleboy/gorush/core" + "github.com/appleboy/gorush/status" "github.com/buger/jsonparser" "github.com/sideshow/apns2" "github.com/stretchr/testify/assert" @@ -761,7 +762,7 @@ func TestPushToIOS(t *testing.T) { PushConf.Ios.KeyPath = "../certificate/certificate-valid.pem" err := InitAPNSClient() assert.Nil(t, err) - err = InitAppStatus() + err = status.InitAppStatus(PushConf) assert.Nil(t, err) req := PushNotification{ @@ -781,7 +782,7 @@ func TestApnsHostFromRequest(t *testing.T) { PushConf.Ios.KeyPath = "../certificate/certificate-valid.pem" err := InitAPNSClient() assert.Nil(t, err) - err = InitAppStatus() + err = status.InitAppStatus(PushConf) assert.Nil(t, err) req := PushNotification{ diff --git a/gorush/notification_fcm.go b/gorush/notification_fcm.go index 067cfb8..fe65b64 100644 --- a/gorush/notification_fcm.go +++ b/gorush/notification_fcm.go @@ -7,6 +7,7 @@ import ( "github.com/appleboy/go-fcm" "github.com/appleboy/gorush/core" "github.com/appleboy/gorush/logx" + "github.com/appleboy/gorush/status" "github.com/sirupsen/logrus" ) @@ -154,7 +155,7 @@ Retry: } }(logx.LogError, createLogPushEntry(core.FailedPush, req.To, req, err), PushConf.Core.FeedbackURL, PushConf.Core.FeedbackTimeout) } - StatStorage.AddAndroidError(1) + status.StatStorage.AddAndroidError(1) } else { for _, token := range req.Tokens { if PushConf.Core.Sync { @@ -168,7 +169,7 @@ Retry: }(logx.LogError, createLogPushEntry(core.FailedPush, token, req, err), PushConf.Core.FeedbackURL, PushConf.Core.FeedbackTimeout) } } - StatStorage.AddAndroidError(int64(len(req.Tokens))) + status.StatStorage.AddAndroidError(int64(len(req.Tokens))) } return } @@ -177,8 +178,8 @@ Retry: logx.LogAccess.Debug(fmt.Sprintf("Android Success count: %d, Failure count: %d", res.Success, res.Failure)) } - StatStorage.AddAndroidSuccess(int64(res.Success)) - StatStorage.AddAndroidError(int64(res.Failure)) + status.StatStorage.AddAndroidSuccess(int64(res.Success)) + status.StatStorage.AddAndroidError(int64(res.Failure)) var newTokens []string // result from Send messages to specific devices diff --git a/gorush/notification_hms.go b/gorush/notification_hms.go index 22a36c1..3fb03f7 100644 --- a/gorush/notification_hms.go +++ b/gorush/notification_hms.go @@ -7,6 +7,7 @@ import ( "sync" "github.com/appleboy/gorush/logx" + "github.com/appleboy/gorush/status" "github.com/msalihkarakasli/go-hms-push/push/config" "github.com/msalihkarakasli/go-hms-push/push/core" @@ -206,11 +207,11 @@ Retry: // Huawei Push Send API does not support exact results for each token if res.Code == "80000000" { - StatStorage.AddHuaweiSuccess(int64(1)) + status.StatStorage.AddHuaweiSuccess(int64(1)) logx.LogAccess.Debug("Huwaei Send Notification is completed successfully!") } else { isError = true - StatStorage.AddHuaweiError(int64(1)) + status.StatStorage.AddHuaweiError(int64(1)) logx.LogAccess.Debug("Huawei Send Notification is failed! Code: " + res.Code) } diff --git a/gorush/server.go b/gorush/server.go index 841977b..ea784fc 100644 --- a/gorush/server.go +++ b/gorush/server.go @@ -7,6 +7,9 @@ import ( "net/http" "os" + "github.com/appleboy/gorush/metric" + "github.com/appleboy/gorush/status" + api "github.com/appleboy/gin-status-api" "github.com/appleboy/gorush/logx" "github.com/gin-contrib/logger" @@ -17,6 +20,7 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/rs/zerolog" "github.com/rs/zerolog/log" + "github.com/thoas/stats" "golang.org/x/crypto/acme/autocert" ) @@ -24,7 +28,9 @@ var isTerm bool func init() { // Support metrics - m := NewMetrics() + m := metric.NewMetrics(func() int { + return len(QueueNotification) + }) prometheus.MustRegister(m) isTerm = isatty.IsTerminal(os.Stdout.Fd()) } @@ -109,6 +115,36 @@ func metricsHandler(c *gin.Context) { promhttp.Handler().ServeHTTP(c.Writer, c.Request) } +func appStatusHandler(c *gin.Context) { + result := status.App{} + + result.Version = GetVersion() + result.QueueMax = cap(QueueNotification) + result.QueueUsage = len(QueueNotification) + result.TotalCount = status.StatStorage.GetTotalCount() + result.Ios.PushSuccess = status.StatStorage.GetIosSuccess() + result.Ios.PushError = status.StatStorage.GetIosError() + result.Android.PushSuccess = status.StatStorage.GetAndroidSuccess() + result.Android.PushError = status.StatStorage.GetAndroidError() + result.Huawei.PushSuccess = status.StatStorage.GetHuaweiSuccess() + result.Huawei.PushError = status.StatStorage.GetHuaweiError() + + c.JSON(http.StatusOK, result) +} + +func sysStatsHandler(c *gin.Context) { + c.JSON(http.StatusOK, status.Stats.Data()) +} + +// StatMiddleware response time, status code count, etc. +func StatMiddleware() gin.HandlerFunc { + return func(c *gin.Context) { + beginning, recorder := status.Stats.Begin(c.Writer) + c.Next() + status.Stats.End(beginning, stats.WithRecorder(recorder)) + } +} + func autoTLSServer() *http.Server { m := autocert.Manager{ Prompt: autocert.AcceptTOS, diff --git a/gorush/worker.go b/gorush/worker.go index e3b8887..fd23e19 100644 --- a/gorush/worker.go +++ b/gorush/worker.go @@ -7,6 +7,7 @@ import ( "github.com/appleboy/gorush/core" "github.com/appleboy/gorush/logx" + "github.com/appleboy/gorush/status" ) // InitWorkers for initialize all workers. @@ -105,7 +106,7 @@ func queueNotification(ctx context.Context, req RequestPush) (int, []logx.LogPus wg.Wait() } - StatStorage.AddTotalCount(int64(count)) + status.StatStorage.AddTotalCount(int64(count)) return count, log } diff --git a/main.go b/main.go index 5f6d207..afc0332 100644 --- a/main.go +++ b/main.go @@ -20,6 +20,7 @@ import ( "github.com/appleboy/gorush/gorush" "github.com/appleboy/gorush/logx" "github.com/appleboy/gorush/rpc" + "github.com/appleboy/gorush/status" "golang.org/x/sync/errgroup" ) @@ -212,7 +213,7 @@ func main() { logx.LogError.Fatal(err) } - if err := gorush.InitAppStatus(); err != nil { + if err := status.InitAppStatus(gorush.PushConf); err != nil { return } @@ -245,7 +246,7 @@ func main() { logx.LogError.Fatal(err) } - if err := gorush.InitAppStatus(); err != nil { + if err := status.InitAppStatus(gorush.PushConf); err != nil { return } @@ -282,7 +283,7 @@ func main() { logx.LogError.Fatal(err) } - if err := gorush.InitAppStatus(); err != nil { + if err := status.InitAppStatus(gorush.PushConf); err != nil { return } @@ -308,7 +309,7 @@ func main() { logx.LogError.Fatal(err) } - if err = gorush.InitAppStatus(); err != nil { + if err = status.InitAppStatus(gorush.PushConf); err != nil { logx.LogError.Fatal(err) } @@ -323,7 +324,7 @@ func main() { close(finished) // close the connection with storage logx.LogAccess.Info("close the storage connection: ", gorush.PushConf.Stat.Engine) - if err := gorush.StatStorage.Close(); err != nil { + if err := status.StatStorage.Close(); err != nil { logx.LogError.Fatal("can't close the storage connection: ", err.Error()) } }) diff --git a/gorush/metrics.go b/metric/metrics.go similarity index 79% rename from gorush/metrics.go rename to metric/metrics.go index 0f47053..e717ef0 100644 --- a/gorush/metrics.go +++ b/metric/metrics.go @@ -1,6 +1,8 @@ -package gorush +package metric import ( + "github.com/appleboy/gorush/status" + "github.com/prometheus/client_golang/prometheus" ) @@ -17,11 +19,14 @@ type Metrics struct { HuaweiSuccess *prometheus.Desc HuaweiError *prometheus.Desc QueueUsage *prometheus.Desc + GetQueueUsage func() int } +var getGetQueueUsage = func() int { return 0 } + // NewMetrics returns a new Metrics with all prometheus.Desc initialized -func NewMetrics() Metrics { - return Metrics{ +func NewMetrics(c ...func() int) Metrics { + m := Metrics{ TotalPushCount: prometheus.NewDesc( namespace+"total_push_count", "Number of push count", @@ -62,7 +67,14 @@ func NewMetrics() Metrics { "Length of internal queue", nil, nil, ), + GetQueueUsage: getGetQueueUsage, } + + if len(c) > 0 { + m.GetQueueUsage = c[0] + } + + return m } // Describe returns all possible prometheus.Desc @@ -82,41 +94,41 @@ func (c Metrics) Collect(ch chan<- prometheus.Metric) { ch <- prometheus.MustNewConstMetric( c.TotalPushCount, prometheus.CounterValue, - float64(StatStorage.GetTotalCount()), + float64(status.StatStorage.GetTotalCount()), ) ch <- prometheus.MustNewConstMetric( c.IosSuccess, prometheus.CounterValue, - float64(StatStorage.GetIosSuccess()), + float64(status.StatStorage.GetIosSuccess()), ) ch <- prometheus.MustNewConstMetric( c.IosError, prometheus.CounterValue, - float64(StatStorage.GetIosError()), + float64(status.StatStorage.GetIosError()), ) ch <- prometheus.MustNewConstMetric( c.AndroidSuccess, prometheus.CounterValue, - float64(StatStorage.GetAndroidSuccess()), + float64(status.StatStorage.GetAndroidSuccess()), ) ch <- prometheus.MustNewConstMetric( c.AndroidError, prometheus.CounterValue, - float64(StatStorage.GetAndroidError()), + float64(status.StatStorage.GetAndroidError()), ) ch <- prometheus.MustNewConstMetric( c.HuaweiSuccess, prometheus.CounterValue, - float64(StatStorage.GetHuaweiSuccess()), + float64(status.StatStorage.GetHuaweiSuccess()), ) ch <- prometheus.MustNewConstMetric( c.HuaweiError, prometheus.CounterValue, - float64(StatStorage.GetHuaweiError()), + float64(status.StatStorage.GetHuaweiError()), ) ch <- prometheus.MustNewConstMetric( c.QueueUsage, prometheus.GaugeValue, - float64(len(QueueNotification)), + float64(c.GetQueueUsage()), ) } diff --git a/metric/metrics_test.go b/metric/metrics_test.go new file mode 100644 index 0000000..2e6c1b2 --- /dev/null +++ b/metric/metrics_test.go @@ -0,0 +1,15 @@ +package metric + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewMetrics(t *testing.T) { + m := NewMetrics() + assert.Equal(t, 0, m.GetQueueUsage()) + + m = NewMetrics(func() int { return 1 }) + assert.Equal(t, 1, m.GetQueueUsage()) +} diff --git a/gorush/status.go b/status/status.go similarity index 54% rename from gorush/status.go rename to status/status.go index 4e99a66..518d6c7 100644 --- a/gorush/status.go +++ b/status/status.go @@ -1,10 +1,11 @@ -package gorush +package status import ( "errors" - "net/http" + "github.com/appleboy/gorush/config" "github.com/appleboy/gorush/logx" + "github.com/appleboy/gorush/storage" "github.com/appleboy/gorush/storage/badger" "github.com/appleboy/gorush/storage/boltdb" "github.com/appleboy/gorush/storage/buntdb" @@ -12,15 +13,17 @@ import ( "github.com/appleboy/gorush/storage/memory" "github.com/appleboy/gorush/storage/redis" - "github.com/gin-gonic/gin" "github.com/thoas/stats" ) // Stats provide response time, status code count, etc. var Stats = stats.New() -// StatusApp is app status structure -type StatusApp struct { +// StatStorage implements the storage interface +var StatStorage storage.Storage + +// App is status structure +type App struct { Version string `json:"version"` QueueMax int `json:"queue_max"` QueueUsage int `json:"queue_usage"` @@ -49,21 +52,21 @@ type HuaweiStatus struct { } // InitAppStatus for initialize app status -func InitAppStatus() error { - logx.LogAccess.Info("Init App Status Engine as ", PushConf.Stat.Engine) - switch PushConf.Stat.Engine { +func InitAppStatus(conf config.ConfYaml) error { + logx.LogAccess.Info("Init App Status Engine as ", conf.Stat.Engine) + switch conf.Stat.Engine { case "memory": StatStorage = memory.New() case "redis": - StatStorage = redis.New(PushConf) + StatStorage = redis.New(conf) case "boltdb": - StatStorage = boltdb.New(PushConf) + StatStorage = boltdb.New(conf) case "buntdb": - StatStorage = buntdb.New(PushConf) + StatStorage = buntdb.New(conf) case "leveldb": - StatStorage = leveldb.New(PushConf) + StatStorage = leveldb.New(conf) case "badger": - StatStorage = badger.New(PushConf) + StatStorage = badger.New(conf) default: logx.LogError.Error("storage error: can't find storage driver") return errors.New("can't find storage driver") @@ -77,33 +80,3 @@ func InitAppStatus() error { return nil } - -func appStatusHandler(c *gin.Context) { - result := StatusApp{} - - result.Version = GetVersion() - result.QueueMax = cap(QueueNotification) - result.QueueUsage = len(QueueNotification) - result.TotalCount = StatStorage.GetTotalCount() - result.Ios.PushSuccess = StatStorage.GetIosSuccess() - result.Ios.PushError = StatStorage.GetIosError() - result.Android.PushSuccess = StatStorage.GetAndroidSuccess() - result.Android.PushError = StatStorage.GetAndroidError() - result.Huawei.PushSuccess = StatStorage.GetHuaweiSuccess() - result.Huawei.PushError = StatStorage.GetHuaweiError() - - c.JSON(http.StatusOK, result) -} - -func sysStatsHandler(c *gin.Context) { - c.JSON(http.StatusOK, Stats.Data()) -} - -// StatMiddleware response time, status code count, etc. -func StatMiddleware() gin.HandlerFunc { - return func(c *gin.Context) { - beginning, recorder := Stats.Begin(c.Writer) - c.Next() - Stats.End(beginning, stats.WithRecorder(recorder)) - } -} diff --git a/gorush/status_test.go b/status/status_test.go similarity index 86% rename from gorush/status_test.go rename to status/status_test.go index 3f59f4a..589eacc 100644 --- a/gorush/status_test.go +++ b/status/status_test.go @@ -1,15 +1,34 @@ -package gorush +package status import ( + "log" "testing" "time" + "github.com/appleboy/gorush/config" + "github.com/appleboy/gorush/logx" + "github.com/stretchr/testify/assert" ) +func TestMain(m *testing.M) { + PushConf, _ := config.LoadConf("") + if err := logx.InitLog( + PushConf.Log.AccessLevel, + PushConf.Log.AccessLog, + PushConf.Log.ErrorLevel, + PushConf.Log.ErrorLog, + ); err != nil { + log.Fatal(err) + } + + m.Run() +} + func TestStorageDriverExist(t *testing.T) { + PushConf, _ := config.LoadConf("") PushConf.Stat.Engine = "Test" - err := InitAppStatus() + err := InitAppStatus(PushConf) assert.Error(t, err) } @@ -18,8 +37,9 @@ func TestStatForMemoryEngine(t *testing.T) { time.Sleep(5 * time.Second) var val int64 + PushConf, _ := config.LoadConf("") PushConf.Stat.Engine = "memory" - err := InitAppStatus() + err := InitAppStatus(PushConf) assert.Nil(t, err) StatStorage.AddTotalCount(100) @@ -41,28 +61,31 @@ func TestStatForMemoryEngine(t *testing.T) { } func TestRedisServerSuccess(t *testing.T) { + PushConf, _ := config.LoadConf("") PushConf.Stat.Engine = "redis" PushConf.Stat.Redis.Addr = "redis:6379" - err := InitAppStatus() + err := InitAppStatus(PushConf) assert.NoError(t, err) } func TestRedisServerError(t *testing.T) { + PushConf, _ := config.LoadConf("") PushConf.Stat.Engine = "redis" PushConf.Stat.Redis.Addr = "redis:6370" - err := InitAppStatus() + err := InitAppStatus(PushConf) assert.Error(t, err) } func TestStatForRedisEngine(t *testing.T) { var val int64 + PushConf, _ := config.LoadConf("") PushConf.Stat.Engine = "redis" PushConf.Stat.Redis.Addr = "redis:6379" - err := InitAppStatus() + err := InitAppStatus(PushConf) assert.Nil(t, err) StatStorage.Init() @@ -89,7 +112,8 @@ func TestStatForRedisEngine(t *testing.T) { func TestDefaultEngine(t *testing.T) { var val int64 // defaul engine as memory - err := InitAppStatus() + PushConf, _ := config.LoadConf("") + err := InitAppStatus(PushConf) assert.Nil(t, err) StatStorage.Reset() @@ -114,8 +138,10 @@ func TestDefaultEngine(t *testing.T) { func TestStatForBoltDBEngine(t *testing.T) { var val int64 + PushConf, _ := config.LoadConf("") PushConf.Stat.Engine = "boltdb" - InitAppStatus() + err := InitAppStatus(PushConf) + assert.Nil(t, err) StatStorage.Reset() diff --git a/storage/memory/memory.go b/storage/memory/memory.go index 7c0e961..4b3c5ff 100644 --- a/storage/memory/memory.go +++ b/storage/memory/memory.go @@ -4,7 +4,7 @@ import ( "sync/atomic" ) -// StatusApp is app status structure +// statApp is app status structure type statApp struct { TotalCount int64 `json:"total_count"` Ios IosStatus `json:"ios"`