From aae223a44f064d69a1636079db4fe169d117b09d Mon Sep 17 00:00:00 2001 From: Fabian Vowie Date: Sun, 23 Jan 2022 18:07:34 +0100 Subject: [PATCH 1/2] Add structured logging --- go.mod | 5 +++-- go.sum | 9 ++++++++- main.go | 19 +++++++++++++++---- pipelines/pipeline.go | 6 +++--- 4 files changed, 29 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index fd0e86d..3bc41c2 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/bxcodec/faker/v3 v3.7.0 github.com/disintegration/imaging v1.6.2 github.com/gorilla/mux v1.8.0 + github.com/sirupsen/logrus v1.8.1 github.com/spf13/afero v1.8.0 github.com/stretchr/testify v1.7.0 github.com/throttled/throttled v2.2.5+incompatible @@ -13,13 +14,13 @@ require ( ) require ( - github.com/davecgh/go-spew v1.1.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/kr/pretty v0.3.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rogpeppe/go-internal v1.8.1-0.20211023094830-115ce09fd6b4 // indirect golang.org/x/image v0.0.0-20211028202545-6944b10bf410 // indirect + golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 // indirect golang.org/x/text v0.3.6 // indirect - gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect ) diff --git a/go.sum b/go.sum index a64e514..b43b423 100644 --- a/go.sum +++ b/go.sum @@ -49,8 +49,9 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -89,6 +90,7 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/gomodule/redigo v1.8.4 h1:Z5JUg94HMTR1XpwBaSH4vq3+PNSIykBLxMdglbw10gg= github.com/gomodule/redigo v1.8.4/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -152,9 +154,12 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.1-0.20211023094830-115ce09fd6b4 h1:Ha8xCaq6ln1a+R91Km45Oq6lPXj2Mla6CRJYcuV2h1w= github.com/rogpeppe/go-internal v1.8.1-0.20211023094830-115ce09fd6b4/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/spf13/afero v1.8.0 h1:5MmtuhAgYeU6qpa7w7bP0dv6MBYuup0vekhSpSkoq60= github.com/spf13/afero v1.8.0/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= @@ -280,6 +285,7 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -303,6 +309,7 @@ golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/main.go b/main.go index cac5e74..c34f205 100644 --- a/main.go +++ b/main.go @@ -5,12 +5,14 @@ import ( "encoding/json" "io" "net/http" + "os" "github.com/geplauder/lithium/middlewares" "github.com/geplauder/lithium/pipelines" "github.com/geplauder/lithium/settings" "github.com/geplauder/lithium/storage" "github.com/gorilla/mux" + "github.com/sirupsen/logrus" "github.com/spf13/afero" ) @@ -124,6 +126,8 @@ func UploadHandler(w http.ResponseWriter, r *http.Request, pipes []pipelines.IPi if err != nil { w.WriteHeader(http.StatusInternalServerError) } + + logrus.Info("Pipeline routes registered successfully") } func RegisterRoutes(r *mux.Router, pipelines []pipelines.IPipeline, storageProvider storage.IStorageProvider) { @@ -144,9 +148,12 @@ func RegisterRoutes(r *mux.Router, pipelines []pipelines.IPipeline, storageProvi } func main() { + logrus.SetFormatter(&logrus.JSONFormatter{}) + logrus.SetOutput(os.Stdout) + appSettings, err := settings.LoadSettings(afero.NewOsFs()) if err != nil { - panic(err) + logrus.Fatal("Unexpected error while loading settings: ", err) } var storageProvider storage.IStorageProvider @@ -154,7 +161,9 @@ func main() { if appSettings.StorageProvider.Type == 0 { storageProvider = storage.GetFileSystemStorageProvider(appSettings.StorageProvider.BasePath, "") } else { - panic("Invalid file system provided!") + logrus.WithFields(logrus.Fields{ + "StorageProviderType": appSettings.StorageProvider.Type, + }).Fatal("Invalid file system provided") } pipes := pipelines.LoadPipelines() @@ -170,7 +179,7 @@ func main() { if appSettings.RateLimiter.Enabled { rateLimiterMiddleware, err := middlewares.CreateRateLimiterMiddleware(appSettings.RateLimiter.RequestsPerMinute, appSettings.RateLimiter.AllowedBurst) if err != nil { - panic(err) + logrus.Fatal("Unexpected error while creating rate limiter middleware: ", err) } r.Use(rateLimiterMiddleware.Middleware) @@ -178,8 +187,10 @@ func main() { RegisterRoutes(r, pipes, storageProvider) + logrus.Info("Lithium started, listening for requests...") + err = http.ListenAndServe(appSettings.Endpoint, r) if err != nil { - panic(err) + logrus.Fatal("Unexpected error while serving http server: ", err) } } diff --git a/pipelines/pipeline.go b/pipelines/pipeline.go index af2e097..89a67dc 100644 --- a/pipelines/pipeline.go +++ b/pipelines/pipeline.go @@ -6,12 +6,12 @@ import ( "errors" "fmt" "io/fs" - "log" "os" "path/filepath" "github.com/disintegration/imaging" "github.com/geplauder/lithium/storage" + "github.com/sirupsen/logrus" ) // Pipelines @@ -126,7 +126,7 @@ func DeserializePipelines(pipelines [][]byte) []IPipeline { var deserializedObject Pipeline err := json.Unmarshal(pipeline, &deserializedObject) if err != nil { - log.Fatalf("Could not deserialize pipelines config: %s", err) + logrus.Fatalf("Could not deserialize pipelines config: %s", err) } values = append(values, deserializedObject) } @@ -149,7 +149,7 @@ func LoadPipelines() []IPipeline { }) if err != nil { - panic(err) + logrus.Fatal("Unexpected error while loading pipelines: ", err) } return DeserializePipelines(files) From 64d2b07d0db4b684b39336cb4bcc710eea15b064 Mon Sep 17 00:00:00 2001 From: Fabian Vowie Date: Sun, 23 Jan 2022 18:31:14 +0100 Subject: [PATCH 2/2] Write log into separate log-file --- .gitignore | 3 ++- main.go | 9 ++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index a6482a3..657294b 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,5 @@ go.work # Lithium specific -settings.json \ No newline at end of file +settings.json +lithium.log \ No newline at end of file diff --git a/main.go b/main.go index c34f205..b05c644 100644 --- a/main.go +++ b/main.go @@ -148,8 +148,15 @@ func RegisterRoutes(r *mux.Router, pipelines []pipelines.IPipeline, storageProvi } func main() { + logFile, err := os.OpenFile("lithium.log", os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + if err == nil { + multiWriter := io.MultiWriter(os.Stdout, logFile) + logrus.SetOutput(multiWriter) + } else { + logrus.SetOutput(os.Stdout) + } + logrus.SetFormatter(&logrus.JSONFormatter{}) - logrus.SetOutput(os.Stdout) appSettings, err := settings.LoadSettings(afero.NewOsFs()) if err != nil {