From 2735d9c9e946b433aaedd1348854a95f45d0eba1 Mon Sep 17 00:00:00 2001 From: Fabian Vowie Date: Sun, 23 Jan 2022 16:28:28 +0100 Subject: [PATCH 1/9] Move authorization middleware to dedicated middlewares package --- main.go | 4 ++-- {auth => middlewares}/authorization.go | 2 +- {auth => middlewares}/authorization_test.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) rename {auth => middlewares}/authorization.go (96%) rename {auth => middlewares}/authorization_test.go (98%) diff --git a/main.go b/main.go index 33cf1f7..792defd 100644 --- a/main.go +++ b/main.go @@ -4,7 +4,7 @@ import ( "encoding/json" "net/http" - "github.com/geplauder/lithium/auth" + "github.com/geplauder/lithium/middlewares" "github.com/geplauder/lithium/pipelines" "github.com/geplauder/lithium/settings" "github.com/geplauder/lithium/storage" @@ -60,7 +60,7 @@ func main() { pipes := pipelines.LoadPipelines() - authMiddleware := auth.CreateAuthenticationMiddleware(appSettings.Token) + authMiddleware := middlewares.CreateAuthenticationMiddleware(appSettings.Token) r := mux.NewRouter() r.Use(authMiddleware.Middleware) diff --git a/auth/authorization.go b/middlewares/authorization.go similarity index 96% rename from auth/authorization.go rename to middlewares/authorization.go index 8d3a14d..14f515e 100644 --- a/auth/authorization.go +++ b/middlewares/authorization.go @@ -1,4 +1,4 @@ -package auth +package middlewares import ( "net/http" diff --git a/auth/authorization_test.go b/middlewares/authorization_test.go similarity index 98% rename from auth/authorization_test.go rename to middlewares/authorization_test.go index 36075ea..b5c1da3 100644 --- a/auth/authorization_test.go +++ b/middlewares/authorization_test.go @@ -1,4 +1,4 @@ -package auth +package middlewares import ( "net/http" From 5f2cbc5064f33a13415c0d7899b3b9bb66734642 Mon Sep 17 00:00:00 2001 From: Fabian Vowie Date: Sun, 23 Jan 2022 16:49:20 +0100 Subject: [PATCH 2/9] Add rate limiting middleware --- go.mod | 3 +++ go.sum | 21 +++++++++++++++ main.go | 6 +++++ middlewares/ratelimiter.go | 42 +++++++++++++++++++++++++++++ middlewares/ratelimiter_test.go | 48 +++++++++++++++++++++++++++++++++ 5 files changed, 120 insertions(+) create mode 100644 middlewares/ratelimiter.go create mode 100644 middlewares/ratelimiter_test.go diff --git a/go.mod b/go.mod index 088e354..fd0e86d 100644 --- a/go.mod +++ b/go.mod @@ -8,10 +8,13 @@ require ( github.com/gorilla/mux v1.8.0 github.com/spf13/afero v1.8.0 github.com/stretchr/testify v1.7.0 + github.com/throttled/throttled v2.2.5+incompatible + github.com/throttled/throttled/v2 v2.9.0 ) require ( github.com/davecgh/go-spew v1.1.0 // indirect + github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/kr/pretty v0.3.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rogpeppe/go-internal v1.8.1-0.20211023094830-115ce09fd6b4 // indirect diff --git a/go.sum b/go.sum index 0d23e14..a64e514 100644 --- a/go.sum +++ b/go.sum @@ -59,9 +59,11 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-redis/redis v6.15.8+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -87,6 +89,7 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/gomodule/redigo v1.8.4/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -120,6 +123,9 @@ github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -133,6 +139,9 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= @@ -150,6 +159,10 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/throttled/throttled v2.2.5+incompatible h1:65UB52X0qNTYiT0Sohp8qLYVFwZQPDw85uSa65OljjQ= +github.com/throttled/throttled v2.2.5+incompatible/go.mod h1:0BjlrEGQmvxps+HuXLsyRdqpSRvJpq0PNIsOtqP9Nos= +github.com/throttled/throttled/v2 v2.9.0 h1:DOkCb1el7NYzRoPb1pyeHVghsUoonVWEjmo34vrcp/8= +github.com/throttled/throttled/v2 v2.9.0/go.mod h1:0JHxhGAidPyqbgD4HF8Y1sNFfG0ffVXK6C8EpkNdLEM= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -205,6 +218,7 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -215,6 +229,7 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -254,6 +269,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -263,6 +279,7 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -443,7 +460,11 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/main.go b/main.go index 792defd..d9cf7df 100644 --- a/main.go +++ b/main.go @@ -61,9 +61,15 @@ func main() { pipes := pipelines.LoadPipelines() authMiddleware := middlewares.CreateAuthenticationMiddleware(appSettings.Token) + rateLimiterMiddleware, err := middlewares.CreateRateLimiterMiddleware(2) + if err != nil { + panic(err) + } r := mux.NewRouter() r.Use(authMiddleware.Middleware) + r.Use(rateLimiterMiddleware.Middleware) + r.HandleFunc("/", IndexHandler) RegisterPipelineRoutes(r, pipes, storageProvider) diff --git a/middlewares/ratelimiter.go b/middlewares/ratelimiter.go new file mode 100644 index 0000000..49fe1d8 --- /dev/null +++ b/middlewares/ratelimiter.go @@ -0,0 +1,42 @@ +package middlewares + +import ( + "net/http" + + "github.com/throttled/throttled/store/memstore" + "github.com/throttled/throttled/v2" +) + +type RateLimiterMiddleware struct { + rateLimiter throttled.HTTPRateLimiter +} + +func (middleware RateLimiterMiddleware) Middleware(next http.Handler) http.Handler { + return middleware.rateLimiter.RateLimit(next) +} + +func CreateRateLimiterMiddleware(requestsPerMinute int) (*RateLimiterMiddleware, error) { + store, err := memstore.New(65536) + + if err != nil { + return nil, err + } + + quota := throttled.RateQuota{ + MaxRate: throttled.PerMin(requestsPerMinute), + } + + rateLimiter, err := throttled.NewGCRARateLimiter(store, quota) + if err != nil { + return nil, err + } + + httpRateLimiter := throttled.HTTPRateLimiter{ + RateLimiter: rateLimiter, + VaryBy: &throttled.VaryBy{Path: true}, + } + + return &RateLimiterMiddleware{ + rateLimiter: httpRateLimiter, + }, nil +} diff --git a/middlewares/ratelimiter_test.go b/middlewares/ratelimiter_test.go new file mode 100644 index 0000000..ce946c0 --- /dev/null +++ b/middlewares/ratelimiter_test.go @@ -0,0 +1,48 @@ +package middlewares + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestRateLimiterMiddleware(t *testing.T) { + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + t.Run("AuthorizationMiddleware returns 200 response when rate limit is not hit", func(t *testing.T) { + middleware, err := CreateRateLimiterMiddleware(1) + assert.Nil(t, err) + + middlewareHandler := middleware.Middleware(handler) + + request, _ := http.NewRequest("GET", "/", nil) + responseRecorder := httptest.NewRecorder() + + middlewareHandler.ServeHTTP(responseRecorder, request) + + assert.Equal(t, 200, responseRecorder.Code) + }) + + t.Run("AuthorizationMiddleware returns 429 response when rate limit is hit", func(t *testing.T) { + middleware, err := CreateRateLimiterMiddleware(1) + assert.Nil(t, err) + + middlewareHandler := middleware.Middleware(handler) + + request, _ := http.NewRequest("GET", "/", nil) + responseRecorder := httptest.NewRecorder() + + middlewareHandler.ServeHTTP(responseRecorder, request) + assert.Equal(t, 200, responseRecorder.Code) + + request, _ = http.NewRequest("GET", "/", nil) + responseRecorder = httptest.NewRecorder() + + middlewareHandler.ServeHTTP(responseRecorder, request) + assert.Equal(t, 429, responseRecorder.Code) + }) +} From e7a6833f71cda4fd8740b9a047b7bf84d78649df Mon Sep 17 00:00:00 2001 From: Fabian Vowie Date: Sun, 23 Jan 2022 16:54:01 +0100 Subject: [PATCH 3/9] Add rate limiting requests per minute field to settings --- main.go | 2 +- settings/settings.go | 12 +++++++----- settings/settings_test.go | 1 + 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/main.go b/main.go index d9cf7df..9d3c095 100644 --- a/main.go +++ b/main.go @@ -61,7 +61,7 @@ func main() { pipes := pipelines.LoadPipelines() authMiddleware := middlewares.CreateAuthenticationMiddleware(appSettings.Token) - rateLimiterMiddleware, err := middlewares.CreateRateLimiterMiddleware(2) + rateLimiterMiddleware, err := middlewares.CreateRateLimiterMiddleware(appSettings.RequestsPerMinute) if err != nil { panic(err) } diff --git a/settings/settings.go b/settings/settings.go index 1716c1e..760b8b6 100644 --- a/settings/settings.go +++ b/settings/settings.go @@ -15,9 +15,10 @@ const ( type FileSystemType int type Settings struct { - Endpoint string `json:"endpoint"` - Token string `json:"token"` - StorageProvider StorageSettings `json:"storage_provider"` + Endpoint string `json:"endpoint"` + Token string `json:"token"` + RequestsPerMinute int `json:"requests_per_minute"` + StorageProvider StorageSettings `json:"storage_provider"` } type StorageSettings struct { @@ -49,8 +50,9 @@ func LoadSettings(fileSystem afero.Fs) (Settings, error) { // If file does not exist, create default settings defaultSettings := Settings{ - Endpoint: "127.0.0.1:8000", - Token: "changeme", + Endpoint: "127.0.0.1:8000", + Token: "changeme", + RequestsPerMinute: 20, StorageProvider: StorageSettings{ Type: Local, BasePath: "assets", diff --git a/settings/settings_test.go b/settings/settings_test.go index d151b92..3621955 100644 --- a/settings/settings_test.go +++ b/settings/settings_test.go @@ -13,6 +13,7 @@ func TestSettingsParsing(t *testing.T) { const file string = `{ "endpoint": "0.0.0.0:8000", "token": "foobar", + "requests_per_minute": 20, "storage_provider": { "type": 0, "base_path": "assets" From 2d0416277e3259c751e869ca67ad05c5896ece3f Mon Sep 17 00:00:00 2001 From: Fabian Vowie Date: Sun, 23 Jan 2022 17:10:16 +0100 Subject: [PATCH 4/9] Add allowed burst for requests to rate limiting middleware --- main.go | 2 +- middlewares/ratelimiter.go | 5 ++-- middlewares/ratelimiter_test.go | 46 ++++++++++++++++++++++----------- 3 files changed, 35 insertions(+), 18 deletions(-) diff --git a/main.go b/main.go index 9d3c095..281005a 100644 --- a/main.go +++ b/main.go @@ -61,7 +61,7 @@ func main() { pipes := pipelines.LoadPipelines() authMiddleware := middlewares.CreateAuthenticationMiddleware(appSettings.Token) - rateLimiterMiddleware, err := middlewares.CreateRateLimiterMiddleware(appSettings.RequestsPerMinute) + rateLimiterMiddleware, err := middlewares.CreateRateLimiterMiddleware(appSettings.RequestsPerMinute, 0) if err != nil { panic(err) } diff --git a/middlewares/ratelimiter.go b/middlewares/ratelimiter.go index 49fe1d8..febe7fb 100644 --- a/middlewares/ratelimiter.go +++ b/middlewares/ratelimiter.go @@ -15,7 +15,7 @@ func (middleware RateLimiterMiddleware) Middleware(next http.Handler) http.Handl return middleware.rateLimiter.RateLimit(next) } -func CreateRateLimiterMiddleware(requestsPerMinute int) (*RateLimiterMiddleware, error) { +func CreateRateLimiterMiddleware(requestsPerMinute int, allowedBurst int) (*RateLimiterMiddleware, error) { store, err := memstore.New(65536) if err != nil { @@ -23,7 +23,8 @@ func CreateRateLimiterMiddleware(requestsPerMinute int) (*RateLimiterMiddleware, } quota := throttled.RateQuota{ - MaxRate: throttled.PerMin(requestsPerMinute), + MaxRate: throttled.PerMin(requestsPerMinute), + MaxBurst: allowedBurst, } rateLimiter, err := throttled.NewGCRARateLimiter(store, quota) diff --git a/middlewares/ratelimiter_test.go b/middlewares/ratelimiter_test.go index ce946c0..808f236 100644 --- a/middlewares/ratelimiter_test.go +++ b/middlewares/ratelimiter_test.go @@ -8,41 +8,57 @@ import ( "github.com/stretchr/testify/assert" ) +func ExecuteRequest(middlewareHandler http.Handler) int { + request, _ := http.NewRequest("GET", "/", nil) + responseRecorder := httptest.NewRecorder() + + middlewareHandler.ServeHTTP(responseRecorder, request) + + return responseRecorder.Code +} + func TestRateLimiterMiddleware(t *testing.T) { handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) }) t.Run("AuthorizationMiddleware returns 200 response when rate limit is not hit", func(t *testing.T) { - middleware, err := CreateRateLimiterMiddleware(1) + middleware, err := CreateRateLimiterMiddleware(1, 0) assert.Nil(t, err) middlewareHandler := middleware.Middleware(handler) - request, _ := http.NewRequest("GET", "/", nil) - responseRecorder := httptest.NewRecorder() + assert.Equal(t, 200, ExecuteRequest(middlewareHandler)) + }) - middlewareHandler.ServeHTTP(responseRecorder, request) + t.Run("AuthorizationMiddleware returns 429 response when rate limit is hit", func(t *testing.T) { + middleware, err := CreateRateLimiterMiddleware(1, 0) + assert.Nil(t, err) - assert.Equal(t, 200, responseRecorder.Code) + middlewareHandler := middleware.Middleware(handler) + + assert.Equal(t, 200, ExecuteRequest(middlewareHandler)) + assert.Equal(t, 429, ExecuteRequest(middlewareHandler)) }) - t.Run("AuthorizationMiddleware returns 429 response when rate limit is hit", func(t *testing.T) { - middleware, err := CreateRateLimiterMiddleware(1) + t.Run("AuthorizationMiddleware returns 200 response when rate limit with burst is not hit", func(t *testing.T) { + middleware, err := CreateRateLimiterMiddleware(1, 1) assert.Nil(t, err) middlewareHandler := middleware.Middleware(handler) - request, _ := http.NewRequest("GET", "/", nil) - responseRecorder := httptest.NewRecorder() + assert.Equal(t, 200, ExecuteRequest(middlewareHandler)) + assert.Equal(t, 200, ExecuteRequest(middlewareHandler)) + }) - middlewareHandler.ServeHTTP(responseRecorder, request) - assert.Equal(t, 200, responseRecorder.Code) + t.Run("AuthorizationMiddleware returns 429 response when rate limit with burst is hit", func(t *testing.T) { + middleware, err := CreateRateLimiterMiddleware(1, 1) + assert.Nil(t, err) - request, _ = http.NewRequest("GET", "/", nil) - responseRecorder = httptest.NewRecorder() + middlewareHandler := middleware.Middleware(handler) - middlewareHandler.ServeHTTP(responseRecorder, request) - assert.Equal(t, 429, responseRecorder.Code) + assert.Equal(t, 200, ExecuteRequest(middlewareHandler)) + assert.Equal(t, 200, ExecuteRequest(middlewareHandler)) + assert.Equal(t, 429, ExecuteRequest(middlewareHandler)) }) } From 49990196f68b6221c0350100d0b36862bd5b91dc Mon Sep 17 00:00:00 2001 From: Fabian Vowie Date: Sun, 23 Jan 2022 17:13:51 +0100 Subject: [PATCH 5/9] Add rate limiting allowed burst field to settings --- main.go | 2 +- settings/settings.go | 22 +++++++++++++++------- settings/settings_test.go | 5 ++++- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/main.go b/main.go index 281005a..0191c1b 100644 --- a/main.go +++ b/main.go @@ -61,7 +61,7 @@ func main() { pipes := pipelines.LoadPipelines() authMiddleware := middlewares.CreateAuthenticationMiddleware(appSettings.Token) - rateLimiterMiddleware, err := middlewares.CreateRateLimiterMiddleware(appSettings.RequestsPerMinute, 0) + rateLimiterMiddleware, err := middlewares.CreateRateLimiterMiddleware(appSettings.RateLimiter.RequestsPerMinute, appSettings.RateLimiter.AllowedBurst) if err != nil { panic(err) } diff --git a/settings/settings.go b/settings/settings.go index 760b8b6..6e33efe 100644 --- a/settings/settings.go +++ b/settings/settings.go @@ -15,10 +15,10 @@ const ( type FileSystemType int type Settings struct { - Endpoint string `json:"endpoint"` - Token string `json:"token"` - RequestsPerMinute int `json:"requests_per_minute"` - StorageProvider StorageSettings `json:"storage_provider"` + Endpoint string `json:"endpoint"` + Token string `json:"token"` + RateLimiter RateLimiterSettings `json:"rate_limiter"` + StorageProvider StorageSettings `json:"storage_provider"` } type StorageSettings struct { @@ -26,6 +26,11 @@ type StorageSettings struct { BasePath string `json:"base_path"` } +type RateLimiterSettings struct { + RequestsPerMinute int `json:"requests_per_minute"` + AllowedBurst int `json:"allowed_burst"` +} + func parseSettings(data []byte) Settings { settings := Settings{} @@ -50,9 +55,12 @@ func LoadSettings(fileSystem afero.Fs) (Settings, error) { // If file does not exist, create default settings defaultSettings := Settings{ - Endpoint: "127.0.0.1:8000", - Token: "changeme", - RequestsPerMinute: 20, + Endpoint: "127.0.0.1:8000", + Token: "changeme", + RateLimiter: RateLimiterSettings{ + RequestsPerMinute: 20, + AllowedBurst: 5, + }, StorageProvider: StorageSettings{ Type: Local, BasePath: "assets", diff --git a/settings/settings_test.go b/settings/settings_test.go index 3621955..fa801ff 100644 --- a/settings/settings_test.go +++ b/settings/settings_test.go @@ -13,7 +13,10 @@ func TestSettingsParsing(t *testing.T) { const file string = `{ "endpoint": "0.0.0.0:8000", "token": "foobar", - "requests_per_minute": 20, + "rate_limiter": { + "requests_per_minute": 20, + "allowed_burst": 5 + }, "storage_provider": { "type": 0, "base_path": "assets" From fafc19088ea0d45b0a87086d3cc3b83603dc9e44 Mon Sep 17 00:00:00 2001 From: Fabian Vowie Date: Sun, 23 Jan 2022 17:32:09 +0100 Subject: [PATCH 6/9] Add rate limiting explanation to readme --- README.md | 54 +++++++++++++++++++++++++++++++++++------------------- 1 file changed, 35 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 7101b4e..f9ceb39 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Micro-service for file storage and processing written in Go. ## Requirements - [Go 1.17+](https://go.dev/) -- [*Docker*](https://docs.docker.com/) (optional) +- [_Docker_](https://docs.docker.com/) (optional) ## Setup @@ -56,13 +56,43 @@ go test ./... -v docker run --rm -v "$PWD":/usr/src/lithium -w /usr/src/lithium golang:1.17 go test ./... ``` +## Configuration + +Config options can be adjusted via the [`settings.json`](settings.json) file in the root directory. + +```json +{ + "endpoint": "0.0.0.0:8000", + "token": "changeme", + "rate_limiter": { + "requests_per_minute": 20, + "allowed_burst": 5 + }, + "storage_provider": { + "type": 0, + "base_path": "assets" + } +} +``` + +## Rate Limiting + +By default, the rate limiting takes place on a per-route basis. When the limit for a specific route is hit, the response will return a status code `429: Too Many Requests`. + +| Headers | Explanation | +| --------------------- | -------------------------------- | +| X-Ratelimit-Limit | Allowed requests per minute | +| X-Ratelimit-Remaining | Remaining requests | +| X-Ratelimit-Reset | Seconds until requests replenish | + ## API ### `GET` `/` -Show application information. +Show application information. **Required headers**: + ```shell Authorization: Bearer ``` @@ -81,6 +111,7 @@ Authorization: Bearer Show pipeline information. **Required headers**: + ```shell Authorization: Bearer ``` @@ -111,27 +142,12 @@ Authorization: Bearer } ``` -## Configuration - -Config options can be adjusted via the [`settings.json`](settings.json) file in the root directory. - -```json -{ - "endpoint": "0.0.0.0:8000", - "token": "changeme", - "storage_provider": { - "type": 0, - "base_path": "assets" - } -} -``` - ## Pipelines The project uses a pipeline system defined by [JSON](https://en.wikipedia.org/wiki/JSON) files located in the [config](config) folder. Take a look at the [example pipeline configuration file](config/example.json). -Available pipeline `type`s: `0` (Image), `1` (Video) +Available pipeline `type`s: `0` (Image), `1` (Video) ```json { @@ -148,7 +164,7 @@ Available pipeline `type`s: `0` (Image), `1` (Video) The image pipeline offers the following additional output options. -Available `format` types: `jpg` (or `jpeg`), `png`, `gif`, `tif` (or `tiff`) and `bmp` +Available `format` types: `jpg` (or `jpeg`), `png`, `gif`, `tif` (or `tiff`) and `bmp` The `quality` field can contain any integer between 1 and 100. ```json From 9793374d0485b29628a96f1652a6cd257d124f03 Mon Sep 17 00:00:00 2001 From: Fabian Vowie Date: Sun, 23 Jan 2022 17:43:56 +0100 Subject: [PATCH 7/9] Add commit hash to index metadata --- build.sh | 3 +++ main.go | 9 ++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) create mode 100755 build.sh diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..77f3431 --- /dev/null +++ b/build.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +GIT_COMMIT=$(git rev-parse --short HEAD); go build -ldflags "-X main.GitCommit=$GIT_COMMIT" diff --git a/main.go b/main.go index 0191c1b..030aa5b 100644 --- a/main.go +++ b/main.go @@ -15,9 +15,12 @@ import ( const Name string = "Lithium" const Version string = "0.1.0" +var GitCommit string + type Metadata struct { - Name string `json:"name"` - Version string `json:"version"` + Name string `json:"name"` + Version string `json:"version"` + CommitHash string `json:"commit_hash"` } func PipelineHandler(pipeline pipelines.IPipeline, storageProvider storage.IStorageProvider, w http.ResponseWriter, r *http.Request) { @@ -30,7 +33,7 @@ func PipelineHandler(pipeline pipelines.IPipeline, storageProvider storage.IStor func IndexHandler(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") - err := json.NewEncoder(w).Encode(Metadata{Name, Version}) + err := json.NewEncoder(w).Encode(Metadata{Name, Version, GitCommit}) if err != nil { w.WriteHeader(http.StatusInternalServerError) } From 66b240943f25ac203b59ac2c122dfd2fa7bff89f Mon Sep 17 00:00:00 2001 From: Fabian Vowie Date: Sun, 23 Jan 2022 18:13:12 +0100 Subject: [PATCH 8/9] Add enabled field to rate limiter settings --- main.go | 5 ++++- settings/settings.go | 6 ++++-- settings/settings_test.go | 1 + 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/main.go b/main.go index 030aa5b..3d2f59d 100644 --- a/main.go +++ b/main.go @@ -71,7 +71,10 @@ func main() { r := mux.NewRouter() r.Use(authMiddleware.Middleware) - r.Use(rateLimiterMiddleware.Middleware) + + if appSettings.RateLimiter.Enabled { + r.Use(rateLimiterMiddleware.Middleware) + } r.HandleFunc("/", IndexHandler) diff --git a/settings/settings.go b/settings/settings.go index 6e33efe..8b0264d 100644 --- a/settings/settings.go +++ b/settings/settings.go @@ -27,8 +27,9 @@ type StorageSettings struct { } type RateLimiterSettings struct { - RequestsPerMinute int `json:"requests_per_minute"` - AllowedBurst int `json:"allowed_burst"` + Enabled bool `json:"enabled"` + RequestsPerMinute int `json:"requests_per_minute"` + AllowedBurst int `json:"allowed_burst"` } func parseSettings(data []byte) Settings { @@ -58,6 +59,7 @@ func LoadSettings(fileSystem afero.Fs) (Settings, error) { Endpoint: "127.0.0.1:8000", Token: "changeme", RateLimiter: RateLimiterSettings{ + Enabled: true, RequestsPerMinute: 20, AllowedBurst: 5, }, diff --git a/settings/settings_test.go b/settings/settings_test.go index fa801ff..8f91392 100644 --- a/settings/settings_test.go +++ b/settings/settings_test.go @@ -14,6 +14,7 @@ func TestSettingsParsing(t *testing.T) { "endpoint": "0.0.0.0:8000", "token": "foobar", "rate_limiter": { + "enabled": true, "requests_per_minute": 20, "allowed_burst": 5 }, From 3aaea90513cc6e299f7aa9f4e43a539748157cc4 Mon Sep 17 00:00:00 2001 From: Fabian Vowie Date: Sun, 23 Jan 2022 18:16:55 +0100 Subject: [PATCH 9/9] Add enabled field to authentication settings --- main.go | 18 +++++++++++------- settings/settings.go | 18 +++++++++++++----- settings/settings_test.go | 7 +++++-- 3 files changed, 29 insertions(+), 14 deletions(-) diff --git a/main.go b/main.go index 3d2f59d..049a604 100644 --- a/main.go +++ b/main.go @@ -63,16 +63,20 @@ func main() { pipes := pipelines.LoadPipelines() - authMiddleware := middlewares.CreateAuthenticationMiddleware(appSettings.Token) - rateLimiterMiddleware, err := middlewares.CreateRateLimiterMiddleware(appSettings.RateLimiter.RequestsPerMinute, appSettings.RateLimiter.AllowedBurst) - if err != nil { - panic(err) - } - r := mux.NewRouter() - r.Use(authMiddleware.Middleware) + + if appSettings.Authentication.Enabled { + authMiddleware := middlewares.CreateAuthenticationMiddleware(appSettings.Authentication.Token) + + r.Use(authMiddleware.Middleware) + } if appSettings.RateLimiter.Enabled { + rateLimiterMiddleware, err := middlewares.CreateRateLimiterMiddleware(appSettings.RateLimiter.RequestsPerMinute, appSettings.RateLimiter.AllowedBurst) + if err != nil { + panic(err) + } + r.Use(rateLimiterMiddleware.Middleware) } diff --git a/settings/settings.go b/settings/settings.go index 8b0264d..81cd6e4 100644 --- a/settings/settings.go +++ b/settings/settings.go @@ -15,10 +15,10 @@ const ( type FileSystemType int type Settings struct { - Endpoint string `json:"endpoint"` - Token string `json:"token"` - RateLimiter RateLimiterSettings `json:"rate_limiter"` - StorageProvider StorageSettings `json:"storage_provider"` + Endpoint string `json:"endpoint"` + Authentication AuthenticationSettings `json:"authentication"` + RateLimiter RateLimiterSettings `json:"rate_limiter"` + StorageProvider StorageSettings `json:"storage_provider"` } type StorageSettings struct { @@ -26,6 +26,11 @@ type StorageSettings struct { BasePath string `json:"base_path"` } +type AuthenticationSettings struct { + Enabled bool `json:"enabled"` + Token string `json:"token"` +} + type RateLimiterSettings struct { Enabled bool `json:"enabled"` RequestsPerMinute int `json:"requests_per_minute"` @@ -57,7 +62,10 @@ func LoadSettings(fileSystem afero.Fs) (Settings, error) { // If file does not exist, create default settings defaultSettings := Settings{ Endpoint: "127.0.0.1:8000", - Token: "changeme", + Authentication: AuthenticationSettings{ + Enabled: true, + Token: "changeme", + }, RateLimiter: RateLimiterSettings{ Enabled: true, RequestsPerMinute: 20, diff --git a/settings/settings_test.go b/settings/settings_test.go index 8f91392..287734e 100644 --- a/settings/settings_test.go +++ b/settings/settings_test.go @@ -12,7 +12,10 @@ import ( func TestSettingsParsing(t *testing.T) { const file string = `{ "endpoint": "0.0.0.0:8000", - "token": "foobar", + "authentication": { + "enabled": true, + "token": "foobar" + }, "rate_limiter": { "enabled": true, "requests_per_minute": 20, @@ -28,7 +31,7 @@ func TestSettingsParsing(t *testing.T) { settings := parseSettings([]byte(file)) assert.Equal(t, "0.0.0.0:8000", settings.Endpoint) - assert.Equal(t, "foobar", settings.Token) + assert.Equal(t, "foobar", settings.Authentication.Token) assert.Equal(t, "assets", settings.StorageProvider.BasePath) }) }