From dcddb6ca45e7eae6df5f3f64bb6bf5dda97126d1 Mon Sep 17 00:00:00 2001 From: Fabian Vowie Date: Sun, 23 Jan 2022 16:28:28 +0100 Subject: [PATCH 1/5] Move authorization middleware to dedicated middlewares package --- main.go | 4 ++-- {auth => middlewares}/authorization.go | 2 +- {auth => middlewares}/authorization_test.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) rename {auth => middlewares}/authorization.go (96%) rename {auth => middlewares}/authorization_test.go (98%) diff --git a/main.go b/main.go index 33cf1f7..792defd 100644 --- a/main.go +++ b/main.go @@ -4,7 +4,7 @@ import ( "encoding/json" "net/http" - "github.com/geplauder/lithium/auth" + "github.com/geplauder/lithium/middlewares" "github.com/geplauder/lithium/pipelines" "github.com/geplauder/lithium/settings" "github.com/geplauder/lithium/storage" @@ -60,7 +60,7 @@ func main() { pipes := pipelines.LoadPipelines() - authMiddleware := auth.CreateAuthenticationMiddleware(appSettings.Token) + authMiddleware := middlewares.CreateAuthenticationMiddleware(appSettings.Token) r := mux.NewRouter() r.Use(authMiddleware.Middleware) diff --git a/auth/authorization.go b/middlewares/authorization.go similarity index 96% rename from auth/authorization.go rename to middlewares/authorization.go index 8d3a14d..14f515e 100644 --- a/auth/authorization.go +++ b/middlewares/authorization.go @@ -1,4 +1,4 @@ -package auth +package middlewares import ( "net/http" diff --git a/auth/authorization_test.go b/middlewares/authorization_test.go similarity index 98% rename from auth/authorization_test.go rename to middlewares/authorization_test.go index 36075ea..b5c1da3 100644 --- a/auth/authorization_test.go +++ b/middlewares/authorization_test.go @@ -1,4 +1,4 @@ -package auth +package middlewares import ( "net/http" From 7a45c373754c2a87110b534a29f599863beeac37 Mon Sep 17 00:00:00 2001 From: Fabian Vowie Date: Sun, 23 Jan 2022 16:49:20 +0100 Subject: [PATCH 2/5] Add rate limiting middleware --- go.mod | 3 +++ go.sum | 21 +++++++++++++++ main.go | 6 +++++ middlewares/ratelimiter.go | 42 +++++++++++++++++++++++++++++ middlewares/ratelimiter_test.go | 48 +++++++++++++++++++++++++++++++++ 5 files changed, 120 insertions(+) create mode 100644 middlewares/ratelimiter.go create mode 100644 middlewares/ratelimiter_test.go diff --git a/go.mod b/go.mod index 088e354..fd0e86d 100644 --- a/go.mod +++ b/go.mod @@ -8,10 +8,13 @@ require ( github.com/gorilla/mux v1.8.0 github.com/spf13/afero v1.8.0 github.com/stretchr/testify v1.7.0 + github.com/throttled/throttled v2.2.5+incompatible + github.com/throttled/throttled/v2 v2.9.0 ) require ( github.com/davecgh/go-spew v1.1.0 // indirect + github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/kr/pretty v0.3.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rogpeppe/go-internal v1.8.1-0.20211023094830-115ce09fd6b4 // indirect diff --git a/go.sum b/go.sum index 0d23e14..a64e514 100644 --- a/go.sum +++ b/go.sum @@ -59,9 +59,11 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-redis/redis v6.15.8+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -87,6 +89,7 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/gomodule/redigo v1.8.4/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -120,6 +123,9 @@ github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -133,6 +139,9 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= @@ -150,6 +159,10 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/throttled/throttled v2.2.5+incompatible h1:65UB52X0qNTYiT0Sohp8qLYVFwZQPDw85uSa65OljjQ= +github.com/throttled/throttled v2.2.5+incompatible/go.mod h1:0BjlrEGQmvxps+HuXLsyRdqpSRvJpq0PNIsOtqP9Nos= +github.com/throttled/throttled/v2 v2.9.0 h1:DOkCb1el7NYzRoPb1pyeHVghsUoonVWEjmo34vrcp/8= +github.com/throttled/throttled/v2 v2.9.0/go.mod h1:0JHxhGAidPyqbgD4HF8Y1sNFfG0ffVXK6C8EpkNdLEM= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -205,6 +218,7 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -215,6 +229,7 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -254,6 +269,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -263,6 +279,7 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -443,7 +460,11 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/main.go b/main.go index 792defd..d9cf7df 100644 --- a/main.go +++ b/main.go @@ -61,9 +61,15 @@ func main() { pipes := pipelines.LoadPipelines() authMiddleware := middlewares.CreateAuthenticationMiddleware(appSettings.Token) + rateLimiterMiddleware, err := middlewares.CreateRateLimiterMiddleware(2) + if err != nil { + panic(err) + } r := mux.NewRouter() r.Use(authMiddleware.Middleware) + r.Use(rateLimiterMiddleware.Middleware) + r.HandleFunc("/", IndexHandler) RegisterPipelineRoutes(r, pipes, storageProvider) diff --git a/middlewares/ratelimiter.go b/middlewares/ratelimiter.go new file mode 100644 index 0000000..49fe1d8 --- /dev/null +++ b/middlewares/ratelimiter.go @@ -0,0 +1,42 @@ +package middlewares + +import ( + "net/http" + + "github.com/throttled/throttled/store/memstore" + "github.com/throttled/throttled/v2" +) + +type RateLimiterMiddleware struct { + rateLimiter throttled.HTTPRateLimiter +} + +func (middleware RateLimiterMiddleware) Middleware(next http.Handler) http.Handler { + return middleware.rateLimiter.RateLimit(next) +} + +func CreateRateLimiterMiddleware(requestsPerMinute int) (*RateLimiterMiddleware, error) { + store, err := memstore.New(65536) + + if err != nil { + return nil, err + } + + quota := throttled.RateQuota{ + MaxRate: throttled.PerMin(requestsPerMinute), + } + + rateLimiter, err := throttled.NewGCRARateLimiter(store, quota) + if err != nil { + return nil, err + } + + httpRateLimiter := throttled.HTTPRateLimiter{ + RateLimiter: rateLimiter, + VaryBy: &throttled.VaryBy{Path: true}, + } + + return &RateLimiterMiddleware{ + rateLimiter: httpRateLimiter, + }, nil +} diff --git a/middlewares/ratelimiter_test.go b/middlewares/ratelimiter_test.go new file mode 100644 index 0000000..ce946c0 --- /dev/null +++ b/middlewares/ratelimiter_test.go @@ -0,0 +1,48 @@ +package middlewares + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestRateLimiterMiddleware(t *testing.T) { + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + t.Run("AuthorizationMiddleware returns 200 response when rate limit is not hit", func(t *testing.T) { + middleware, err := CreateRateLimiterMiddleware(1) + assert.Nil(t, err) + + middlewareHandler := middleware.Middleware(handler) + + request, _ := http.NewRequest("GET", "/", nil) + responseRecorder := httptest.NewRecorder() + + middlewareHandler.ServeHTTP(responseRecorder, request) + + assert.Equal(t, 200, responseRecorder.Code) + }) + + t.Run("AuthorizationMiddleware returns 429 response when rate limit is hit", func(t *testing.T) { + middleware, err := CreateRateLimiterMiddleware(1) + assert.Nil(t, err) + + middlewareHandler := middleware.Middleware(handler) + + request, _ := http.NewRequest("GET", "/", nil) + responseRecorder := httptest.NewRecorder() + + middlewareHandler.ServeHTTP(responseRecorder, request) + assert.Equal(t, 200, responseRecorder.Code) + + request, _ = http.NewRequest("GET", "/", nil) + responseRecorder = httptest.NewRecorder() + + middlewareHandler.ServeHTTP(responseRecorder, request) + assert.Equal(t, 429, responseRecorder.Code) + }) +} From b87a2e4b0ba1f2f20a29364b1d39cc7901c28ff2 Mon Sep 17 00:00:00 2001 From: Fabian Vowie Date: Sun, 23 Jan 2022 16:54:01 +0100 Subject: [PATCH 3/5] Add rate limiting requests per minute field to settings --- main.go | 2 +- settings/settings.go | 12 +++++++----- settings/settings_test.go | 1 + 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/main.go b/main.go index d9cf7df..9d3c095 100644 --- a/main.go +++ b/main.go @@ -61,7 +61,7 @@ func main() { pipes := pipelines.LoadPipelines() authMiddleware := middlewares.CreateAuthenticationMiddleware(appSettings.Token) - rateLimiterMiddleware, err := middlewares.CreateRateLimiterMiddleware(2) + rateLimiterMiddleware, err := middlewares.CreateRateLimiterMiddleware(appSettings.RequestsPerMinute) if err != nil { panic(err) } diff --git a/settings/settings.go b/settings/settings.go index 1716c1e..760b8b6 100644 --- a/settings/settings.go +++ b/settings/settings.go @@ -15,9 +15,10 @@ const ( type FileSystemType int type Settings struct { - Endpoint string `json:"endpoint"` - Token string `json:"token"` - StorageProvider StorageSettings `json:"storage_provider"` + Endpoint string `json:"endpoint"` + Token string `json:"token"` + RequestsPerMinute int `json:"requests_per_minute"` + StorageProvider StorageSettings `json:"storage_provider"` } type StorageSettings struct { @@ -49,8 +50,9 @@ func LoadSettings(fileSystem afero.Fs) (Settings, error) { // If file does not exist, create default settings defaultSettings := Settings{ - Endpoint: "127.0.0.1:8000", - Token: "changeme", + Endpoint: "127.0.0.1:8000", + Token: "changeme", + RequestsPerMinute: 20, StorageProvider: StorageSettings{ Type: Local, BasePath: "assets", diff --git a/settings/settings_test.go b/settings/settings_test.go index d151b92..3621955 100644 --- a/settings/settings_test.go +++ b/settings/settings_test.go @@ -13,6 +13,7 @@ func TestSettingsParsing(t *testing.T) { const file string = `{ "endpoint": "0.0.0.0:8000", "token": "foobar", + "requests_per_minute": 20, "storage_provider": { "type": 0, "base_path": "assets" From 069a5ffe53844ebbedec671a5a779f1c6458031d Mon Sep 17 00:00:00 2001 From: Fabian Vowie Date: Sun, 23 Jan 2022 17:10:16 +0100 Subject: [PATCH 4/5] Add allowed burst for requests to rate limiting middleware --- main.go | 2 +- middlewares/ratelimiter.go | 5 ++-- middlewares/ratelimiter_test.go | 46 ++++++++++++++++++++++----------- 3 files changed, 35 insertions(+), 18 deletions(-) diff --git a/main.go b/main.go index 9d3c095..281005a 100644 --- a/main.go +++ b/main.go @@ -61,7 +61,7 @@ func main() { pipes := pipelines.LoadPipelines() authMiddleware := middlewares.CreateAuthenticationMiddleware(appSettings.Token) - rateLimiterMiddleware, err := middlewares.CreateRateLimiterMiddleware(appSettings.RequestsPerMinute) + rateLimiterMiddleware, err := middlewares.CreateRateLimiterMiddleware(appSettings.RequestsPerMinute, 0) if err != nil { panic(err) } diff --git a/middlewares/ratelimiter.go b/middlewares/ratelimiter.go index 49fe1d8..febe7fb 100644 --- a/middlewares/ratelimiter.go +++ b/middlewares/ratelimiter.go @@ -15,7 +15,7 @@ func (middleware RateLimiterMiddleware) Middleware(next http.Handler) http.Handl return middleware.rateLimiter.RateLimit(next) } -func CreateRateLimiterMiddleware(requestsPerMinute int) (*RateLimiterMiddleware, error) { +func CreateRateLimiterMiddleware(requestsPerMinute int, allowedBurst int) (*RateLimiterMiddleware, error) { store, err := memstore.New(65536) if err != nil { @@ -23,7 +23,8 @@ func CreateRateLimiterMiddleware(requestsPerMinute int) (*RateLimiterMiddleware, } quota := throttled.RateQuota{ - MaxRate: throttled.PerMin(requestsPerMinute), + MaxRate: throttled.PerMin(requestsPerMinute), + MaxBurst: allowedBurst, } rateLimiter, err := throttled.NewGCRARateLimiter(store, quota) diff --git a/middlewares/ratelimiter_test.go b/middlewares/ratelimiter_test.go index ce946c0..808f236 100644 --- a/middlewares/ratelimiter_test.go +++ b/middlewares/ratelimiter_test.go @@ -8,41 +8,57 @@ import ( "github.com/stretchr/testify/assert" ) +func ExecuteRequest(middlewareHandler http.Handler) int { + request, _ := http.NewRequest("GET", "/", nil) + responseRecorder := httptest.NewRecorder() + + middlewareHandler.ServeHTTP(responseRecorder, request) + + return responseRecorder.Code +} + func TestRateLimiterMiddleware(t *testing.T) { handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) }) t.Run("AuthorizationMiddleware returns 200 response when rate limit is not hit", func(t *testing.T) { - middleware, err := CreateRateLimiterMiddleware(1) + middleware, err := CreateRateLimiterMiddleware(1, 0) assert.Nil(t, err) middlewareHandler := middleware.Middleware(handler) - request, _ := http.NewRequest("GET", "/", nil) - responseRecorder := httptest.NewRecorder() + assert.Equal(t, 200, ExecuteRequest(middlewareHandler)) + }) - middlewareHandler.ServeHTTP(responseRecorder, request) + t.Run("AuthorizationMiddleware returns 429 response when rate limit is hit", func(t *testing.T) { + middleware, err := CreateRateLimiterMiddleware(1, 0) + assert.Nil(t, err) - assert.Equal(t, 200, responseRecorder.Code) + middlewareHandler := middleware.Middleware(handler) + + assert.Equal(t, 200, ExecuteRequest(middlewareHandler)) + assert.Equal(t, 429, ExecuteRequest(middlewareHandler)) }) - t.Run("AuthorizationMiddleware returns 429 response when rate limit is hit", func(t *testing.T) { - middleware, err := CreateRateLimiterMiddleware(1) + t.Run("AuthorizationMiddleware returns 200 response when rate limit with burst is not hit", func(t *testing.T) { + middleware, err := CreateRateLimiterMiddleware(1, 1) assert.Nil(t, err) middlewareHandler := middleware.Middleware(handler) - request, _ := http.NewRequest("GET", "/", nil) - responseRecorder := httptest.NewRecorder() + assert.Equal(t, 200, ExecuteRequest(middlewareHandler)) + assert.Equal(t, 200, ExecuteRequest(middlewareHandler)) + }) - middlewareHandler.ServeHTTP(responseRecorder, request) - assert.Equal(t, 200, responseRecorder.Code) + t.Run("AuthorizationMiddleware returns 429 response when rate limit with burst is hit", func(t *testing.T) { + middleware, err := CreateRateLimiterMiddleware(1, 1) + assert.Nil(t, err) - request, _ = http.NewRequest("GET", "/", nil) - responseRecorder = httptest.NewRecorder() + middlewareHandler := middleware.Middleware(handler) - middlewareHandler.ServeHTTP(responseRecorder, request) - assert.Equal(t, 429, responseRecorder.Code) + assert.Equal(t, 200, ExecuteRequest(middlewareHandler)) + assert.Equal(t, 200, ExecuteRequest(middlewareHandler)) + assert.Equal(t, 429, ExecuteRequest(middlewareHandler)) }) } From f3149d540bade35e3a1501354113097b2c9644f0 Mon Sep 17 00:00:00 2001 From: Fabian Vowie Date: Sun, 23 Jan 2022 17:13:51 +0100 Subject: [PATCH 5/5] Add rate limiting allowed burst field to settings --- main.go | 2 +- settings/settings.go | 22 +++++++++++++++------- settings/settings_test.go | 5 ++++- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/main.go b/main.go index 281005a..0191c1b 100644 --- a/main.go +++ b/main.go @@ -61,7 +61,7 @@ func main() { pipes := pipelines.LoadPipelines() authMiddleware := middlewares.CreateAuthenticationMiddleware(appSettings.Token) - rateLimiterMiddleware, err := middlewares.CreateRateLimiterMiddleware(appSettings.RequestsPerMinute, 0) + rateLimiterMiddleware, err := middlewares.CreateRateLimiterMiddleware(appSettings.RateLimiter.RequestsPerMinute, appSettings.RateLimiter.AllowedBurst) if err != nil { panic(err) } diff --git a/settings/settings.go b/settings/settings.go index 760b8b6..6e33efe 100644 --- a/settings/settings.go +++ b/settings/settings.go @@ -15,10 +15,10 @@ const ( type FileSystemType int type Settings struct { - Endpoint string `json:"endpoint"` - Token string `json:"token"` - RequestsPerMinute int `json:"requests_per_minute"` - StorageProvider StorageSettings `json:"storage_provider"` + Endpoint string `json:"endpoint"` + Token string `json:"token"` + RateLimiter RateLimiterSettings `json:"rate_limiter"` + StorageProvider StorageSettings `json:"storage_provider"` } type StorageSettings struct { @@ -26,6 +26,11 @@ type StorageSettings struct { BasePath string `json:"base_path"` } +type RateLimiterSettings struct { + RequestsPerMinute int `json:"requests_per_minute"` + AllowedBurst int `json:"allowed_burst"` +} + func parseSettings(data []byte) Settings { settings := Settings{} @@ -50,9 +55,12 @@ func LoadSettings(fileSystem afero.Fs) (Settings, error) { // If file does not exist, create default settings defaultSettings := Settings{ - Endpoint: "127.0.0.1:8000", - Token: "changeme", - RequestsPerMinute: 20, + Endpoint: "127.0.0.1:8000", + Token: "changeme", + RateLimiter: RateLimiterSettings{ + RequestsPerMinute: 20, + AllowedBurst: 5, + }, StorageProvider: StorageSettings{ Type: Local, BasePath: "assets", diff --git a/settings/settings_test.go b/settings/settings_test.go index 3621955..fa801ff 100644 --- a/settings/settings_test.go +++ b/settings/settings_test.go @@ -13,7 +13,10 @@ func TestSettingsParsing(t *testing.T) { const file string = `{ "endpoint": "0.0.0.0:8000", "token": "foobar", - "requests_per_minute": 20, + "rate_limiter": { + "requests_per_minute": 20, + "allowed_burst": 5 + }, "storage_provider": { "type": 0, "base_path": "assets"