Compare commits

...

32 Commits

Author SHA1 Message Date
Fabian Vowie 6ac87dc7dc
Add builder function to create AuthenticationMiddleware objects 3 years ago
Fabian Vowie 5bbae63e6c
Require 'Bearer' prefix in authorization header 3 years ago
Fabian Vowie a0370a78ec
Fix assertion usage and use randomly generated token in auth middleware tests 3 years ago
Fabian Vowie 7aec1fb513
Add token-based authorization middleware 3 years ago
Jenkins d172b0edf5
Merge commit '226099211ea11628acea089f2efe7e952310a3db' into HEAD 3 years ago
Roman Zipp 2841299689
Merge branch 'main' into pr-feature/add-image-processing 3 years ago
Roman Zipp 308dad3479
Add image flip step 3 years ago
Roman Zipp 6637a60c24
Add grayscale image step 3 years ago
Roman Zipp f41510c110
Refactor pipeline tests 3 years ago
Roman Zipp ac414d866b
Add pipeline test image dimension assertions 3 years ago
Roman Zipp e712b52e77
Add storage provider open method 3 years ago
Roman Zipp 1a002ee46e
Add pipeline execution test without any available steps 3 years ago
Roman Zipp 565e4a3bad
Add rotating image step 3 years ago
Roman Zipp ebf4b0010e
Move jpeg quality setting from pipeline step to new output options 3 years ago
Roman Zipp 287880e72b
Add resizing images 3 years ago
Roman Zipp 6c87a1833b Add storage provider GetPath method 3 years ago
Roman Zipp 9692c79a26 Update storage provider StoreExisting method to take absolute file paths 3 years ago
Roman Zipp 906941e897 Add storage provider error handling on reading existing files 3 years ago
Roman Zipp 3821f5c7fe Add storage provider working directory parameter 3 years ago
Roman Zipp ae9f318a3e Add storage module error handling 3 years ago
Jenkins b9921ec0d4 Merge commit '994677cf3412ad972b93083b24dd6a39148e2d58' into HEAD 3 years ago
Fabian Vowie 994677cf34
Add tags for json serialization to settings structs 3 years ago
Fabian Vowie 96436acd89
Improve testability for LoadSettings function 3 years ago
Fabian Vowie 0f16daa99b
Add settings.json to .gitignore 3 years ago
Fabian Vowie 07ae302df9
Rename FileSystem in settings to StorageProvider 3 years ago
Fabian Vowie 48f82f3891
Pretty-print default settings json 3 years ago
Fabian Vowie 9a6bdc2552
Remove leftover debug code 3 years ago
Fabian Vowie 583103c384
Load settings on program start or create default if none exist 3 years ago
Fabian Vowie fdb4d59448
Add settings parsing 3 years ago
Jenkins f36fb2725c Merge commit '08ca3c5b2c25af6eb9e278991e4ea9e7bffb839d' into HEAD 3 years ago
Fabian Vowie 08ca3c5b2c
Temporarily hardcode filesystem storage provider 3 years ago
Fabian Vowie 44038e1626
Fix store methods not being public 3 years ago
  1. 5
      .gitignore
  2. 28
      auth/authorization.go
  3. 40
      auth/authorization_test.go
  4. 18
      config/example.json
  5. 4
      go.mod
  6. 8
      go.sum
  7. 29
      main.go
  8. 7
      main_test.go
  9. 58
      pipelines/executable_step.go
  10. 15
      pipelines/executable_step_test.go
  11. 52
      pipelines/pipeline.go
  12. 361
      pipelines/pipeline_test.go
  13. 19
      pipelines/step.go
  14. 61
      settings/settings.go
  15. 48
      settings/settings_test.go
  16. 57
      storage/storage.go
  17. 23
      storage/storage_test.go
  18. BIN
      tests/files/800x500.jpg
  19. BIN
      tests/files/900x900.jpg

5
.gitignore

@ -12,4 +12,7 @@
*.out *.out
# Go workspace file # Go workspace file
go.work
go.work
# Lithium specific
settings.json

28
auth/authorization.go

@ -0,0 +1,28 @@
package auth
import (
"net/http"
"strings"
)
type AuthenticationMiddleware struct {
secret string
}
func (middleware AuthenticationMiddleware) Middleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
authToken := r.Header.Get("Authorization")
if authToken == "" || strings.HasPrefix(authToken, "Bearer ") == false || authToken[7:] != middleware.secret {
http.Error(w, "Forbidden", http.StatusForbidden)
} else {
next.ServeHTTP(w, r)
}
})
}
func CreateAuthenticationMiddleware(secret string) AuthenticationMiddleware {
return AuthenticationMiddleware{
secret: secret,
}
}

40
auth/authorization_test.go

@ -0,0 +1,40 @@
package auth
import (
"net/http"
"net/http/httptest"
"testing"
"github.com/bxcodec/faker/v3"
"github.com/stretchr/testify/assert"
)
func TestAuthorizationMiddleware(t *testing.T) {
token := faker.Word()
middleware := CreateAuthenticationMiddleware(token)
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
})
middlewareHandler := middleware.Middleware(handler)
t.Run("AuthorizationMiddleware returns 403 response when authorization header is incorrect", func(t *testing.T) {
request, _ := http.NewRequest("GET", "/", nil)
responseRecorder := httptest.NewRecorder()
middlewareHandler.ServeHTTP(responseRecorder, request)
assert.Equal(t, 403, responseRecorder.Code)
})
t.Run("AuthorizationMiddleware continues when authorization header is correct", func(t *testing.T) {
request, _ := http.NewRequest("GET", "/", nil)
request.Header.Set("Authorization", "Bearer "+token)
responseRecorder := httptest.NewRecorder()
middlewareHandler.ServeHTTP(responseRecorder, request)
assert.Equal(t, 200, responseRecorder.Code)
})
}

18
config/example.json

@ -14,11 +14,23 @@
} }
}, },
{ {
"name": "compress image",
"name": "rotate image",
"type": 1, "type": 1,
"options": { "options": {
"quality": 80
"angle": 90.0
} }
},
{
"name": "flip image",
"type": 2
},
{
"name": "grayscale",
"type": 3
} }
]
],
"output": {
"format": 0,
"quality": 90
}
} }

4
go.mod

@ -4,6 +4,7 @@ go 1.17
require ( require (
github.com/bxcodec/faker/v3 v3.7.0 github.com/bxcodec/faker/v3 v3.7.0
github.com/disintegration/imaging v1.6.2
github.com/gorilla/mux v1.8.0 github.com/gorilla/mux v1.8.0
github.com/spf13/afero v1.8.0 github.com/spf13/afero v1.8.0
github.com/stretchr/testify v1.7.0 github.com/stretchr/testify v1.7.0
@ -14,7 +15,8 @@ require (
github.com/kr/pretty v0.3.0 // indirect github.com/kr/pretty v0.3.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rogpeppe/go-internal v1.8.1-0.20211023094830-115ce09fd6b4 // indirect github.com/rogpeppe/go-internal v1.8.1-0.20211023094830-115ce09fd6b4 // indirect
golang.org/x/text v0.3.4 // indirect
golang.org/x/image v0.0.0-20211028202545-6944b10bf410 // indirect
golang.org/x/text v0.3.6 // indirect
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
) )

8
go.sum

@ -51,6 +51,8 @@ github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnht
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c=
github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
@ -177,6 +179,9 @@ golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EH
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/image v0.0.0-20211028202545-6944b10bf410 h1:hTftEOvwiOq2+O8k2D5/Q7COC7k5Qcrgc2TFURJYnvQ=
golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@ -288,8 +293,9 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc=
golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

29
main.go

@ -2,10 +2,14 @@ package main
import ( import (
"encoding/json" "encoding/json"
"github.com/geplauder/lithium/pipelines"
"net/http" "net/http"
"github.com/geplauder/lithium/auth"
"github.com/geplauder/lithium/pipelines"
"github.com/geplauder/lithium/settings"
"github.com/geplauder/lithium/storage"
"github.com/gorilla/mux" "github.com/gorilla/mux"
"github.com/spf13/afero"
) )
const Name string = "Lithium" const Name string = "Lithium"
@ -16,7 +20,7 @@ type Metadata struct {
Version string Version string
} }
func PipelineHandler(pipeline pipelines.IPipeline, w http.ResponseWriter, r *http.Request) {
func PipelineHandler(pipeline pipelines.IPipeline, storageProvider storage.IStorageProvider, w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")
err := json.NewEncoder(w).Encode(pipeline) err := json.NewEncoder(w).Encode(pipeline)
if err != nil { if err != nil {
@ -32,23 +36,36 @@ func IndexHandler(w http.ResponseWriter, r *http.Request) {
} }
} }
func RegisterPipelineRoutes(r *mux.Router, pipelines []pipelines.IPipeline) {
func RegisterPipelineRoutes(r *mux.Router, pipelines []pipelines.IPipeline, storageProvider storage.IStorageProvider) {
for _, pipeline := range pipelines { for _, pipeline := range pipelines {
r.HandleFunc("/"+pipeline.GetSlug(), func(w http.ResponseWriter, r *http.Request) { r.HandleFunc("/"+pipeline.GetSlug(), func(w http.ResponseWriter, r *http.Request) {
PipelineHandler(pipeline, w, r)
PipelineHandler(pipeline, storageProvider, w, r)
}) })
} }
} }
func main() { func main() {
settings := settings.LoadSettings(afero.NewOsFs())
var storageProvider storage.IStorageProvider
if settings.StorageProvider.Type == 0 {
storageProvider = storage.GetFileSystemStorageProvider(settings.StorageProvider.BasePath, "")
} else {
panic("Invalid file system provided!")
}
pipes := pipelines.LoadPipelines() pipes := pipelines.LoadPipelines()
authMiddleware := auth.CreateAuthenticationMiddleware(settings.Token)
r := mux.NewRouter() r := mux.NewRouter()
r.Use(authMiddleware.Middleware)
r.HandleFunc("/", IndexHandler) r.HandleFunc("/", IndexHandler)
RegisterPipelineRoutes(r, pipes)
RegisterPipelineRoutes(r, pipes, storageProvider)
err := http.ListenAndServe(":8000", r)
err := http.ListenAndServe(settings.Endpoint, r)
if err != nil { if err != nil {
panic(err) panic(err)
} }

7
main_test.go

@ -3,12 +3,13 @@ package main
import ( import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"github.com/geplauder/lithium/pipelines"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"testing" "testing"
"github.com/bxcodec/faker/v3" "github.com/bxcodec/faker/v3"
"github.com/geplauder/lithium/pipelines"
"github.com/geplauder/lithium/storage"
"github.com/gorilla/mux" "github.com/gorilla/mux"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
) )
@ -34,7 +35,9 @@ func TestEndpointRoute(t *testing.T) {
t.Run("Registered pipelines are valid routes", func(t *testing.T) { t.Run("Registered pipelines are valid routes", func(t *testing.T) {
router := mux.NewRouter() router := mux.NewRouter()
RegisterPipelineRoutes(router, []pipelines.IPipeline{data})
fs := storage.GetMemoryStorageProvider()
RegisterPipelineRoutes(router, []pipelines.IPipeline{data}, fs)
request, _ := http.NewRequest("GET", "/"+data.Slug, nil) request, _ := http.NewRequest("GET", "/"+data.Slug, nil)
responseRecorder := httptest.NewRecorder() responseRecorder := httptest.NewRecorder()

58
pipelines/executable_step.go

@ -1,7 +1,14 @@
package pipelines package pipelines
import (
"errors"
"fmt"
"github.com/disintegration/imaging"
"image"
)
type IExecutableStep interface { type IExecutableStep interface {
Execute()
Execute(src image.Image) (image.Image, error)
} }
// Resize image // Resize image
@ -15,19 +22,54 @@ type ResizeImageStep struct {
} `json:"options"` } `json:"options"`
} }
func (s ResizeImageStep) Execute() {
// TODO
func (s ResizeImageStep) Execute(src image.Image) (image.Image, error) {
src = imaging.Resize(src, s.Options.Width, s.Options.Height, imaging.Linear)
return src, nil
}
// Rotate image
type RotateImageStep struct {
Step
Options struct {
Angle float64 `json:"angle"`
} `json:"options"`
}
func (s RotateImageStep) Execute(src image.Image) (image.Image, error) {
src = imaging.Rotate(src, s.Options.Angle, image.Black)
return src, nil
} }
// Compress image
// Flip image
type CompressImageStep struct {
type FlipImageStep struct {
Step Step
Options struct { Options struct {
Quality int `json:"quality"`
Direction string `json:"direction"`
} `json:"options"` } `json:"options"`
} }
func (s CompressImageStep) Execute() {
// TODO
func (s FlipImageStep) Execute(src image.Image) (image.Image, error) {
switch s.Options.Direction {
case "h":
src = imaging.FlipH(src)
case "v":
src = imaging.FlipH(src)
default:
return src, errors.New(fmt.Sprintf("invalid flip direction: %s", s.Options.Direction))
}
return src, nil
}
// Grayscale image
type GrayscaleImageStep struct {
Step
}
func (s GrayscaleImageStep) Execute(src image.Image) (image.Image, error) {
src = imaging.Grayscale(src)
return src, nil
} }

15
pipelines/executable_step_test.go

@ -32,18 +32,15 @@ func TestDeserializeOptionsResizeImage(t *testing.T) {
}) })
} }
func TestDeserializeOptionsCompressImage(t *testing.T) {
func TestDeserializeMissingOptions(t *testing.T) {
const Payload string = `{ const Payload string = `{
"name": "example pipeline", "name": "example pipeline",
"type": 0, "type": 0,
"removeMetadata": false, "removeMetadata": false,
"steps": [ "steps": [
{ {
"name": "compress image",
"type": 1,
"options": {
"quality": 80
}
"name": "resize image",
"type": 0
} }
] ]
}` }`
@ -53,11 +50,11 @@ func TestDeserializeOptionsCompressImage(t *testing.T) {
_, err := values[0].GetSteps()[0].GetExecutable() _, err := values[0].GetSteps()[0].GetExecutable()
assert.Equal(t, nil, err)
assert.EqualError(t, err, "unexpected end of JSON input")
}) })
} }
func TestDeserializeMissingOptions(t *testing.T) {
func TestLoadingImage(t *testing.T) {
const Payload string = `{ const Payload string = `{
"name": "example pipeline", "name": "example pipeline",
"type": 0, "type": 0,
@ -70,7 +67,7 @@ func TestDeserializeMissingOptions(t *testing.T) {
] ]
}` }`
t.Run("Image pipeline deserialization is successful", func(t *testing.T) {
t.Run("Loading image from filesystem to pipeline is successful", func(t *testing.T) {
values := DeserializePipelines([][]byte{[]byte(Payload)}) values := DeserializePipelines([][]byte{[]byte(Payload)})
_, err := values[0].GetSteps()[0].GetExecutable() _, err := values[0].GetSteps()[0].GetExecutable()

52
pipelines/pipeline.go

@ -1,8 +1,12 @@
package pipelines package pipelines
import ( import (
"bytes"
"encoding/json" "encoding/json"
"errors"
"fmt" "fmt"
"github.com/disintegration/imaging"
"github.com/geplauder/lithium/storage"
"io/fs" "io/fs"
"log" "log"
"os" "os"
@ -23,6 +27,7 @@ type IPipeline interface {
GetSlug() string GetSlug() string
GetType() PipelineType GetType() PipelineType
GetSteps() []Step GetSteps() []Step
Run(string, string, storage.IStorageProvider) (string, error)
} }
type Pipeline struct { type Pipeline struct {
@ -31,6 +36,53 @@ type Pipeline struct {
Type PipelineType `json:"type" faker:"-"` Type PipelineType `json:"type" faker:"-"`
RemoveMetadata bool `json:"remove_metadata" faker:"-"` RemoveMetadata bool `json:"remove_metadata" faker:"-"`
Steps []Step `json:"steps" faker:"-"` Steps []Step `json:"steps" faker:"-"`
Output struct {
Format int `json:"format"`
Quality int `json:"quality"`
} `json:"output" faker:"-"`
}
func (p Pipeline) Run(srcPath, bucketName string, storageProvider storage.IStorageProvider) (string, error) {
fmt.Println("path: ", storageProvider.GetPath(bucketName, srcPath))
src, err := imaging.Open(storageProvider.GetPath(bucketName, srcPath))
if err != nil {
return "", errors.New(fmt.Sprintf("error opening file for processing: %s", err))
}
for _, step := range p.GetSteps() {
runner, err := step.GetExecutable()
if err != nil {
return "", err
}
src, err = runner.Execute(src)
if err != nil {
return "", err
}
}
format := imaging.Format(p.Output.Format)
var options []imaging.EncodeOption
if p.Output.Quality != 0 {
options = append(options, imaging.JPEGQuality(p.Output.Quality))
}
// encode image to io buffer
buffer := new(bytes.Buffer)
if err := imaging.Encode(buffer, src, format, options...); err != nil {
return "", err
}
const fileName = "output.jpg" // TODO make variable
_, err = storageProvider.StoreRaw(bucketName, fileName, buffer.Bytes())
if err != nil {
return "", err
}
return fileName, nil
} }
func (p Pipeline) GetName() string { func (p Pipeline) GetName() string {

361
pipelines/pipeline_test.go

@ -1,38 +1,326 @@
package pipelines package pipelines
import ( import (
"github.com/geplauder/lithium/storage"
"image"
"os"
"path/filepath"
"testing" "testing"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
) )
func TestImagePipelineDeserialization(t *testing.T) {
const Payload string = `{
"name": "example pipeline",
"type": 0,
"removeMetadata": false,
"steps": [
{
"name": "resize image",
"type": 0
},
{
"name": "compress image",
"type": 1
}
]
}`
// pipeline deserialization
func TestPipelineDeserialization(t *testing.T) {
t.Run("Image pipeline deserialization is successful", func(t *testing.T) { t.Run("Image pipeline deserialization is successful", func(t *testing.T) {
const Payload string = `{
"name": "example pipeline",
"type": 0,
"removeMetadata": false,
"steps": [
{
"name": "resize image",
"type": 0
},
{
"name": "compress image",
"type": 1
}
]
}`
values := DeserializePipelines([][]byte{[]byte(Payload)}) values := DeserializePipelines([][]byte{[]byte(Payload)})
assert.Equal(t, 1, len(values), "Output should contain one element") assert.Equal(t, 1, len(values), "Output should contain one element")
assert.Equal(t, "example pipeline", values[0].GetName()) assert.Equal(t, "example pipeline", values[0].GetName())
assert.Equal(t, Image, values[0].GetType()) assert.Equal(t, Image, values[0].GetType())
}) })
t.Run("Video pipelines deserialization is successful", func(t *testing.T) {
const Payload string = `{
"name": "example pipeline",
"type": 1,
"removeMetadata": false,
"steps": [
{
"name": "resize image",
"type": 0
},
{
"name": "compress image",
"type": 1
}
]
}`
values := DeserializePipelines([][]byte{[]byte(Payload)})
assert.Equal(t, 1, len(values), "Output should contain one element")
assert.Equal(t, "example pipeline", values[0].GetName())
assert.Equal(t, Video, values[0].GetType())
})
} }
func TestVideoPipelineDeserialization(t *testing.T) {
// image pipeline steps
func TestExecuteSteps(t *testing.T) {
t.Run("Pipeline executes with no steps", func(t *testing.T) {
const Bucket string = "pipeline_test_01"
const Payload string = `{
"name": "example pipeline",
"type": 1,
"removeMetadata": false,
"steps": []
}`
wd, _ := os.Getwd()
pipe := DeserializePipelines([][]byte{[]byte(Payload)})[0]
storageProvider := storage.GetFileSystemStorageProvider("test", "..")
// copy test file to storage bucket
_, err := storageProvider.StoreExisting(Bucket, "source.jpg", filepath.Join(wd, "../tests/files/900x900.jpg"))
assert.Nil(t, err, "Test file should be readable")
assert.FileExists(t, storageProvider.GetPath(Bucket, "source.jpg"))
// run pipeline steps
dest, err := pipe.Run("source.jpg", Bucket, storageProvider)
assert.Nil(t, err)
assert.FileExists(t, storageProvider.GetPath(Bucket, dest))
// clean up
os.Remove(storageProvider.GetPath(Bucket, "source.jpg"))
os.Remove(storageProvider.GetPath(Bucket, dest))
})
t.Run("Image resizing is successful", func(t *testing.T) {
const Bucket string = "pipeline_test_02"
const Payload string = `{
"name": "example pipeline",
"type": 1,
"removeMetadata": false,
"steps": [
{
"name": "resize image",
"type": 0,
"options": {
"width": 1280,
"height": 720,
"upscale": false
}
}
]
}`
wd, _ := os.Getwd()
pipe := DeserializePipelines([][]byte{[]byte(Payload)})[0]
storageProvider := storage.GetFileSystemStorageProvider("test", "..")
// copy test file to storage bucket
_, err := storageProvider.StoreExisting(Bucket, "source.jpg", filepath.Join(wd, "../tests/files/900x900.jpg"))
assert.Nil(t, err, "Test file should be readable")
assert.FileExists(t, storageProvider.GetPath(Bucket, "source.jpg"))
// run pipeline steps
dest, err := pipe.Run("source.jpg", Bucket, storageProvider)
assert.Nil(t, err)
assert.FileExists(t, storageProvider.GetPath(Bucket, dest))
// read image config
file, err := storageProvider.OpenFile(Bucket, dest)
assert.Nil(t, err)
imgConf, _, err := image.DecodeConfig(file)
assert.Nil(t, err)
assert.Equal(t, 1280, imgConf.Width)
assert.Equal(t, 720, imgConf.Height)
// clean up
os.Remove(storageProvider.GetPath(Bucket, "source.jpg"))
os.Remove(storageProvider.GetPath(Bucket, dest))
})
t.Run("Image rotation step is successful", func(t *testing.T) {
const Bucket string = "pipeline_test_03"
const Payload string = `{
"name": "example pipeline",
"type": 1,
"removeMetadata": false,
"steps": [
{
"name": "rotate image",
"type": 1,
"options": {
"angle": 90.0
}
}
]
}`
wd, _ := os.Getwd()
pipe := DeserializePipelines([][]byte{[]byte(Payload)})[0]
storageProvider := storage.GetFileSystemStorageProvider("test", "..")
// copy test file to storage bucket
_, err := storageProvider.StoreExisting(Bucket, "source.jpg", filepath.Join(wd, "../tests/files/800x500.jpg"))
assert.Nil(t, err, "Test file should be readable")
assert.FileExists(t, storageProvider.GetPath(Bucket, "source.jpg"))
// run pipeline steps
dest, err := pipe.Run("source.jpg", Bucket, storageProvider)
assert.Nil(t, err)
assert.FileExists(t, storageProvider.GetPath(Bucket, dest))
// read image config
file, err := storageProvider.OpenFile(Bucket, dest)
assert.Nil(t, err)
imgConf, _, err := image.DecodeConfig(file)
assert.Nil(t, err)
assert.Equal(t, 500, imgConf.Width)
assert.Equal(t, 800, imgConf.Height)
// clean up
os.Remove(storageProvider.GetPath(Bucket, "source.jpg"))
os.Remove(storageProvider.GetPath(Bucket, dest))
})
t.Run("Image flip step is successful", func(t *testing.T) {
const Bucket string = "pipeline_test_06"
const Payload string = `{
"name": "example pipeline",
"type": 1,
"removeMetadata": false,
"steps": [
{
"name": "flip image",
"type": 2,
"options": {
"direction": "h"
}
}
]
}`
wd, _ := os.Getwd()
pipe := DeserializePipelines([][]byte{[]byte(Payload)})[0]
storageProvider := storage.GetFileSystemStorageProvider("test", "..")
// copy test file to storage bucket
_, err := storageProvider.StoreExisting(Bucket, "source.jpg", filepath.Join(wd, "../tests/files/800x500.jpg"))
assert.Nil(t, err, "Test file should be readable")
assert.FileExists(t, storageProvider.GetPath(Bucket, "source.jpg"))
// run pipeline steps
dest, err := pipe.Run("source.jpg", Bucket, storageProvider)
assert.Nil(t, err)
assert.FileExists(t, storageProvider.GetPath(Bucket, dest))
// read image config
file, err := storageProvider.OpenFile(Bucket, dest)
assert.Nil(t, err)
imgConf, _, err := image.DecodeConfig(file)
assert.Nil(t, err)
assert.Equal(t, 800, imgConf.Width)
assert.Equal(t, 500, imgConf.Height)
// clean up
os.Remove(storageProvider.GetPath(Bucket, "source.jpg"))
os.Remove(storageProvider.GetPath(Bucket, dest))
})
t.Run("Image flip step direction validation is successful", func(t *testing.T) {
const Bucket string = "pipeline_test_06"
const Payload string = `{
"name": "example pipeline",
"type": 1,
"removeMetadata": false,
"steps": [
{
"name": "flip image",
"type": 2,
"options": {
"direction": "f"
}
}
]
}`
wd, _ := os.Getwd()
pipe := DeserializePipelines([][]byte{[]byte(Payload)})[0]
storageProvider := storage.GetFileSystemStorageProvider("test", "..")
// copy test file to storage bucket
_, err := storageProvider.StoreExisting(Bucket, "source.jpg", filepath.Join(wd, "../tests/files/800x500.jpg"))
assert.Nil(t, err, "Test file should be readable")
assert.FileExists(t, storageProvider.GetPath(Bucket, "source.jpg"))
// run pipeline steps
_, err = pipe.Run("source.jpg", Bucket, storageProvider)
assert.EqualError(t, err, "invalid flip direction: f")
// clean up
os.Remove(storageProvider.GetPath(Bucket, "source.jpg"))
})
t.Run("Image grayscale step is successful", func(t *testing.T) {
const Bucket string = "pipeline_test_05"
const Payload string = `{
"name": "example pipeline",
"type": 1,
"removeMetadata": false,
"steps": [
{
"name": "grayscale",
"type": 3
}
]
}`
wd, _ := os.Getwd()
pipe := DeserializePipelines([][]byte{[]byte(Payload)})[0]
storageProvider := storage.GetFileSystemStorageProvider("test", "..")
// copy test file to storage bucket
_, err := storageProvider.StoreExisting(Bucket, "source.jpg", filepath.Join(wd, "../tests/files/900x900.jpg"))
assert.Nil(t, err, "Test file should be readable")
assert.FileExists(t, storageProvider.GetPath(Bucket, "source.jpg"))
// run pipeline steps
dest, err := pipe.Run("source.jpg", Bucket, storageProvider)
assert.Nil(t, err)
assert.FileExists(t, storageProvider.GetPath(Bucket, dest))
// read image config
file, err := storageProvider.OpenFile(Bucket, dest)
assert.Nil(t, err)
imgConf, _, err := image.DecodeConfig(file)
assert.Nil(t, err)
assert.Equal(t, 900, imgConf.Width)
assert.Equal(t, 900, imgConf.Height)
// clean up
os.Remove(storageProvider.GetPath(Bucket, "source.jpg"))
os.Remove(storageProvider.GetPath(Bucket, dest))
})
}
// output options
func TestEncoding(t *testing.T) {
const Bucket string = "pipeline_test_04"
const Payload string = `{ const Payload string = `{
"name": "example pipeline", "name": "example pipeline",
"type": 1, "type": 1,
@ -40,20 +328,37 @@ func TestVideoPipelineDeserialization(t *testing.T) {
"steps": [ "steps": [
{ {
"name": "resize image", "name": "resize image",
"type": 0
},
{
"name": "compress image",
"type": 1
"type": 0,
"options": {
"width": 1280,
"height": 720,
"upscale": false
}
} }
]
],
"output": {
"quality": 50
}
}` }`
t.Run("Video pipelines deserialization is successful", func(t *testing.T) {
values := DeserializePipelines([][]byte{[]byte(Payload)})
t.Run("Image encoding with jpeg quality is successful", func(t *testing.T) {
wd, _ := os.Getwd()
pipe := DeserializePipelines([][]byte{[]byte(Payload)})[0]
assert.Equal(t, 1, len(values), "Output should contain one element")
assert.Equal(t, "example pipeline", values[0].GetName())
assert.Equal(t, Video, values[0].GetType())
storageProvider := storage.GetFileSystemStorageProvider("test", "..")
// copy test file to storage bucket
_, err := storageProvider.StoreExisting(Bucket, "source.jpg", filepath.Join(wd, "../tests/files/900x900.jpg"))
assert.Nil(t, err, "Test file should be readable")
assert.FileExists(t, storageProvider.GetPath(Bucket, "source.jpg"))
// run pipeline steps
dest, err := pipe.Run("source.jpg", Bucket, storageProvider)
assert.Nil(t, err)
assert.FileExists(t, storageProvider.GetPath(Bucket, dest))
// clean up
os.Remove(storageProvider.GetPath(Bucket, "source.jpg"))
os.Remove(storageProvider.GetPath(Bucket, dest))
}) })
} }

19
pipelines/step.go

@ -9,7 +9,9 @@ type StepType int
const ( const (
TypeResizeImageStep StepType = iota TypeResizeImageStep StepType = iota
TypeCompressImageStep
TypeRotateImageStep
TypeFlipImageStep
TypeGrayscaleImageStep
) )
type Step struct { type Step struct {
@ -20,6 +22,7 @@ type Step struct {
func (s Step) GetExecutable() (IExecutableStep, error) { func (s Step) GetExecutable() (IExecutableStep, error) {
switch s.GetType() { switch s.GetType() {
case TypeResizeImageStep: case TypeResizeImageStep:
step := ResizeImageStep{} step := ResizeImageStep{}
if err := json.Unmarshal(s.Options, &step.Options); err != nil { if err := json.Unmarshal(s.Options, &step.Options); err != nil {
@ -27,12 +30,22 @@ func (s Step) GetExecutable() (IExecutableStep, error) {
} }
return step, nil return step, nil
case TypeCompressImageStep:
step := CompressImageStep{}
case TypeRotateImageStep:
step := RotateImageStep{}
if err := json.Unmarshal(s.Options, &step.Options); err != nil { if err := json.Unmarshal(s.Options, &step.Options); err != nil {
return nil, err return nil, err
} }
return step, nil return step, nil
case TypeFlipImageStep:
step := FlipImageStep{}
if err := json.Unmarshal(s.Options, &step.Options); err != nil {
return nil, err
}
return step, nil
case TypeGrayscaleImageStep:
return GrayscaleImageStep{}, nil
} }
return nil, errors.New("invalid type") return nil, errors.New("invalid type")

61
settings/settings.go

@ -0,0 +1,61 @@
package settings
import (
"encoding/json"
"os"
"path/filepath"
"github.com/spf13/afero"
)
const (
Local FileSystemType = iota
)
type FileSystemType int
type Settings struct {
Endpoint string `json:"endpoint"`
Token string `json:"token"`
StorageProvider StorageSettings `json:"storage_provider"`
}
type StorageSettings struct {
Type FileSystemType `json:"type"`
BasePath string `json:"base_path"`
}
func parseSettings(data []byte) Settings {
settings := Settings{}
json.Unmarshal(data, &settings)
return settings
}
func LoadSettings(fileSystem afero.Fs) Settings {
workingDirectory, _ := os.Getwd()
path := filepath.Join(workingDirectory, "settings.json")
// Load file and parse file
data, err := afero.ReadFile(fileSystem, path)
if err == nil {
return parseSettings(data)
}
// If file does not exist, create default settings
defaultSettings := Settings{
Endpoint: "127.0.0.1:8000",
Token: "changeme",
StorageProvider: StorageSettings{
Type: Local,
BasePath: "assets",
},
}
serializedSettings, err := json.MarshalIndent(defaultSettings, "", "\t")
afero.WriteFile(fileSystem, path, serializedSettings, os.ModePerm)
return defaultSettings
}

48
settings/settings_test.go

@ -0,0 +1,48 @@
package settings
import (
"os"
"path/filepath"
"testing"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
)
func TestSettingsParsing(t *testing.T) {
const file string = `{
"endpoint": "0.0.0.0:8000",
"token": "foobar",
"storage_provider": {
"type": 0,
"base_path": "assets"
}
}`
t.Run("Settings parsing is successful", func(t *testing.T) {
settings := parseSettings([]byte(file))
assert.Equal(t, "0.0.0.0:8000", settings.Endpoint)
assert.Equal(t, "foobar", settings.Token)
assert.Equal(t, "assets", settings.StorageProvider.BasePath)
})
}
func TestSettingsLoading(t *testing.T) {
t.Run("Settings loading creates default settings.json when none is present", func(t *testing.T) {
fileSystem := afero.NewMemMapFs()
workingDirectory, _ := os.Getwd()
path := filepath.Join(workingDirectory, "settings.json")
// Settings file does not exist in the beginning
doesFileExist, _ := afero.Exists(fileSystem, path)
assert.False(t, doesFileExist)
LoadSettings(fileSystem)
// Settings file should be present after calling LoadSettings
doesFileExist, _ = afero.Exists(fileSystem, path)
assert.True(t, doesFileExist)
})
}

57
storage/storage.go

@ -7,30 +7,69 @@ import (
"github.com/spf13/afero" "github.com/spf13/afero"
) )
const StorageFolderName = "files"
type IStorageProvider interface { type IStorageProvider interface {
storeRaw(bucketName string, objectName string, data []byte) string
storeExisting(bucketName string, objectName string, existingFilePath string) string
StoreRaw(bucketName string, objectName string, data []byte) (string, error)
StoreExisting(bucketName string, objectName string, existingFilePath string) (string, error)
GetPath(bucketName string, objectName string) string
} }
type FileSystemStorageProvider struct { type FileSystemStorageProvider struct {
fileSystem afero.Fs fileSystem afero.Fs
basePath string basePath string
wd string
} }
func (sp FileSystemStorageProvider) storeRaw(bucketName string, objectName string, data []byte) string {
func (sp FileSystemStorageProvider) StoreRaw(bucketName string, objectName string, data []byte) (string, error) {
directoryPath := filepath.Join(sp.basePath, bucketName) directoryPath := filepath.Join(sp.basePath, bucketName)
sp.fileSystem.MkdirAll(directoryPath, os.ModePerm)
if err := sp.fileSystem.MkdirAll(directoryPath, os.ModePerm); err != nil {
return "", err
}
filePath := filepath.Join(directoryPath, objectName) filePath := filepath.Join(directoryPath, objectName)
afero.WriteFile(sp.fileSystem, filePath, data, os.ModePerm)
if err := afero.WriteFile(sp.fileSystem, filePath, data, os.ModePerm); err != nil {
return "", err
}
return filePath, nil
}
func (sp FileSystemStorageProvider) StoreExisting(bucketName string, objectName string, existingFilePath string) (string, error) {
bytesRead, err := os.ReadFile(existingFilePath)
if err != nil {
return "", err
}
return sp.StoreRaw(bucketName, objectName, bytesRead)
}
func (sp FileSystemStorageProvider) GetPath(bucketName string, objectName string) string {
return filepath.Join(sp.wd, StorageFolderName, sp.basePath, bucketName, objectName)
}
return filePath
func (sp FileSystemStorageProvider) OpenFile(bucketName string, objectName string) (*os.File, error) {
return os.Open(sp.GetPath(bucketName, objectName))
} }
func (sp FileSystemStorageProvider) storeExisting(bucketName string, objectName string, existingFilePath string) string {
bytesRead, _ := afero.ReadFile(sp.fileSystem, existingFilePath)
func GetFileSystemStorageProvider(basePath string, wd string) FileSystemStorageProvider {
if wd == "" {
wd, _ = os.Getwd()
}
return FileSystemStorageProvider{
fileSystem: afero.NewBasePathFs(afero.NewOsFs(), filepath.Join(wd, StorageFolderName)),
basePath: basePath,
wd: wd,
}
}
return sp.storeRaw(bucketName, objectName, bytesRead)
// TODO: Move this out of this file
func GetMemoryStorageProvider() FileSystemStorageProvider {
return FileSystemStorageProvider{
fileSystem: afero.NewBasePathFs(afero.NewMemMapFs(), "/"),
basePath: "/tmp/foo/bar",
}
} }

23
storage/storage_test.go

@ -19,7 +19,8 @@ func TestFileSystemStorageProvider(t *testing.T) {
basePath: "/tmp/foo/bar", basePath: "/tmp/foo/bar",
} }
finalPath := provider.storeRaw("test", "test.bin", dummyData)
finalPath, err := provider.StoreRaw("test", "test.bin", dummyData)
assert.Nil(t, err)
assert.Equal(t, "/tmp/foo/bar/test/test.bin", finalPath) assert.Equal(t, "/tmp/foo/bar/test/test.bin", finalPath)
exists, _ := afero.Exists(fileSystem, "/tmp/foo/bar/test/test.bin") exists, _ := afero.Exists(fileSystem, "/tmp/foo/bar/test/test.bin")
@ -32,14 +33,16 @@ func TestFileSystemStorageProvider(t *testing.T) {
t.Run("storeExisting method stores files in filesystem", func(t *testing.T) { t.Run("storeExisting method stores files in filesystem", func(t *testing.T) {
fileSystem := afero.NewMemMapFs() fileSystem := afero.NewMemMapFs()
afero.WriteFile(fileSystem, "/tmp/existing.bin", dummyData, os.ModePerm)
err := os.WriteFile("/tmp/existing.bin", dummyData, os.ModePerm)
assert.Nil(t, err)
provider := FileSystemStorageProvider{ provider := FileSystemStorageProvider{
fileSystem: fileSystem, fileSystem: fileSystem,
basePath: "/tmp/foo/bar", basePath: "/tmp/foo/bar",
} }
finalPath := provider.storeExisting("test", "test.bin", "/tmp/existing.bin")
finalPath, err := provider.StoreExisting("test", "test.bin", "/tmp/existing.bin")
assert.Nil(t, err)
assert.Equal(t, "/tmp/foo/bar/test/test.bin", finalPath) assert.Equal(t, "/tmp/foo/bar/test/test.bin", finalPath)
exists, _ := afero.Exists(fileSystem, "/tmp/foo/bar/test/test.bin") exists, _ := afero.Exists(fileSystem, "/tmp/foo/bar/test/test.bin")
@ -48,4 +51,18 @@ func TestFileSystemStorageProvider(t *testing.T) {
content, _ := afero.ReadFile(fileSystem, "/tmp/foo/bar/test/test.bin") content, _ := afero.ReadFile(fileSystem, "/tmp/foo/bar/test/test.bin")
assert.Equal(t, dummyData, content) assert.Equal(t, dummyData, content)
}) })
t.Run("getPath method returns correct path", func(t *testing.T) {
fileSystem := afero.NewMemMapFs()
provider := FileSystemStorageProvider{
fileSystem: fileSystem,
basePath: "/tmp/foo/bar",
}
_, err := provider.StoreRaw("test", "test.bin", dummyData)
assert.Nil(t, err)
assert.Equal(t, "files/tmp/foo/bar/test/test.bin", provider.GetPath("test", "test.bin"))
})
} }

BIN
tests/files/800x500.jpg

Before

Width: 800  |  Height: 500  |  Size: 66 KiB

After

Width: 800  |  Height: 500  |  Size: 66 KiB

BIN
tests/files/900x900.jpg

Before

Width: 900  |  Height: 900  |  Size: 128 KiB

After

Width: 900  |  Height: 900  |  Size: 128 KiB

Loading…
Cancel
Save