Show slug instead of name for pipelines in index metadata

.gitignore

@@ -12,4 +12,9 @@
 *.out
 
 # Go workspace file
-go.work
+go.work
+
+# Lithium specific
+settings.json
+lithium.log
+files/

README.md

@@ -0,0 +1,297 @@
+# Lithium
+
+Micro-service for file storage and processing written in Go.
+
+## Features
+
+- Image processing pipelines for various transformations
+- Web API for storing & retrieving files
+
+## Requirements
+
+- [Go 1.17+](https://go.dev/)
+- [_Docker_](https://docs.docker.com/) (optional)
+
+## Setup
+
+#### 1. Clone repository
+
+```shell
+git clone git@gogs.informatik.hs-fulda.de:FabianVowie/Lithium.git
+```
+
+#### 2. Pull dependencies
+
+```shell
+go get
+```
+
+#### 3. Build & start application
+
+```shell
+go run .
+```
+
+**Run using [Docker](https://docs.docker.com/) container**
+
+```shell
+docker run --rm -p 8000:8000 -v "$PWD":/usr/src/lithium -w /usr/src/lithium golang:1.17 go run .
+```
+
+## Testing
+
+```shell
+go test ./...
+```
+
+### Run tests in verbose logging mode
+
+```shell
+go test ./... -v
+```
+
+### Run tests in [Docker](https://docs.docker.com/) container
+
+```shell
+docker run --rm -v "$PWD":/usr/src/lithium -w /usr/src/lithium golang:1.17 go test ./...
+```
+
+## Configuration
+
+Config options can be adjusted via the [`settings.json`](settings.json) file in the root directory.
+
+```json
+{
+  "endpoint": "0.0.0.0:8000",
+  "token": "changeme",
+  "rate_limiter": {
+    "requests_per_minute": 20,
+    "allowed_burst": 5
+  },
+  "storage_provider": {
+    "type": 0,
+    "base_path": "assets"
+  }
+}
+```
+
+## Rate Limiting
+
+By default, the rate limiting takes place on a per-route basis. When the limit for a specific route is hit, the response will return a status code `429: Too Many Requests`.
+
+| Headers               | Explanation                      |
+| --------------------- | -------------------------------- |
+| X-Ratelimit-Limit     | Allowed requests per minute      |
+| X-Ratelimit-Remaining | Remaining requests               |
+| X-Ratelimit-Reset     | Seconds until requests replenish |
+
+## API
+
+### `GET` `/`
+
+Show application information.
+
+**Required headers**:
+
+```shell
+Authorization: Bearer <Token>
+```
+
+**Example response**:
+
+```json
+{
+  "name": "Lithium",
+  "version": "0.1.0"
+}
+```
+
+### `GET` `/pipelines/{pipeline}`
+
+Show pipeline information.
+
+**Required headers**:
+
+```shell
+Authorization: Bearer <Token>
+```
+
+**Example response**:
+
+```json
+{
+  "name": "example pipeline",
+  "slug": "example",
+  "type": 0,
+  "remove_metadata": false,
+  "steps": [
+    {
+      "name": "resize image",
+      "type": 0,
+      "options": {
+        "width": 1280,
+        "height": 720,
+        "upscale": false
+      }
+    }
+  ],
+  "output": {
+    "format": "jpg",
+    "quality": 90
+  }
+}
+```
+
+## Pipelines
+
+The project uses a pipeline system defined by [JSON](https://en.wikipedia.org/wiki/JSON) files located in the [config](config) folder.
+Take a look at the [example pipeline configuration file](config/example.json).
+
+Available pipeline `type`s: `0` (Image), `1` (Video)
+
+```json
+{
+  "name": "example pipeline",
+  "slug": "example",
+  "type": 0,
+  "removeMetadata": false,
+  "steps": [],
+  "output": {}
+}
+```
+
+### Images pipeline
+
+The image pipeline offers the following additional output options.
+
+Available `format` types: `jpg` (or `jpeg`), `png`, `gif`, `tif` (or `tiff`) and `bmp`  
+The `quality` field can contain any integer between 1 and 100.
+
+```json
+{
+  "output": {
+    "format": "jpg",
+    "quality": 90
+  }
+}
+```
+
+### Available pipeline steps
+
+Each pipeline step consists of an optional `name`, a predefined `type` and configurable `options`.
+See the available options below for more information.
+
+```json
+{
+  "name": "step name",
+  "type": 0,
+  "options": {}
+}
+```
+
+#### Resizing images
+
+Resize an image by a given `width` and `height`.
+
+**Step definition**:
+
+```json
+{
+  "type": 0,
+  "options": {
+    "width": 1280,
+    "height": 720
+  }
+}
+```
+
+#### Rotating images
+
+Rotate an image with a given `angle` in degrees.
+
+**Step definition**:
+
+```json
+{
+  "type": 1,
+  "options": {
+    "angle": 90.0
+  }
+}
+```
+
+#### Flipping images
+
+Flip an image with a given direction.
+Allowed values for the `direction` option are `"h"` (horizontal), `"v"` (vertical)
+
+**Step definition**:
+
+```json
+{
+  "type": 2,
+  "options": {
+    "direction": "h"
+  }
+}
+```
+
+#### Grayscale
+
+Convert the colorspace of an image into grayscale.
+
+**Step definition**:
+
+```json
+{
+  "type": 3
+}
+```
+
+#### Fit
+
+Scales down the image to fit the specified maximum width and height.
+
+**Step definition**:
+
+```json
+{
+  "type": 4,
+  "options": {
+    "height": 300,
+    "width": 200
+  }
+}
+```
+
+#### Invert
+
+Invert image colors.
+
+**Step definition**:
+
+```json
+{
+  "type": 5
+}
+```
+
+#### Blur
+
+Blur image using Gaussian functions.
+
+**Step definition**:
+
+```json
+{
+  "type": 6,
+  "options": {
+    "sigma": 50.0
+  }
+}
+```
+
+## Authors
+
+- [Fabian Vowie](https://gogs.informatik.hs-fulda.de/FabianVowie)
+- [Roman Zipp](https://gogs.informatik.hs-fulda.de/roman.zipp)

build.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+GIT_COMMIT=$(git rev-parse --short HEAD); go build -ldflags "-X main.GitCommit=$GIT_COMMIT"

config/example.json

@@ -14,11 +14,26 @@
             }
         },
         {
-            "name": "compress image",
+            "name": "rotate image",
             "type": 1,
             "options": {
-                "quality": 80
+                "angle": 90.0
             }
+        },
+        {
+            "name": "flip image",
+            "type": 2,
+            "options": {
+                "direction": "h"
+            }
+        },
+        {
+            "name": "grayscale",
+            "type": 3
         }
-    ]
+    ],
+    "output": {
+        "format": "jpg",
+        "quality": 90
+    }
 }

go.mod

@@ -4,17 +4,23 @@ go 1.17
 
 require (
 	github.com/bxcodec/faker/v3 v3.7.0
+	github.com/disintegration/imaging v1.6.2
 	github.com/gorilla/mux v1.8.0
+	github.com/sirupsen/logrus v1.8.1
 	github.com/spf13/afero v1.8.0
 	github.com/stretchr/testify v1.7.0
+	github.com/throttled/throttled v2.2.5+incompatible
+	github.com/throttled/throttled/v2 v2.9.0
 )
 
 require (
-	github.com/davecgh/go-spew v1.1.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/hashicorp/golang-lru v0.5.4 // indirect
 	github.com/kr/pretty v0.3.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rogpeppe/go-internal v1.8.1-0.20211023094830-115ce09fd6b4 // indirect
-	golang.org/x/text v0.3.4 // indirect
-	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
+	golang.org/x/image v0.0.0-20211028202545-6944b10bf410 // indirect
+	golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 // indirect
+	golang.org/x/text v0.3.6 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
 )

go.sum

@@ -49,17 +49,22 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX
 github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
-github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c=
+github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-redis/redis v6.15.8+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -85,6 +90,8 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq
 github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/gomodule/redigo v1.8.4 h1:Z5JUg94HMTR1XpwBaSH4vq3+PNSIykBLxMdglbw10gg=
+github.com/gomodule/redigo v1.8.4/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
@@ -118,6 +125,9 @@ github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
+github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
@@ -131,6 +141,9 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg=
@@ -141,13 +154,20 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
 github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
 github.com/rogpeppe/go-internal v1.8.1-0.20211023094830-115ce09fd6b4 h1:Ha8xCaq6ln1a+R91Km45Oq6lPXj2Mla6CRJYcuV2h1w=
 github.com/rogpeppe/go-internal v1.8.1-0.20211023094830-115ce09fd6b4/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o=
+github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
+github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/spf13/afero v1.8.0 h1:5MmtuhAgYeU6qpa7w7bP0dv6MBYuup0vekhSpSkoq60=
 github.com/spf13/afero v1.8.0/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/throttled/throttled v2.2.5+incompatible h1:65UB52X0qNTYiT0Sohp8qLYVFwZQPDw85uSa65OljjQ=
+github.com/throttled/throttled v2.2.5+incompatible/go.mod h1:0BjlrEGQmvxps+HuXLsyRdqpSRvJpq0PNIsOtqP9Nos=
+github.com/throttled/throttled/v2 v2.9.0 h1:DOkCb1el7NYzRoPb1pyeHVghsUoonVWEjmo34vrcp/8=
+github.com/throttled/throttled/v2 v2.9.0/go.mod h1:0JHxhGAidPyqbgD4HF8Y1sNFfG0ffVXK6C8EpkNdLEM=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -177,6 +197,9 @@ golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EH
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/image v0.0.0-20211028202545-6944b10bf410 h1:hTftEOvwiOq2+O8k2D5/Q7COC7k5Qcrgc2TFURJYnvQ=
+golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -200,6 +223,7 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@@ -210,6 +234,7 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -249,6 +274,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -258,6 +284,8 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -281,6 +309,7 @@ golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -288,8 +317,9 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc=
 golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -437,7 +467,11 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

lithium.md

@@ -1,16 +0,0 @@
-# Lithium
-
-Micro-service for file storage and processing.
-
-## Features
-
-- File storing with various providers
-  - S3 (or S3 compatible like MinIO)
-  - Locally (for development purposes)
-- File processing "pipelines" for various formats
-  - Compression, Resizing for images
-  - Encoding for videos
-  - Remove metadata (e.g. EXIF)
-- File-based configuration for pipelines
-  - JSON/YAML/TOML?
-- Web api to store and retrieve files

main.go

@@ -1,22 +1,34 @@
 package main
 
 import (
+	"bytes"
 	"encoding/json"
-	"github.com/geplauder/lithium/pipelines"
+	"io"
 	"net/http"
+	"os"
 
+	"github.com/geplauder/lithium/middlewares"
+	"github.com/geplauder/lithium/pipelines"
+	"github.com/geplauder/lithium/settings"
+	"github.com/geplauder/lithium/storage"
 	"github.com/gorilla/mux"
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/afero"
 )
 
 const Name string = "Lithium"
 const Version string = "0.1.0"
 
+var GitCommit string
+
 type Metadata struct {
-	Name    string
-	Version string
+	Name       string   `json:"name"`
+	Version    string   `json:"version"`
+	CommitHash string   `json:"commit_hash"`
+	Pipelines  []string `json:"pipelines"`
 }
 
-func PipelineHandler(pipeline pipelines.IPipeline, w http.ResponseWriter, r *http.Request) {
+func PipelineHandler(pipeline pipelines.IPipeline, storageProvider storage.IStorageProvider, w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "application/json")
 	err := json.NewEncoder(w).Encode(pipeline)
 	if err != nil {
@@ -24,32 +36,182 @@ func PipelineHandler(pipeline pipelines.IPipeline, w http.ResponseWriter, r *htt
 	}
 }
 
-func IndexHandler(w http.ResponseWriter, r *http.Request) {
+func IndexHandler(pipelines []pipelines.IPipeline, w http.ResponseWriter, r *http.Request) {
+	var pipelineNames []string
+	for _, x := range pipelines {
+		pipelineNames = append(pipelineNames, x.GetSlug())
+	}
+
 	w.Header().Set("Content-Type", "application/json")
-	err := json.NewEncoder(w).Encode(Metadata{Name, Version})
+	err := json.NewEncoder(w).Encode(Metadata{Name, Version, GitCommit, pipelineNames})
 	if err != nil {
 		w.WriteHeader(http.StatusInternalServerError)
 	}
 }
 
-func RegisterPipelineRoutes(r *mux.Router, pipelines []pipelines.IPipeline) {
-	for _, pipeline := range pipelines {
-		r.HandleFunc("/"+pipeline.GetSlug(), func(w http.ResponseWriter, r *http.Request) {
-			PipelineHandler(pipeline, w, r)
-		})
+func writeError(w http.ResponseWriter, status int, errStr string) {
+	w.WriteHeader(status)
+	json.NewEncoder(w).Encode(struct {
+		Error string `json:"error"`
+	}{errStr})
+}
+
+func UploadHandler(w http.ResponseWriter, r *http.Request, pipes []pipelines.IPipeline, storageProvider storage.IStorageProvider) {
+	// open file handler
+	formFile, handler, err := r.FormFile("file")
+	if err != nil {
+		writeError(w, http.StatusUnprocessableEntity, err.Error())
+		return
+	}
+
+	defer formFile.Close()
+
+	// check pipelines form param
+	formPipeline := r.FormValue("pipeline")
+	if formPipeline == "" {
+		writeError(w, http.StatusUnprocessableEntity, "pipeline parameter missing")
+		return
+	}
+
+	var execPipe pipelines.IPipeline
+	for _, pipe := range pipes {
+		if formPipeline == pipe.GetSlug() {
+			execPipe = pipe
+			break
+		}
+	}
+
+	if execPipe == nil {
+		writeError(w, http.StatusUnprocessableEntity, "pipeline not found")
+		return
+	}
+
+	bucket := r.FormValue("bucket")
+	if bucket == "" {
+		writeError(w, http.StatusUnprocessableEntity, "bucket parameter missing")
+		return
+	}
+
+	// open file
+	file, err := handler.Open()
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, "error reading uploaded file")
+		return
+	}
+
+	defer file.Close()
+
+	// read file to buffer
+	buf := bytes.NewBuffer(nil)
+	_, err = io.Copy(buf, file)
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, "error reading file from buffer")
+		return
+	}
+
+	// store uploaded file
+	_, err = storageProvider.StoreRaw(bucket, "source.jpg", buf.Bytes())
+	if err != nil {
+		return
+	}
+
+	// execute pipeline
+	output, err := execPipe.Run("source.jpg", bucket, storageProvider)
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, "error executing pipeline")
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+
+	err = json.NewEncoder(w).Encode(struct {
+		Message     string   `json:"message"`
+		OutputFiles []string `json:"output_files"`
+	}{"ok", []string{output}})
+
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+	}
+
+	logrus.Info("Pipeline routes registered successfully")
+}
+
+func RegisterRoutes(r *mux.Router, appSettings settings.Settings, pipelines []pipelines.IPipeline, storageProvider storage.IStorageProvider) {
+	index := r.Methods(http.MethodGet).Subrouter()
+	index.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		IndexHandler(pipelines, w, r)
+	})
+
+	upload := r.Methods(http.MethodPost).Subrouter()
+	upload.HandleFunc("/upload", func(w http.ResponseWriter, r *http.Request) {
+		UploadHandler(w, r, pipelines, storageProvider)
+	})
+
+	pipeline := r.Methods(http.MethodGet).Subrouter()
+	pipeline.HandleFunc("/pipelines/{pipeline}", func(w http.ResponseWriter, r *http.Request) {
+		for _, pipeline := range pipelines {
+			if pipeline.GetSlug() == mux.Vars(r)["pipeline"] {
+				PipelineHandler(pipeline, storageProvider, w, r)
+				return
+			}
+		}
+
+		w.WriteHeader(404)
+	})
+
+	if appSettings.Authentication.Enabled {
+		authMiddleware := middlewares.CreateAuthenticationMiddleware(appSettings.Authentication.Token)
+
+		upload.Use(authMiddleware.Middleware)
+		pipeline.Use(authMiddleware.Middleware)
 	}
 }
 
 func main() {
+	logFile, err := os.OpenFile("lithium.log", os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
+	if err == nil {
+		multiWriter := io.MultiWriter(os.Stdout, logFile)
+		logrus.SetOutput(multiWriter)
+	} else {
+		logrus.SetOutput(os.Stdout)
+	}
+
+	logrus.SetFormatter(&logrus.JSONFormatter{})
+
+	appSettings, err := settings.LoadSettings(afero.NewOsFs())
+	if err != nil {
+		logrus.Fatal("Unexpected error while loading settings: ", err)
+	}
+
+	var storageProvider storage.IStorageProvider
+
+	if appSettings.StorageProvider.Type == 0 {
+		storageProvider = storage.GetFileSystemStorageProvider(appSettings.StorageProvider.BasePath, "")
+	} else {
+		logrus.WithFields(logrus.Fields{
+			"StorageProviderType": appSettings.StorageProvider.Type,
+		}).Fatal("Invalid file system provided")
+	}
+
 	pipes := pipelines.LoadPipelines()
 
 	r := mux.NewRouter()
-	r.HandleFunc("/", IndexHandler)
 
-	RegisterPipelineRoutes(r, pipes)
+	if appSettings.RateLimiter.Enabled {
+		rateLimiterMiddleware, err := middlewares.CreateRateLimiterMiddleware(appSettings.RateLimiter.RequestsPerMinute, appSettings.RateLimiter.AllowedBurst)
+		if err != nil {
+			logrus.Fatal("Unexpected error while creating rate limiter middleware: ", err)
+		}
+
+		r.Use(rateLimiterMiddleware.Middleware)
+	}
+
+	RegisterRoutes(r, appSettings, pipes, storageProvider)
+
+	logrus.Info("Lithium started, listening for requests...")
 
-	err := http.ListenAndServe(":8000", r)
+	err = http.ListenAndServe(appSettings.Endpoint, r)
 	if err != nil {
-		panic(err)
+		logrus.Fatal("Unexpected error while serving http server: ", err)
 	}
 }

main_test.go

@@ -1,47 +1,71 @@
 package main
 
 import (
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
-	"github.com/geplauder/lithium/pipelines"
 	"net/http"
 	"net/http/httptest"
 	"testing"
 
 	"github.com/bxcodec/faker/v3"
+	"github.com/geplauder/lithium/pipelines"
+	"github.com/geplauder/lithium/settings"
+	"github.com/geplauder/lithium/storage"
 	"github.com/gorilla/mux"
+	"github.com/spf13/afero"
 	"github.com/stretchr/testify/assert"
 )
 
 func TestIndexRoute(t *testing.T) {
+	data := pipelines.Pipeline{}
+	err := faker.FakeData(&data)
+	assert.Nil(t, err)
+
 	t.Run("Index route returns valid response", func(t *testing.T) {
 		request := httptest.NewRequest(http.MethodGet, "/", nil)
 		responseRecorder := httptest.NewRecorder()
 
-		IndexHandler(responseRecorder, request)
+		IndexHandler([]pipelines.IPipeline{}, responseRecorder, request)
 
-		assert.Equal(t, responseRecorder.Code, 200, "Response code should be 200")
+		assert.Equal(t, 200, responseRecorder.Code, "Response code should be 200")
 		assert.NotNil(t, responseRecorder.Body, "Response should contain body")
 	})
+
+	t.Run("Index route returns valid amount of pipelines", func(t *testing.T) {
+		request := httptest.NewRequest(http.MethodGet, "/", nil)
+		responseRecorder := httptest.NewRecorder()
+
+		IndexHandler([]pipelines.IPipeline{data}, responseRecorder, request)
+
+		assert.Equal(t, 200, responseRecorder.Code, "Response code should be 200")
+
+		var body = Metadata{}
+		err = json.Unmarshal(responseRecorder.Body.Bytes(), &body)
+		assert.Nil(t, err)
+		assert.Equal(t, 1, len(body.Pipelines))
+		assert.Equal(t, data.Slug, body.Pipelines[0])
+	})
 }
 
 func TestEndpointRoute(t *testing.T) {
 	data := pipelines.Pipeline{}
 	err := faker.FakeData(&data)
-	if err != nil {
-		fmt.Println(err)
-	}
+	assert.Nil(t, err)
 
 	t.Run("Registered pipelines are valid routes", func(t *testing.T) {
 		router := mux.NewRouter()
-		RegisterPipelineRoutes(router, []pipelines.IPipeline{data})
+		fs := storage.GetMemoryStorageProvider()
+		appSettings, _ := settings.LoadSettings(afero.NewMemMapFs())
 
-		request, _ := http.NewRequest("GET", "/"+data.Slug, nil)
+		RegisterRoutes(router, appSettings, []pipelines.IPipeline{data}, fs)
+
+		request, _ := http.NewRequest("GET", "/pipelines/"+data.Slug, nil)
 		responseRecorder := httptest.NewRecorder()
 
 		router.ServeHTTP(responseRecorder, request)
 
-		assert.Equal(t, responseRecorder.Code, 200)
+		assert.Equal(t, 200, responseRecorder.Code)
 		body, _ := json.Marshal(data)
 		assert.JSONEq(t, string(body), responseRecorder.Body.String())
 	})
@@ -49,11 +73,71 @@ func TestEndpointRoute(t *testing.T) {
 	t.Run("Unregistered pipelines return 404", func(t *testing.T) {
 		router := mux.NewRouter()
 
-		request, _ := http.NewRequest("GET", "/"+data.Slug, nil)
+		request, _ := http.NewRequest("GET", "/pipelines/"+data.Slug, nil)
+		responseRecorder := httptest.NewRecorder()
+
+		router.ServeHTTP(responseRecorder, request)
+
+		assert.Equal(t, 404, responseRecorder.Code)
+	})
+}
+
+func TestUploadRoute(t *testing.T) {
+	t.Run("Test uploads missing multipart boundary", func(t *testing.T) {
+		router := mux.NewRouter()
+		fs := storage.GetMemoryStorageProvider()
+		appSettings, _ := settings.LoadSettings(afero.NewMemMapFs())
+
+		RegisterRoutes(router, appSettings, []pipelines.IPipeline{pipelines.Pipeline{
+			Name:           "",
+			Slug:           "",
+			Type:           0,
+			RemoveMetadata: false,
+			Steps:          []pipelines.Step{},
+			Output: struct {
+				Format  string `json:"format"`
+				Quality int    `json:"quality"`
+			}{"jpeg", 10},
+		}}, fs)
+
+		request, _ := http.NewRequest("POST", "/upload", nil)
+		request.Header["Content-Type"] = []string{"multipart/form-data"}
+		responseRecorder := httptest.NewRecorder()
+
+		router.ServeHTTP(responseRecorder, request)
+
+		assert.Equal(t, 0x1A6, responseRecorder.Code)
+		str, _ := base64.StdEncoding.DecodeString("eyJlcnJvciI6Im5" +
+			"vIG11bHRpcGFydCBib3VuZGFyeSBwYXJhbSBpbiBDb250ZW50LVR5cGUifQ==")
+		assert.JSONEq(t, string(str), responseRecorder.Body.String())
+	})
+
+	t.Run("Test uploads missing multipart boundary", func(t *testing.T) {
+		router := mux.NewRouter()
+		fs := storage.GetMemoryStorageProvider()
+		appSettings, _ := settings.LoadSettings(afero.NewMemMapFs())
+
+		RegisterRoutes(router, appSettings, []pipelines.IPipeline{pipelines.Pipeline{
+			Name:           "",
+			Slug:           "",
+			Type:           0,
+			RemoveMetadata: false,
+			Steps:          []pipelines.Step{},
+			Output: struct {
+				Format  string `json:"format"`
+				Quality int    `json:"quality"`
+			}{"jpeg", 10},
+		}}, fs)
+
+		request, _ := http.NewRequest("POST", "/upload", nil)
+		request.Header["Content-Type"] = []string{"multipart/form-data", "boundary=X-INSOMNIA-BOUNDARY"}
 		responseRecorder := httptest.NewRecorder()
 
 		router.ServeHTTP(responseRecorder, request)
 
-		assert.Equal(t, responseRecorder.Code, 404)
+		assert.Equal(t, 0x1A6, responseRecorder.Code)
+		str, _ := base64.StdEncoding.DecodeString("eyJlcnJvciI6Im5vIG11bHRpcGFydCBib3VuZGFyeSBwYXJhbSBpbiBDb250ZW50LVR5cGUifQ==")
+		assert.JSONEq(t, string(str), responseRecorder.Body.String())
+		fmt.Println(responseRecorder.Body.String())
 	})
 }

middlewares/authorization.go

@@ -0,0 +1,28 @@
+package middlewares
+
+import (
+	"net/http"
+	"strings"
+)
+
+type AuthenticationMiddleware struct {
+	secret string
+}
+
+func (middleware AuthenticationMiddleware) Middleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		authToken := r.Header.Get("Authorization")
+
+		if authToken == "" || strings.HasPrefix(authToken, "Bearer ") == false || authToken[7:] != middleware.secret {
+			http.Error(w, "Forbidden", http.StatusForbidden)
+		} else {
+			next.ServeHTTP(w, r)
+		}
+	})
+}
+
+func CreateAuthenticationMiddleware(secret string) AuthenticationMiddleware {
+	return AuthenticationMiddleware{
+		secret,
+	}
+}

middlewares/authorization_test.go

@@ -0,0 +1,50 @@
+package middlewares
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/bxcodec/faker/v3"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestAuthenticationMiddleware(t *testing.T) {
+	token := faker.Word()
+	middleware := CreateAuthenticationMiddleware(token)
+
+	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	})
+
+	middlewareHandler := middleware.Middleware(handler)
+
+	t.Run("AuthenticationMiddleware returns 403 response when authorization header is incorrect", func(t *testing.T) {
+		request, _ := http.NewRequest("GET", "/", nil)
+		responseRecorder := httptest.NewRecorder()
+
+		middlewareHandler.ServeHTTP(responseRecorder, request)
+
+		assert.Equal(t, 403, responseRecorder.Code)
+	})
+
+	t.Run("AuthenticationMiddleware returns 403 response when authorization header is missing Bearer prefix", func(t *testing.T) {
+		request, _ := http.NewRequest("GET", "/", nil)
+		request.Header.Set("Authorization", token)
+		responseRecorder := httptest.NewRecorder()
+
+		middlewareHandler.ServeHTTP(responseRecorder, request)
+
+		assert.Equal(t, 403, responseRecorder.Code)
+	})
+
+	t.Run("AuthenticationMiddleware continues when authorization header is correct", func(t *testing.T) {
+		request, _ := http.NewRequest("GET", "/", nil)
+		request.Header.Set("Authorization", "Bearer "+token)
+		responseRecorder := httptest.NewRecorder()
+
+		middlewareHandler.ServeHTTP(responseRecorder, request)
+
+		assert.Equal(t, 200, responseRecorder.Code)
+	})
+}

middlewares/ratelimiter.go

@@ -0,0 +1,43 @@
+package middlewares
+
+import (
+	"net/http"
+
+	"github.com/throttled/throttled/store/memstore"
+	"github.com/throttled/throttled/v2"
+)
+
+type RateLimiterMiddleware struct {
+	rateLimiter throttled.HTTPRateLimiter
+}
+
+func (middleware RateLimiterMiddleware) Middleware(next http.Handler) http.Handler {
+	return middleware.rateLimiter.RateLimit(next)
+}
+
+func CreateRateLimiterMiddleware(requestsPerMinute int, allowedBurst int) (*RateLimiterMiddleware, error) {
+	store, err := memstore.New(65536)
+
+	if err != nil {
+		return nil, err
+	}
+
+	quota := throttled.RateQuota{
+		MaxRate:  throttled.PerMin(requestsPerMinute),
+		MaxBurst: allowedBurst,
+	}
+
+	rateLimiter, err := throttled.NewGCRARateLimiter(store, quota)
+	if err != nil {
+		return nil, err
+	}
+
+	httpRateLimiter := throttled.HTTPRateLimiter{
+		RateLimiter: rateLimiter,
+		VaryBy:      &throttled.VaryBy{Path: true},
+	}
+
+	return &RateLimiterMiddleware{
+		rateLimiter: httpRateLimiter,
+	}, nil
+}

middlewares/ratelimiter_test.go

@@ -0,0 +1,64 @@
+package middlewares
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func ExecuteRequest(middlewareHandler http.Handler) int {
+	request, _ := http.NewRequest("GET", "/", nil)
+	responseRecorder := httptest.NewRecorder()
+
+	middlewareHandler.ServeHTTP(responseRecorder, request)
+
+	return responseRecorder.Code
+}
+
+func TestRateLimiterMiddleware(t *testing.T) {
+	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	})
+
+	t.Run("AuthorizationMiddleware returns 200 response when rate limit is not hit", func(t *testing.T) {
+		middleware, err := CreateRateLimiterMiddleware(1, 0)
+		assert.Nil(t, err)
+
+		middlewareHandler := middleware.Middleware(handler)
+
+		assert.Equal(t, 200, ExecuteRequest(middlewareHandler))
+	})
+
+	t.Run("AuthorizationMiddleware returns 429 response when rate limit is hit", func(t *testing.T) {
+		middleware, err := CreateRateLimiterMiddleware(1, 0)
+		assert.Nil(t, err)
+
+		middlewareHandler := middleware.Middleware(handler)
+
+		assert.Equal(t, 200, ExecuteRequest(middlewareHandler))
+		assert.Equal(t, 429, ExecuteRequest(middlewareHandler))
+	})
+
+	t.Run("AuthorizationMiddleware returns 200 response when rate limit with burst is not hit", func(t *testing.T) {
+		middleware, err := CreateRateLimiterMiddleware(1, 1)
+		assert.Nil(t, err)
+
+		middlewareHandler := middleware.Middleware(handler)
+
+		assert.Equal(t, 200, ExecuteRequest(middlewareHandler))
+		assert.Equal(t, 200, ExecuteRequest(middlewareHandler))
+	})
+
+	t.Run("AuthorizationMiddleware returns 429 response when rate limit with burst is hit", func(t *testing.T) {
+		middleware, err := CreateRateLimiterMiddleware(1, 1)
+		assert.Nil(t, err)
+
+		middlewareHandler := middleware.Middleware(handler)
+
+		assert.Equal(t, 200, ExecuteRequest(middlewareHandler))
+		assert.Equal(t, 200, ExecuteRequest(middlewareHandler))
+		assert.Equal(t, 429, ExecuteRequest(middlewareHandler))
+	})
+}

pipelines/executable_step.go

@@ -1,7 +1,15 @@
 package pipelines
 
+import (
+	"errors"
+	"fmt"
+	"image"
+
+	"github.com/disintegration/imaging"
+)
+
 type IExecutableStep interface {
-	Execute()
+	Execute(src image.Image) (image.Image, error)
 }
 
 // Resize image
@@ -15,19 +23,94 @@ type ResizeImageStep struct {
 	} `json:"options"`
 }
 
-func (s ResizeImageStep) Execute() {
-	// TODO
+func (s ResizeImageStep) Execute(src image.Image) (image.Image, error) {
+	src = imaging.Resize(src, s.Options.Width, s.Options.Height, imaging.Linear)
+	return src, nil
+}
+
+// Rotate image
+
+type RotateImageStep struct {
+	Step
+	Options struct {
+		Angle float64 `json:"angle"`
+	} `json:"options"`
+}
+
+func (s RotateImageStep) Execute(src image.Image) (image.Image, error) {
+	src = imaging.Rotate(src, s.Options.Angle, image.Black)
+	return src, nil
+}
+
+// Flip image
+
+type FlipImageStep struct {
+	Step
+	Options struct {
+		Direction string `json:"direction"`
+	} `json:"options"`
+}
+
+func (s FlipImageStep) Execute(src image.Image) (image.Image, error) {
+	switch s.Options.Direction {
+	case "h":
+		src = imaging.FlipH(src)
+	case "v":
+		src = imaging.FlipH(src)
+	default:
+		return src, errors.New(fmt.Sprintf("invalid flip direction: %s", s.Options.Direction))
+	}
+
+	return src, nil
+}
+
+// Grayscale image
+
+type GrayscaleImageStep struct {
+	Step
+}
+
+func (s GrayscaleImageStep) Execute(src image.Image) (image.Image, error) {
+	src = imaging.Grayscale(src)
+	return src, nil
+}
+
+// Fit image
+
+type FitImageStep struct {
+	Step
+	Options struct {
+		Height int `json:"height"`
+		Width  int `json:"width"`
+	} `json:"options"`
+}
+
+func (s FitImageStep) Execute(src image.Image) (image.Image, error) {
+	src = imaging.Fit(src, s.Options.Width, s.Options.Height, imaging.Lanczos)
+	return src, nil
+}
+
+// Invert image
+
+type InvertImageStep struct {
+	Step
+}
+
+func (s InvertImageStep) Execute(src image.Image) (image.Image, error) {
+	src = imaging.Invert(src)
+	return src, nil
 }
 
-// Compress image
+// Blur image
 
-type CompressImageStep struct {
+type BlurImageStep struct {
 	Step
 	Options struct {
-		Quality int `json:"quality"`
+		Sigma float64 `json:"sigma"`
 	} `json:"options"`
 }
 
-func (s CompressImageStep) Execute() {
-	// TODO
+func (s BlurImageStep) Execute(src image.Image) (image.Image, error) {
+	src = imaging.Blur(src, s.Options.Sigma)
+	return src, nil
 }

pipelines/executable_step_test.go

@@ -1,8 +1,9 @@
 package pipelines
 
 import (
-	"github.com/stretchr/testify/assert"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
 )
 
 func TestDeserializeOptionsResizeImage(t *testing.T) {
@@ -32,18 +33,15 @@ func TestDeserializeOptionsResizeImage(t *testing.T) {
 	})
 }
 
-func TestDeserializeOptionsCompressImage(t *testing.T) {
+func TestDeserializeMissingOptions(t *testing.T) {
 	const Payload string = `{
 		"name": "example pipeline",
 		"type": 0,
 		"removeMetadata": false,
 		"steps": [
 			{
-				"name": "compress image",
-				"type": 1,
-				"options": {
-					"quality": 80
-				}
+				"name": "resize image",
+				"type": 0
 			}
 		]
 	}`
@@ -53,11 +51,11 @@ func TestDeserializeOptionsCompressImage(t *testing.T) {
 
 		_, err := values[0].GetSteps()[0].GetExecutable()
 
-		assert.Equal(t, nil, err)
+		assert.EqualError(t, err, "unexpected end of JSON input")
 	})
 }
 
-func TestDeserializeMissingOptions(t *testing.T) {
+func TestLoadingImage(t *testing.T) {
 	const Payload string = `{
 		"name": "example pipeline",
 		"type": 0,
@@ -70,7 +68,7 @@ func TestDeserializeMissingOptions(t *testing.T) {
 		]
 	}`
 
-	t.Run("Image pipeline deserialization is successful", func(t *testing.T) {
+	t.Run("Loading image from filesystem to pipeline is successful", func(t *testing.T) {
 		values := DeserializePipelines([][]byte{[]byte(Payload)})
 
 		_, err := values[0].GetSteps()[0].GetExecutable()

pipelines/pipeline.go

@@ -1,12 +1,17 @@
 package pipelines
 
 import (
+	"bytes"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io/fs"
-	"log"
 	"os"
 	"path/filepath"
+
+	"github.com/disintegration/imaging"
+	"github.com/geplauder/lithium/storage"
+	"github.com/sirupsen/logrus"
 )
 
 // Pipelines
@@ -23,6 +28,7 @@ type IPipeline interface {
 	GetSlug() string
 	GetType() PipelineType
 	GetSteps() []Step
+	Run(string, string, storage.IStorageProvider) (string, error)
 }
 
 type Pipeline struct {
@@ -31,6 +37,60 @@ type Pipeline struct {
 	Type           PipelineType `json:"type" faker:"-"`
 	RemoveMetadata bool         `json:"remove_metadata" faker:"-"`
 	Steps          []Step       `json:"steps" faker:"-"`
+	Output         struct {
+		Format  string `json:"format"`
+		Quality int    `json:"quality"`
+	} `json:"output" faker:"-"`
+}
+
+func (p Pipeline) Run(srcPath, bucketName string, storageProvider storage.IStorageProvider) (string, error) {
+	src, err := imaging.Open(storageProvider.GetPath(bucketName, srcPath))
+	if err != nil {
+		return "", errors.New(fmt.Sprintf("error opening file for processing: %s", err))
+	}
+
+	for _, step := range p.GetSteps() {
+		runner, err := step.GetExecutable()
+		if err != nil {
+			return "", err
+		}
+
+		src, err = runner.Execute(src)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	outputFormat := p.Output.Format
+	if outputFormat == "" {
+		outputFormat = "jpg"
+	}
+
+	format, err := imaging.FormatFromExtension(outputFormat)
+	if err != nil {
+		return "", errors.New(fmt.Sprintf("output format '%s' is not supported", outputFormat))
+	}
+
+	var options []imaging.EncodeOption
+
+	if p.Output.Quality != 0 {
+		options = append(options, imaging.JPEGQuality(p.Output.Quality))
+	}
+
+	// encode image to io buffer
+	buffer := new(bytes.Buffer)
+	if err := imaging.Encode(buffer, src, format, options...); err != nil {
+		return "", err
+	}
+
+	const fileName = "output.jpg" // TODO make variable
+
+	_, err = storageProvider.StoreRaw(bucketName, fileName, buffer.Bytes())
+	if err != nil {
+		return "", err
+	}
+
+	return fileName, nil
 }
 
 func (p Pipeline) GetName() string {
@@ -66,7 +126,7 @@ func DeserializePipelines(pipelines [][]byte) []IPipeline {
 		var deserializedObject Pipeline
 		err := json.Unmarshal(pipeline, &deserializedObject)
 		if err != nil {
-			log.Fatalf("Could not deserialize pipelines config: %s", err)
+			logrus.Fatalf("Could not deserialize pipelines config: %s", err)
 		}
 		values = append(values, deserializedObject)
 	}
@@ -81,7 +141,6 @@ func LoadPipelines() []IPipeline {
 
 	err := filepath.Walk(path+"/config", func(path string, info fs.FileInfo, err error) error {
 		if err == nil && info.IsDir() == false {
-			fmt.Println(path)
 			data, _ := os.ReadFile(path)
 			files = append(files, data)
 		}
@@ -90,7 +149,7 @@ func LoadPipelines() []IPipeline {
 	})
 
 	if err != nil {
-		panic(err)
+		logrus.Fatal("Unexpected error while loading pipelines: ", err)
 	}
 
 	return DeserializePipelines(files)

pipelines/pipeline_test.go

@@ -1,38 +1,466 @@
 package pipelines
 
 import (
+	"image"
+	"os"
+	"path/filepath"
 	"testing"
 
+	"github.com/geplauder/lithium/storage"
+
 	"github.com/stretchr/testify/assert"
 )
 
-func TestImagePipelineDeserialization(t *testing.T) {
-	const Payload string = `{
-		"name": "example pipeline",
-		"type": 0,
-		"removeMetadata": false,
-		"steps": [
-			{
-				"name": "resize image",
-				"type": 0
-			},
-			{
-				"name": "compress image",
-				"type": 1
-			}
-		]
-	}`
+// pipeline deserialization
 
+func TestPipelineDeserialization(t *testing.T) {
 	t.Run("Image pipeline deserialization is successful", func(t *testing.T) {
+		const Payload string = `{
+			"name": "example pipeline",
+			"type": 0,
+			"removeMetadata": false,
+			"steps": [
+				{
+					"name": "resize image",
+					"type": 0
+				},
+				{
+					"name": "compress image",
+					"type": 1
+				}
+			]
+		}`
+
 		values := DeserializePipelines([][]byte{[]byte(Payload)})
 
 		assert.Equal(t, 1, len(values), "Output should contain one element")
 		assert.Equal(t, "example pipeline", values[0].GetName())
 		assert.Equal(t, Image, values[0].GetType())
 	})
+
+	t.Run("Video pipelines deserialization is successful", func(t *testing.T) {
+		const Payload string = `{
+			"name": "example pipeline",
+			"type": 1,
+			"removeMetadata": false,
+			"steps": [
+				{
+					"name": "resize image",
+					"type": 0
+				},
+				{
+					"name": "compress image",
+					"type": 1
+				}
+			]
+		}`
+
+		values := DeserializePipelines([][]byte{[]byte(Payload)})
+
+		assert.Equal(t, 1, len(values), "Output should contain one element")
+		assert.Equal(t, "example pipeline", values[0].GetName())
+		assert.Equal(t, Video, values[0].GetType())
+	})
+}
+
+// image pipeline steps
+
+func TestExecuteSteps(t *testing.T) {
+	t.Run("Pipeline executes with no steps", func(t *testing.T) {
+		const Bucket string = "pipeline_test_01"
+		const Payload string = `{
+			"name": "example pipeline",
+			"type": 1,
+			"removeMetadata": false,
+			"steps": []
+		}`
+
+		wd, _ := os.Getwd()
+		pipe := DeserializePipelines([][]byte{[]byte(Payload)})[0]
+
+		storageProvider := storage.GetFileSystemStorageProvider("test", "..")
+
+		// copy test file to storage bucket
+		_, err := storageProvider.StoreExisting(Bucket, "source.jpg", filepath.Join(wd, "../tests/files/900x900.jpg"))
+		assert.Nil(t, err, "Test file should be readable")
+		assert.FileExists(t, storageProvider.GetPath(Bucket, "source.jpg"))
+
+		// run pipeline steps
+		dest, err := pipe.Run("source.jpg", Bucket, storageProvider)
+		assert.Nil(t, err)
+		assert.FileExists(t, storageProvider.GetPath(Bucket, dest))
+
+		// clean up
+		os.Remove(storageProvider.GetPath(Bucket, "source.jpg"))
+		os.Remove(storageProvider.GetPath(Bucket, dest))
+	})
+
+	t.Run("Image resizing is successful", func(t *testing.T) {
+		const Bucket string = "pipeline_test_02"
+		const Payload string = `{
+			"name": "example pipeline",
+			"type": 1,
+			"removeMetadata": false,
+			"steps": [
+				{
+					"name": "resize image",
+					"type": 0,
+					"options": {
+						"width": 1280,
+						"height": 720,
+						"upscale": false
+					}
+				}
+			]
+		}`
+
+		wd, _ := os.Getwd()
+		pipe := DeserializePipelines([][]byte{[]byte(Payload)})[0]
+
+		storageProvider := storage.GetFileSystemStorageProvider("test", "..")
+
+		// copy test file to storage bucket
+		_, err := storageProvider.StoreExisting(Bucket, "source.jpg", filepath.Join(wd, "../tests/files/900x900.jpg"))
+		assert.Nil(t, err, "Test file should be readable")
+		assert.FileExists(t, storageProvider.GetPath(Bucket, "source.jpg"))
+
+		// run pipeline steps
+		dest, err := pipe.Run("source.jpg", Bucket, storageProvider)
+		assert.Nil(t, err)
+		assert.FileExists(t, storageProvider.GetPath(Bucket, dest))
+
+		// read image config
+		file, err := storageProvider.OpenFile(Bucket, dest)
+		assert.Nil(t, err)
+
+		imgConf, _, err := image.DecodeConfig(file)
+		assert.Nil(t, err)
+
+		assert.Equal(t, 1280, imgConf.Width)
+		assert.Equal(t, 720, imgConf.Height)
+
+		// clean up
+		os.Remove(storageProvider.GetPath(Bucket, "source.jpg"))
+		os.Remove(storageProvider.GetPath(Bucket, dest))
+	})
+
+	t.Run("Image rotation step is successful", func(t *testing.T) {
+		const Bucket string = "pipeline_test_03"
+		const Payload string = `{
+			"name": "example pipeline",
+			"type": 1,
+			"removeMetadata": false,
+			"steps": [
+				{
+					"name": "rotate image",
+					"type": 1,
+					"options": {
+						"angle": 90.0
+					}
+				}
+			]
+		}`
+
+		wd, _ := os.Getwd()
+		pipe := DeserializePipelines([][]byte{[]byte(Payload)})[0]
+
+		storageProvider := storage.GetFileSystemStorageProvider("test", "..")
+
+		// copy test file to storage bucket
+		_, err := storageProvider.StoreExisting(Bucket, "source.jpg", filepath.Join(wd, "../tests/files/800x500.jpg"))
+		assert.Nil(t, err, "Test file should be readable")
+		assert.FileExists(t, storageProvider.GetPath(Bucket, "source.jpg"))
+
+		// run pipeline steps
+		dest, err := pipe.Run("source.jpg", Bucket, storageProvider)
+		assert.Nil(t, err)
+		assert.FileExists(t, storageProvider.GetPath(Bucket, dest))
+
+		// read image config
+		file, err := storageProvider.OpenFile(Bucket, dest)
+		assert.Nil(t, err)
+
+		imgConf, _, err := image.DecodeConfig(file)
+		assert.Nil(t, err)
+
+		assert.Equal(t, 500, imgConf.Width)
+		assert.Equal(t, 800, imgConf.Height)
+
+		// clean up
+		os.Remove(storageProvider.GetPath(Bucket, "source.jpg"))
+		os.Remove(storageProvider.GetPath(Bucket, dest))
+	})
+
+	t.Run("Image flip step is successful", func(t *testing.T) {
+		const Bucket string = "pipeline_test_06"
+		const Payload string = `{
+			"name": "example pipeline",
+			"type": 1,
+			"removeMetadata": false,
+			"steps": [
+				{
+					"name": "flip image",
+					"type": 2,
+					"options": {
+						"direction": "h"
+					}
+				}
+			]
+		}`
+
+		wd, _ := os.Getwd()
+		pipe := DeserializePipelines([][]byte{[]byte(Payload)})[0]
+
+		storageProvider := storage.GetFileSystemStorageProvider("test", "..")
+
+		// copy test file to storage bucket
+		_, err := storageProvider.StoreExisting(Bucket, "source.jpg", filepath.Join(wd, "../tests/files/800x500.jpg"))
+		assert.Nil(t, err, "Test file should be readable")
+		assert.FileExists(t, storageProvider.GetPath(Bucket, "source.jpg"))
+
+		// run pipeline steps
+		dest, err := pipe.Run("source.jpg", Bucket, storageProvider)
+		assert.Nil(t, err)
+		assert.FileExists(t, storageProvider.GetPath(Bucket, dest))
+
+		// read image config
+		file, err := storageProvider.OpenFile(Bucket, dest)
+		assert.Nil(t, err)
+
+		imgConf, _, err := image.DecodeConfig(file)
+		assert.Nil(t, err)
+
+		assert.Equal(t, 800, imgConf.Width)
+		assert.Equal(t, 500, imgConf.Height)
+
+		// clean up
+		os.Remove(storageProvider.GetPath(Bucket, "source.jpg"))
+		os.Remove(storageProvider.GetPath(Bucket, dest))
+	})
+
+	t.Run("Image flip step direction validation is successful", func(t *testing.T) {
+		const Bucket string = "pipeline_test_07"
+		const Payload string = `{
+			"name": "example pipeline",
+			"type": 1,
+			"removeMetadata": false,
+			"steps": [
+				{
+					"name": "flip image",
+					"type": 2,
+					"options": {
+						"direction": "f"
+					}
+				}
+			]
+		}`
+
+		wd, _ := os.Getwd()
+		pipe := DeserializePipelines([][]byte{[]byte(Payload)})[0]
+
+		storageProvider := storage.GetFileSystemStorageProvider("test", "..")
+
+		// copy test file to storage bucket
+		_, err := storageProvider.StoreExisting(Bucket, "source.jpg", filepath.Join(wd, "../tests/files/800x500.jpg"))
+		assert.Nil(t, err, "Test file should be readable")
+		assert.FileExists(t, storageProvider.GetPath(Bucket, "source.jpg"))
+
+		// run pipeline steps
+		_, err = pipe.Run("source.jpg", Bucket, storageProvider)
+		assert.EqualError(t, err, "invalid flip direction: f")
+
+		// clean up
+		os.Remove(storageProvider.GetPath(Bucket, "source.jpg"))
+	})
+
+	t.Run("Image grayscale step is successful", func(t *testing.T) {
+		const Bucket string = "pipeline_test_05"
+		const Payload string = `{
+			"name": "example pipeline",
+			"type": 1,
+			"removeMetadata": false,
+			"steps": [
+				{
+					"name": "grayscale",
+					"type": 3
+				}
+			]
+		}`
+
+		wd, _ := os.Getwd()
+		pipe := DeserializePipelines([][]byte{[]byte(Payload)})[0]
+
+		storageProvider := storage.GetFileSystemStorageProvider("test", "..")
+
+		// copy test file to storage bucket
+		_, err := storageProvider.StoreExisting(Bucket, "source.jpg", filepath.Join(wd, "../tests/files/900x900.jpg"))
+		assert.Nil(t, err, "Test file should be readable")
+		assert.FileExists(t, storageProvider.GetPath(Bucket, "source.jpg"))
+
+		// run pipeline steps
+		dest, err := pipe.Run("source.jpg", Bucket, storageProvider)
+		assert.Nil(t, err)
+		assert.FileExists(t, storageProvider.GetPath(Bucket, dest))
+
+		// read image config
+		file, err := storageProvider.OpenFile(Bucket, dest)
+		assert.Nil(t, err)
+
+		imgConf, _, err := image.DecodeConfig(file)
+		assert.Nil(t, err)
+
+		assert.Equal(t, 900, imgConf.Width)
+		assert.Equal(t, 900, imgConf.Height)
+
+		// clean up
+		os.Remove(storageProvider.GetPath(Bucket, "source.jpg"))
+		os.Remove(storageProvider.GetPath(Bucket, dest))
+	})
+
+	t.Run("Image fit step is successful", func(t *testing.T) {
+		const Bucket string = "pipeline_test_08"
+		const Payload string = `{
+			"name": "example pipeline",
+			"type": 1,
+			"removeMetadata": false,
+			"steps": [
+				{
+					"name": "fit",
+					"type": 4,
+					"options": {
+						"width": 300,
+						"height": 200
+					}
+				}
+			]
+		}`
+
+		wd, _ := os.Getwd()
+		pipe := DeserializePipelines([][]byte{[]byte(Payload)})[0]
+
+		storageProvider := storage.GetFileSystemStorageProvider("test", "..")
+
+		// copy test file to storage bucket
+		_, err := storageProvider.StoreExisting(Bucket, "source.jpg", filepath.Join(wd, "../tests/files/900x900.jpg"))
+		assert.Nil(t, err, "Test file should be readable")
+		assert.FileExists(t, storageProvider.GetPath(Bucket, "source.jpg"))
+
+		// run pipeline steps
+		dest, err := pipe.Run("source.jpg", Bucket, storageProvider)
+		assert.Nil(t, err)
+		assert.FileExists(t, storageProvider.GetPath(Bucket, dest))
+
+		// read image config
+		file, err := storageProvider.OpenFile(Bucket, dest)
+		assert.Nil(t, err)
+
+		imgConf, _, err := image.DecodeConfig(file)
+		assert.Nil(t, err)
+
+		assert.Equal(t, 200, imgConf.Width)
+		assert.Equal(t, 200, imgConf.Height)
+
+		// clean up
+		os.Remove(storageProvider.GetPath(Bucket, "source.jpg"))
+		os.Remove(storageProvider.GetPath(Bucket, dest))
+	})
+
+	t.Run("Image invert step is successful", func(t *testing.T) {
+		const Bucket string = "pipeline_test_09"
+		const Payload string = `{
+			"name": "example pipeline",
+			"type": 1,
+			"removeMetadata": false,
+			"steps": [
+				{
+					"name": "invert",
+					"type": 5
+				}
+			]
+		}`
+
+		wd, _ := os.Getwd()
+		pipe := DeserializePipelines([][]byte{[]byte(Payload)})[0]
+
+		storageProvider := storage.GetFileSystemStorageProvider("test", "..")
+
+		// copy test file to storage bucket
+		_, err := storageProvider.StoreExisting(Bucket, "source.jpg", filepath.Join(wd, "../tests/files/900x900.jpg"))
+		assert.Nil(t, err, "Test file should be readable")
+		assert.FileExists(t, storageProvider.GetPath(Bucket, "source.jpg"))
+
+		// run pipeline steps
+		dest, err := pipe.Run("source.jpg", Bucket, storageProvider)
+		assert.Nil(t, err)
+		assert.FileExists(t, storageProvider.GetPath(Bucket, dest))
+
+		// read image config
+		file, err := storageProvider.OpenFile(Bucket, dest)
+		assert.Nil(t, err)
+
+		imgConf, _, err := image.DecodeConfig(file)
+		assert.Nil(t, err)
+
+		assert.Equal(t, 900, imgConf.Width)
+		assert.Equal(t, 900, imgConf.Height)
+
+		// clean up
+		os.Remove(storageProvider.GetPath(Bucket, "source.jpg"))
+		os.Remove(storageProvider.GetPath(Bucket, dest))
+	})
+
+	t.Run("Image blur step is successful", func(t *testing.T) {
+		const Bucket string = "pipeline_test_10"
+		const Payload string = `{
+			"name": "example pipeline",
+			"type": 1,
+			"removeMetadata": false,
+			"steps": [
+				{
+					"name": "blur",
+					"type": 6,
+					"options": {
+						"sigma": 50.0
+					}
+				}
+			]
+		}`
+
+		wd, _ := os.Getwd()
+		pipe := DeserializePipelines([][]byte{[]byte(Payload)})[0]
+
+		storageProvider := storage.GetFileSystemStorageProvider("test", "..")
+
+		// copy test file to storage bucket
+		_, err := storageProvider.StoreExisting(Bucket, "source.jpg", filepath.Join(wd, "../tests/files/900x900.jpg"))
+		assert.Nil(t, err, "Test file should be readable")
+		assert.FileExists(t, storageProvider.GetPath(Bucket, "source.jpg"))
+
+		// run pipeline steps
+		dest, err := pipe.Run("source.jpg", Bucket, storageProvider)
+		assert.Nil(t, err)
+		assert.FileExists(t, storageProvider.GetPath(Bucket, dest))
+
+		// read image config
+		file, err := storageProvider.OpenFile(Bucket, dest)
+		assert.Nil(t, err)
+
+		imgConf, _, err := image.DecodeConfig(file)
+		assert.Nil(t, err)
+
+		assert.Equal(t, 900, imgConf.Width)
+		assert.Equal(t, 900, imgConf.Height)
+
+		// clean up
+		os.Remove(storageProvider.GetPath(Bucket, "source.jpg"))
+		os.Remove(storageProvider.GetPath(Bucket, dest))
+	})
 }
 
-func TestVideoPipelineDeserialization(t *testing.T) {
+// output options
+
+func TestEncoding(t *testing.T) {
+	const Bucket string = "pipeline_test_04"
 	const Payload string = `{
 		"name": "example pipeline",
 		"type": 1,
@@ -40,20 +468,77 @@ func TestVideoPipelineDeserialization(t *testing.T) {
 		"steps": [
 			{
 				"name": "resize image",
-				"type": 0
-			},
-			{
-				"name": "compress image",
-				"type": 1
+				"type": 0,
+				"options": {
+					"width": 1280,
+					"height": 720,
+					"upscale": false
+				}
 			}
-		]
+		],
+		"output": {
+			"quality": 50
+		}
 	}`
 
-	t.Run("Video pipelines deserialization is successful", func(t *testing.T) {
-		values := DeserializePipelines([][]byte{[]byte(Payload)})
+	t.Run("Image encoding with jpeg quality is successful", func(t *testing.T) {
+		wd, _ := os.Getwd()
+		pipe := DeserializePipelines([][]byte{[]byte(Payload)})[0]
 
-		assert.Equal(t, 1, len(values), "Output should contain one element")
-		assert.Equal(t, "example pipeline", values[0].GetName())
-		assert.Equal(t, Video, values[0].GetType())
+		storageProvider := storage.GetFileSystemStorageProvider("test", "..")
+
+		// copy test file to storage bucket
+		_, err := storageProvider.StoreExisting(Bucket, "source.jpg", filepath.Join(wd, "../tests/files/900x900.jpg"))
+		assert.Nil(t, err, "Test file should be readable")
+		assert.FileExists(t, storageProvider.GetPath(Bucket, "source.jpg"))
+
+		// run pipeline steps
+		dest, err := pipe.Run("source.jpg", Bucket, storageProvider)
+		assert.Nil(t, err)
+		assert.FileExists(t, storageProvider.GetPath(Bucket, dest))
+
+		// clean up
+		os.Remove(storageProvider.GetPath(Bucket, "source.jpg"))
+		os.Remove(storageProvider.GetPath(Bucket, dest))
+	})
+
+	t.Run("Wrong output format results in error", func(t *testing.T) {
+		const InvalidPayload string = `{
+			"name": "example pipeline",
+			"type": 1,
+			"removeMetadata": false,
+			"steps": [
+				{
+					"name": "resize image",
+					"type": 0,
+					"options": {
+						"width": 1280,
+						"height": 720,
+						"upscale": false
+					}
+				}
+			],
+			"output": {
+				"format": "foo",
+				"quality": 50
+			}
+		}`
+
+		wd, _ := os.Getwd()
+		pipe := DeserializePipelines([][]byte{[]byte(InvalidPayload)})[0]
+
+		storageProvider := storage.GetFileSystemStorageProvider("test", "..")
+
+		// copy test file to storage bucket
+		_, err := storageProvider.StoreExisting(Bucket, "source.jpg", filepath.Join(wd, "../tests/files/900x900.jpg"))
+		assert.Nil(t, err, "Test file should be readable")
+		assert.FileExists(t, storageProvider.GetPath(Bucket, "source.jpg"))
+
+		// run pipeline steps
+		_, err = pipe.Run("source.jpg", Bucket, storageProvider)
+		assert.EqualError(t, err, "output format 'foo' is not supported")
+
+		// clean up
+		os.Remove(storageProvider.GetPath(Bucket, "source.jpg"))
 	})
 }

pipelines/step.go

@@ -9,7 +9,12 @@ type StepType int
 
 const (
 	TypeResizeImageStep StepType = iota
-	TypeCompressImageStep
+	TypeRotateImageStep
+	TypeFlipImageStep
+	TypeGrayscaleImageStep
+	TypeFitImageStep
+	TypeInvertImageStep
+	TypeBlurImageStep
 )
 
 type Step struct {
@@ -20,6 +25,7 @@ type Step struct {
 
 func (s Step) GetExecutable() (IExecutableStep, error) {
 	switch s.GetType() {
+
 	case TypeResizeImageStep:
 		step := ResizeImageStep{}
 		if err := json.Unmarshal(s.Options, &step.Options); err != nil {
@@ -27,8 +33,35 @@ func (s Step) GetExecutable() (IExecutableStep, error) {
 		}
 		return step, nil
 
-	case TypeCompressImageStep:
-		step := CompressImageStep{}
+	case TypeRotateImageStep:
+		step := RotateImageStep{}
+		if err := json.Unmarshal(s.Options, &step.Options); err != nil {
+			return nil, err
+		}
+		return step, nil
+
+	case TypeFlipImageStep:
+		step := FlipImageStep{}
+		if err := json.Unmarshal(s.Options, &step.Options); err != nil {
+			return nil, err
+		}
+		return step, nil
+
+	case TypeFitImageStep:
+		step := FitImageStep{}
+		if err := json.Unmarshal(s.Options, &step.Options); err != nil {
+			return nil, err
+		}
+		return step, nil
+
+	case TypeInvertImageStep:
+		return InvertImageStep{}, nil
+
+	case TypeGrayscaleImageStep:
+		return GrayscaleImageStep{}, nil
+
+	case TypeBlurImageStep:
+		step := BlurImageStep{}
 		if err := json.Unmarshal(s.Options, &step.Options); err != nil {
 			return nil, err
 		}

pipelines/step_test.go

@@ -1,8 +1,9 @@
 package pipelines
 
 import (
-	"github.com/stretchr/testify/assert"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
 )
 
 func TestTranslateStep(t *testing.T) {

settings/settings.go

@@ -0,0 +1,91 @@
+package settings
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+
+	"github.com/spf13/afero"
+)
+
+const (
+	Local FileSystemType = iota
+)
+
+type FileSystemType int
+
+type Settings struct {
+	Endpoint        string                 `json:"endpoint"`
+	Authentication  AuthenticationSettings `json:"authentication"`
+	RateLimiter     RateLimiterSettings    `json:"rate_limiter"`
+	StorageProvider StorageSettings        `json:"storage_provider"`
+}
+
+type StorageSettings struct {
+	Type     FileSystemType `json:"type"`
+	BasePath string         `json:"base_path"`
+}
+
+type AuthenticationSettings struct {
+	Enabled bool   `json:"enabled"`
+	Token   string `json:"token"`
+}
+
+type RateLimiterSettings struct {
+	Enabled           bool `json:"enabled"`
+	RequestsPerMinute int  `json:"requests_per_minute"`
+	AllowedBurst      int  `json:"allowed_burst"`
+}
+
+func parseSettings(data []byte) Settings {
+	settings := Settings{}
+
+	err := json.Unmarshal(data, &settings)
+	if err != nil {
+		return Settings{}
+	}
+
+	return settings
+}
+
+func LoadSettings(fileSystem afero.Fs) (Settings, error) {
+	workingDirectory, _ := os.Getwd()
+	path := filepath.Join(workingDirectory, "settings.json")
+
+	// Load file and parse file
+	data, err := afero.ReadFile(fileSystem, path)
+
+	if err == nil {
+		return parseSettings(data), nil
+	}
+
+	// If file does not exist, create default settings
+	defaultSettings := Settings{
+		Endpoint: "127.0.0.1:8000",
+		Authentication: AuthenticationSettings{
+			Enabled: false,
+			Token:   "changeme",
+		},
+		RateLimiter: RateLimiterSettings{
+			Enabled:           true,
+			RequestsPerMinute: 20,
+			AllowedBurst:      5,
+		},
+		StorageProvider: StorageSettings{
+			Type:     Local,
+			BasePath: "assets",
+		},
+	}
+
+	serializedSettings, err := json.MarshalIndent(defaultSettings, "", "\t")
+	if err != nil {
+		return Settings{}, err
+	}
+
+	err = afero.WriteFile(fileSystem, path, serializedSettings, os.ModePerm)
+	if err != nil {
+		return Settings{}, err
+	}
+
+	return defaultSettings, nil
+}

settings/settings_test.go

@@ -0,0 +1,57 @@
+package settings
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSettingsParsing(t *testing.T) {
+	const file string = `{
+		"endpoint": "0.0.0.0:8000",
+		"authentication": {
+			"enabled": true,
+			"token": "foobar"
+		},
+		"rate_limiter": {
+			"enabled": true,
+			"requests_per_minute": 20,
+			"allowed_burst": 5
+		},
+		"storage_provider": {
+			"type": 0,
+			"base_path": "assets"
+		}
+	}`
+
+	t.Run("Settings parsing is successful", func(t *testing.T) {
+		settings := parseSettings([]byte(file))
+
+		assert.Equal(t, "0.0.0.0:8000", settings.Endpoint)
+		assert.Equal(t, "foobar", settings.Authentication.Token)
+		assert.Equal(t, "assets", settings.StorageProvider.BasePath)
+	})
+}
+
+func TestSettingsLoading(t *testing.T) {
+	t.Run("Settings loading creates default settings.json when none is present", func(t *testing.T) {
+		fileSystem := afero.NewMemMapFs()
+		workingDirectory, _ := os.Getwd()
+
+		path := filepath.Join(workingDirectory, "settings.json")
+
+		// Settings file does not exist in the beginning
+		doesFileExist, _ := afero.Exists(fileSystem, path)
+		assert.False(t, doesFileExist)
+
+		_, err := LoadSettings(fileSystem)
+		assert.Nil(t, err)
+
+		// Settings file should be present after calling LoadSettings
+		doesFileExist, _ = afero.Exists(fileSystem, path)
+		assert.True(t, doesFileExist)
+	})
+}

storage/storage.go

@@ -7,30 +7,69 @@ import (
 	"github.com/spf13/afero"
 )
 
+const StorageFolderName = "files"
+
 type IStorageProvider interface {
-	storeRaw(bucketName string, objectName string, data []byte) string
-	storeExisting(bucketName string, objectName string, existingFilePath string) string
+	StoreRaw(bucketName string, objectName string, data []byte) (string, error)
+	StoreExisting(bucketName string, objectName string, existingFilePath string) (string, error)
+	GetPath(bucketName string, objectName string) string
 }
 
 type FileSystemStorageProvider struct {
 	fileSystem afero.Fs
 	basePath   string
+	wd         string
 }
 
-func (sp FileSystemStorageProvider) storeRaw(bucketName string, objectName string, data []byte) string {
+func (sp FileSystemStorageProvider) StoreRaw(bucketName string, objectName string, data []byte) (string, error) {
 	directoryPath := filepath.Join(sp.basePath, bucketName)
 
-	sp.fileSystem.MkdirAll(directoryPath, os.ModePerm)
+	if err := sp.fileSystem.MkdirAll(directoryPath, os.ModePerm); err != nil {
+		return "", err
+	}
 
 	filePath := filepath.Join(directoryPath, objectName)
 
-	afero.WriteFile(sp.fileSystem, filePath, data, os.ModePerm)
+	if err := afero.WriteFile(sp.fileSystem, filePath, data, os.ModePerm); err != nil {
+		return "", err
+	}
+
+	return filePath, nil
+}
+
+func (sp FileSystemStorageProvider) StoreExisting(bucketName string, objectName string, existingFilePath string) (string, error) {
+	bytesRead, err := os.ReadFile(existingFilePath)
+	if err != nil {
+		return "", err
+	}
+
+	return sp.StoreRaw(bucketName, objectName, bytesRead)
+}
+
+func (sp FileSystemStorageProvider) GetPath(bucketName string, objectName string) string {
+	return filepath.Join(sp.wd, StorageFolderName, sp.basePath, bucketName, objectName)
+}
 
-	return filePath
+func (sp FileSystemStorageProvider) OpenFile(bucketName string, objectName string) (*os.File, error) {
+	return os.Open(sp.GetPath(bucketName, objectName))
 }
 
-func (sp FileSystemStorageProvider) storeExisting(bucketName string, objectName string, existingFilePath string) string {
-	bytesRead, _ := afero.ReadFile(sp.fileSystem, existingFilePath)
+func GetFileSystemStorageProvider(basePath string, wd string) FileSystemStorageProvider {
+	if wd == "" {
+		wd, _ = os.Getwd()
+	}
+
+	return FileSystemStorageProvider{
+		fileSystem: afero.NewBasePathFs(afero.NewOsFs(), filepath.Join(wd, StorageFolderName)),
+		basePath:   basePath,
+		wd:         wd,
+	}
+}
 
-	return sp.storeRaw(bucketName, objectName, bytesRead)
+// TODO: Move this out of this file
+func GetMemoryStorageProvider() FileSystemStorageProvider {
+	return FileSystemStorageProvider{
+		fileSystem: afero.NewBasePathFs(afero.NewMemMapFs(), "/"),
+		basePath:   "/tmp/foo/bar",
+	}
 }

storage/storage_test.go

@@ -19,7 +19,8 @@ func TestFileSystemStorageProvider(t *testing.T) {
 			basePath:   "/tmp/foo/bar",
 		}
 
-		finalPath := provider.storeRaw("test", "test.bin", dummyData)
+		finalPath, err := provider.StoreRaw("test", "test.bin", dummyData)
+		assert.Nil(t, err)
 		assert.Equal(t, "/tmp/foo/bar/test/test.bin", finalPath)
 
 		exists, _ := afero.Exists(fileSystem, "/tmp/foo/bar/test/test.bin")
@@ -32,14 +33,16 @@ func TestFileSystemStorageProvider(t *testing.T) {
 	t.Run("storeExisting method stores files in filesystem", func(t *testing.T) {
 		fileSystem := afero.NewMemMapFs()
 
-		afero.WriteFile(fileSystem, "/tmp/existing.bin", dummyData, os.ModePerm)
+		err := os.WriteFile("/tmp/existing.bin", dummyData, os.ModePerm)
+		assert.Nil(t, err)
 
 		provider := FileSystemStorageProvider{
 			fileSystem: fileSystem,
 			basePath:   "/tmp/foo/bar",
 		}
 
-		finalPath := provider.storeExisting("test", "test.bin", "/tmp/existing.bin")
+		finalPath, err := provider.StoreExisting("test", "test.bin", "/tmp/existing.bin")
+		assert.Nil(t, err)
 		assert.Equal(t, "/tmp/foo/bar/test/test.bin", finalPath)
 
 		exists, _ := afero.Exists(fileSystem, "/tmp/foo/bar/test/test.bin")
@@ -48,4 +51,18 @@ func TestFileSystemStorageProvider(t *testing.T) {
 		content, _ := afero.ReadFile(fileSystem, "/tmp/foo/bar/test/test.bin")
 		assert.Equal(t, dummyData, content)
 	})
+
+	t.Run("getPath method returns correct path", func(t *testing.T) {
+		fileSystem := afero.NewMemMapFs()
+
+		provider := FileSystemStorageProvider{
+			fileSystem: fileSystem,
+			basePath:   "/tmp/foo/bar",
+		}
+
+		_, err := provider.StoreRaw("test", "test.bin", dummyData)
+		assert.Nil(t, err)
+
+		assert.Equal(t, "files/tmp/foo/bar/test/test.bin", provider.GetPath("test", "test.bin"))
+	})
 }