Micro-service for file storage and processing written in Go
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

405 lines
11 KiB

  1. package pipelines
  2. import (
  3. "image"
  4. "os"
  5. "path/filepath"
  6. "testing"
  7. "github.com/geplauder/lithium/storage"
  8. "github.com/stretchr/testify/assert"
  9. )
  10. // pipeline deserialization
  11. func TestPipelineDeserialization(t *testing.T) {
  12. t.Run("Image pipeline deserialization is successful", func(t *testing.T) {
  13. const Payload string = `{
  14. "name": "example pipeline",
  15. "type": 0,
  16. "removeMetadata": false,
  17. "steps": [
  18. {
  19. "name": "resize image",
  20. "type": 0
  21. },
  22. {
  23. "name": "compress image",
  24. "type": 1
  25. }
  26. ]
  27. }`
  28. values := DeserializePipelines([][]byte{[]byte(Payload)})
  29. assert.Equal(t, 1, len(values), "Output should contain one element")
  30. assert.Equal(t, "example pipeline", values[0].GetName())
  31. assert.Equal(t, Image, values[0].GetType())
  32. })
  33. t.Run("Video pipelines deserialization is successful", func(t *testing.T) {
  34. const Payload string = `{
  35. "name": "example pipeline",
  36. "type": 1,
  37. "removeMetadata": false,
  38. "steps": [
  39. {
  40. "name": "resize image",
  41. "type": 0
  42. },
  43. {
  44. "name": "compress image",
  45. "type": 1
  46. }
  47. ]
  48. }`
  49. values := DeserializePipelines([][]byte{[]byte(Payload)})
  50. assert.Equal(t, 1, len(values), "Output should contain one element")
  51. assert.Equal(t, "example pipeline", values[0].GetName())
  52. assert.Equal(t, Video, values[0].GetType())
  53. })
  54. }
  55. // image pipeline steps
  56. func TestExecuteSteps(t *testing.T) {
  57. t.Run("Pipeline executes with no steps", func(t *testing.T) {
  58. const Bucket string = "pipeline_test_01"
  59. const Payload string = `{
  60. "name": "example pipeline",
  61. "type": 1,
  62. "removeMetadata": false,
  63. "steps": []
  64. }`
  65. wd, _ := os.Getwd()
  66. pipe := DeserializePipelines([][]byte{[]byte(Payload)})[0]
  67. storageProvider := storage.GetFileSystemStorageProvider("test", "..")
  68. // copy test file to storage bucket
  69. _, err := storageProvider.StoreExisting(Bucket, "source.jpg", filepath.Join(wd, "../tests/files/900x900.jpg"))
  70. assert.Nil(t, err, "Test file should be readable")
  71. assert.FileExists(t, storageProvider.GetPath(Bucket, "source.jpg"))
  72. // run pipeline steps
  73. dest, err := pipe.Run("source.jpg", Bucket, storageProvider)
  74. assert.Nil(t, err)
  75. assert.FileExists(t, storageProvider.GetPath(Bucket, dest))
  76. // clean up
  77. os.Remove(storageProvider.GetPath(Bucket, "source.jpg"))
  78. os.Remove(storageProvider.GetPath(Bucket, dest))
  79. })
  80. t.Run("Image resizing is successful", func(t *testing.T) {
  81. const Bucket string = "pipeline_test_02"
  82. const Payload string = `{
  83. "name": "example pipeline",
  84. "type": 1,
  85. "removeMetadata": false,
  86. "steps": [
  87. {
  88. "name": "resize image",
  89. "type": 0,
  90. "options": {
  91. "width": 1280,
  92. "height": 720,
  93. "upscale": false
  94. }
  95. }
  96. ]
  97. }`
  98. wd, _ := os.Getwd()
  99. pipe := DeserializePipelines([][]byte{[]byte(Payload)})[0]
  100. storageProvider := storage.GetFileSystemStorageProvider("test", "..")
  101. // copy test file to storage bucket
  102. _, err := storageProvider.StoreExisting(Bucket, "source.jpg", filepath.Join(wd, "../tests/files/900x900.jpg"))
  103. assert.Nil(t, err, "Test file should be readable")
  104. assert.FileExists(t, storageProvider.GetPath(Bucket, "source.jpg"))
  105. // run pipeline steps
  106. dest, err := pipe.Run("source.jpg", Bucket, storageProvider)
  107. assert.Nil(t, err)
  108. assert.FileExists(t, storageProvider.GetPath(Bucket, dest))
  109. // read image config
  110. file, err := storageProvider.OpenFile(Bucket, dest)
  111. assert.Nil(t, err)
  112. imgConf, _, err := image.DecodeConfig(file)
  113. assert.Nil(t, err)
  114. assert.Equal(t, 1280, imgConf.Width)
  115. assert.Equal(t, 720, imgConf.Height)
  116. // clean up
  117. os.Remove(storageProvider.GetPath(Bucket, "source.jpg"))
  118. os.Remove(storageProvider.GetPath(Bucket, dest))
  119. })
  120. t.Run("Image rotation step is successful", func(t *testing.T) {
  121. const Bucket string = "pipeline_test_03"
  122. const Payload string = `{
  123. "name": "example pipeline",
  124. "type": 1,
  125. "removeMetadata": false,
  126. "steps": [
  127. {
  128. "name": "rotate image",
  129. "type": 1,
  130. "options": {
  131. "angle": 90.0
  132. }
  133. }
  134. ]
  135. }`
  136. wd, _ := os.Getwd()
  137. pipe := DeserializePipelines([][]byte{[]byte(Payload)})[0]
  138. storageProvider := storage.GetFileSystemStorageProvider("test", "..")
  139. // copy test file to storage bucket
  140. _, err := storageProvider.StoreExisting(Bucket, "source.jpg", filepath.Join(wd, "../tests/files/800x500.jpg"))
  141. assert.Nil(t, err, "Test file should be readable")
  142. assert.FileExists(t, storageProvider.GetPath(Bucket, "source.jpg"))
  143. // run pipeline steps
  144. dest, err := pipe.Run("source.jpg", Bucket, storageProvider)
  145. assert.Nil(t, err)
  146. assert.FileExists(t, storageProvider.GetPath(Bucket, dest))
  147. // read image config
  148. file, err := storageProvider.OpenFile(Bucket, dest)
  149. assert.Nil(t, err)
  150. imgConf, _, err := image.DecodeConfig(file)
  151. assert.Nil(t, err)
  152. assert.Equal(t, 500, imgConf.Width)
  153. assert.Equal(t, 800, imgConf.Height)
  154. // clean up
  155. os.Remove(storageProvider.GetPath(Bucket, "source.jpg"))
  156. os.Remove(storageProvider.GetPath(Bucket, dest))
  157. })
  158. t.Run("Image flip step is successful", func(t *testing.T) {
  159. const Bucket string = "pipeline_test_06"
  160. const Payload string = `{
  161. "name": "example pipeline",
  162. "type": 1,
  163. "removeMetadata": false,
  164. "steps": [
  165. {
  166. "name": "flip image",
  167. "type": 2,
  168. "options": {
  169. "direction": "h"
  170. }
  171. }
  172. ]
  173. }`
  174. wd, _ := os.Getwd()
  175. pipe := DeserializePipelines([][]byte{[]byte(Payload)})[0]
  176. storageProvider := storage.GetFileSystemStorageProvider("test", "..")
  177. // copy test file to storage bucket
  178. _, err := storageProvider.StoreExisting(Bucket, "source.jpg", filepath.Join(wd, "../tests/files/800x500.jpg"))
  179. assert.Nil(t, err, "Test file should be readable")
  180. assert.FileExists(t, storageProvider.GetPath(Bucket, "source.jpg"))
  181. // run pipeline steps
  182. dest, err := pipe.Run("source.jpg", Bucket, storageProvider)
  183. assert.Nil(t, err)
  184. assert.FileExists(t, storageProvider.GetPath(Bucket, dest))
  185. // read image config
  186. file, err := storageProvider.OpenFile(Bucket, dest)
  187. assert.Nil(t, err)
  188. imgConf, _, err := image.DecodeConfig(file)
  189. assert.Nil(t, err)
  190. assert.Equal(t, 800, imgConf.Width)
  191. assert.Equal(t, 500, imgConf.Height)
  192. // clean up
  193. os.Remove(storageProvider.GetPath(Bucket, "source.jpg"))
  194. os.Remove(storageProvider.GetPath(Bucket, dest))
  195. })
  196. t.Run("Image flip step direction validation is successful", func(t *testing.T) {
  197. const Bucket string = "pipeline_test_06"
  198. const Payload string = `{
  199. "name": "example pipeline",
  200. "type": 1,
  201. "removeMetadata": false,
  202. "steps": [
  203. {
  204. "name": "flip image",
  205. "type": 2,
  206. "options": {
  207. "direction": "f"
  208. }
  209. }
  210. ]
  211. }`
  212. wd, _ := os.Getwd()
  213. pipe := DeserializePipelines([][]byte{[]byte(Payload)})[0]
  214. storageProvider := storage.GetFileSystemStorageProvider("test", "..")
  215. // copy test file to storage bucket
  216. _, err := storageProvider.StoreExisting(Bucket, "source.jpg", filepath.Join(wd, "../tests/files/800x500.jpg"))
  217. assert.Nil(t, err, "Test file should be readable")
  218. assert.FileExists(t, storageProvider.GetPath(Bucket, "source.jpg"))
  219. // run pipeline steps
  220. _, err = pipe.Run("source.jpg", Bucket, storageProvider)
  221. assert.EqualError(t, err, "invalid flip direction: f")
  222. // clean up
  223. os.Remove(storageProvider.GetPath(Bucket, "source.jpg"))
  224. })
  225. t.Run("Image grayscale step is successful", func(t *testing.T) {
  226. const Bucket string = "pipeline_test_05"
  227. const Payload string = `{
  228. "name": "example pipeline",
  229. "type": 1,
  230. "removeMetadata": false,
  231. "steps": [
  232. {
  233. "name": "grayscale",
  234. "type": 3
  235. }
  236. ]
  237. }`
  238. wd, _ := os.Getwd()
  239. pipe := DeserializePipelines([][]byte{[]byte(Payload)})[0]
  240. storageProvider := storage.GetFileSystemStorageProvider("test", "..")
  241. // copy test file to storage bucket
  242. _, err := storageProvider.StoreExisting(Bucket, "source.jpg", filepath.Join(wd, "../tests/files/900x900.jpg"))
  243. assert.Nil(t, err, "Test file should be readable")
  244. assert.FileExists(t, storageProvider.GetPath(Bucket, "source.jpg"))
  245. // run pipeline steps
  246. dest, err := pipe.Run("source.jpg", Bucket, storageProvider)
  247. assert.Nil(t, err)
  248. assert.FileExists(t, storageProvider.GetPath(Bucket, dest))
  249. // read image config
  250. file, err := storageProvider.OpenFile(Bucket, dest)
  251. assert.Nil(t, err)
  252. imgConf, _, err := image.DecodeConfig(file)
  253. assert.Nil(t, err)
  254. assert.Equal(t, 900, imgConf.Width)
  255. assert.Equal(t, 900, imgConf.Height)
  256. // clean up
  257. os.Remove(storageProvider.GetPath(Bucket, "source.jpg"))
  258. os.Remove(storageProvider.GetPath(Bucket, dest))
  259. })
  260. }
  261. // output options
  262. func TestEncoding(t *testing.T) {
  263. const Bucket string = "pipeline_test_04"
  264. const Payload string = `{
  265. "name": "example pipeline",
  266. "type": 1,
  267. "removeMetadata": false,
  268. "steps": [
  269. {
  270. "name": "resize image",
  271. "type": 0,
  272. "options": {
  273. "width": 1280,
  274. "height": 720,
  275. "upscale": false
  276. }
  277. }
  278. ],
  279. "output": {
  280. "quality": 50
  281. }
  282. }`
  283. t.Run("Image encoding with jpeg quality is successful", func(t *testing.T) {
  284. wd, _ := os.Getwd()
  285. pipe := DeserializePipelines([][]byte{[]byte(Payload)})[0]
  286. storageProvider := storage.GetFileSystemStorageProvider("test", "..")
  287. // copy test file to storage bucket
  288. _, err := storageProvider.StoreExisting(Bucket, "source.jpg", filepath.Join(wd, "../tests/files/900x900.jpg"))
  289. assert.Nil(t, err, "Test file should be readable")
  290. assert.FileExists(t, storageProvider.GetPath(Bucket, "source.jpg"))
  291. // run pipeline steps
  292. dest, err := pipe.Run("source.jpg", Bucket, storageProvider)
  293. assert.Nil(t, err)
  294. assert.FileExists(t, storageProvider.GetPath(Bucket, dest))
  295. // clean up
  296. os.Remove(storageProvider.GetPath(Bucket, "source.jpg"))
  297. os.Remove(storageProvider.GetPath(Bucket, dest))
  298. })
  299. t.Run("Wrong output format results in error", func(t *testing.T) {
  300. const InvalidPayload string = `{
  301. "name": "example pipeline",
  302. "type": 1,
  303. "removeMetadata": false,
  304. "steps": [
  305. {
  306. "name": "resize image",
  307. "type": 0,
  308. "options": {
  309. "width": 1280,
  310. "height": 720,
  311. "upscale": false
  312. }
  313. }
  314. ],
  315. "output": {
  316. "format": "foo",
  317. "quality": 50
  318. }
  319. }`
  320. wd, _ := os.Getwd()
  321. pipe := DeserializePipelines([][]byte{[]byte(InvalidPayload)})[0]
  322. storageProvider := storage.GetFileSystemStorageProvider("test", "..")
  323. // copy test file to storage bucket
  324. _, err := storageProvider.StoreExisting(Bucket, "source.jpg", filepath.Join(wd, "../tests/files/900x900.jpg"))
  325. assert.Nil(t, err, "Test file should be readable")
  326. assert.FileExists(t, storageProvider.GetPath(Bucket, "source.jpg"))
  327. // run pipeline steps
  328. _, err = pipe.Run("source.jpg", Bucket, storageProvider)
  329. assert.EqualError(t, err, "output format 'foo' is not supported")
  330. // clean up
  331. os.Remove(storageProvider.GetPath(Bucket, "source.jpg"))
  332. })
  333. }