From bb9c53958038bb74ad76a9d887b15e6decb5249c Mon Sep 17 00:00:00 2001 From: Maksim Date: Sat, 10 Aug 2019 11:27:59 +0000 Subject: [PATCH] Uploader.S3 added support stream uploads --- lib/pleroma/uploaders/s3.ex | 12 ++--- mix.exs | 3 +- mix.lock | 1 + test/uploaders/s3_test.exs | 90 +++++++++++++++++++++++++++++++++++++ 4 files changed, 100 insertions(+), 6 deletions(-) create mode 100644 test/uploaders/s3_test.exs diff --git a/lib/pleroma/uploaders/s3.ex b/lib/pleroma/uploaders/s3.ex index 521daa93b..8c353bed3 100644 --- a/lib/pleroma/uploaders/s3.ex +++ b/lib/pleroma/uploaders/s3.ex @@ -6,10 +6,12 @@ defmodule Pleroma.Uploaders.S3 do @behaviour Pleroma.Uploaders.Uploader require Logger + alias Pleroma.Config + # The file name is re-encoded with S3's constraints here to comply with previous # links with less strict filenames def get_file(file) do - config = Pleroma.Config.get([__MODULE__]) + config = Config.get([__MODULE__]) bucket = Keyword.fetch!(config, :bucket) bucket_with_namespace = @@ -34,15 +36,15 @@ def get_file(file) do end def put_file(%Pleroma.Upload{} = upload) do - config = Pleroma.Config.get([__MODULE__]) + config = Config.get([__MODULE__]) bucket = Keyword.get(config, :bucket) - {:ok, file_data} = File.read(upload.tempfile) - s3_name = strict_encode(upload.path) op = - ExAws.S3.put_object(bucket, s3_name, file_data, [ + upload.tempfile + |> ExAws.S3.Upload.stream_file() + |> ExAws.S3.upload(bucket, s3_name, [ {:acl, :public_read}, {:content_type, upload.content_type} ]) diff --git a/mix.exs b/mix.exs index ac175dfed..334fabb33 100644 --- a/mix.exs +++ b/mix.exs @@ -114,8 +114,9 @@ defp deps do {:tesla, "~> 1.2"}, {:jason, "~> 1.0"}, {:mogrify, "~> 0.6.1"}, - {:ex_aws, "~> 2.0"}, + {:ex_aws, "~> 2.1"}, {:ex_aws_s3, "~> 2.0"}, + {:sweet_xml, "~> 0.6.6"}, {:earmark, "~> 1.3"}, {:bbcode, "~> 0.1.1"}, {:ex_machina, "~> 2.3", only: :test}, diff --git a/mix.lock b/mix.lock index 13728d11f..f8ee80c83 100644 --- a/mix.lock +++ b/mix.lock @@ -80,6 +80,7 @@ "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm"}, "recon": {:git, "https://github.com/ferd/recon.git", "75d70c7c08926d2f24f1ee6de14ee50fe8a52763", [tag: "2.4.0"]}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"}, + "sweet_xml": {:hex, :sweet_xml, "0.6.6", "fc3e91ec5dd7c787b6195757fbcf0abc670cee1e4172687b45183032221b66b8", [:mix], [], "hexpm"}, "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, diff --git a/test/uploaders/s3_test.exs b/test/uploaders/s3_test.exs new file mode 100644 index 000000000..a0a1cfdf0 --- /dev/null +++ b/test/uploaders/s3_test.exs @@ -0,0 +1,90 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Uploaders.S3Test do + use Pleroma.DataCase + + alias Pleroma.Config + alias Pleroma.Uploaders.S3 + + import Mock + import ExUnit.CaptureLog + + setup do + config = Config.get([Pleroma.Uploaders.S3]) + + Config.put([Pleroma.Uploaders.S3], + bucket: "test_bucket", + public_endpoint: "https://s3.amazonaws.com" + ) + + on_exit(fn -> + Config.put([Pleroma.Uploaders.S3], config) + end) + + :ok + end + + describe "get_file/1" do + test "it returns path to local folder for files" do + assert S3.get_file("test_image.jpg") == { + :ok, + {:url, "https://s3.amazonaws.com/test_bucket/test_image.jpg"} + } + end + + test "it returns path without bucket when truncated_namespace set to ''" do + Config.put([Pleroma.Uploaders.S3], + bucket: "test_bucket", + public_endpoint: "https://s3.amazonaws.com", + truncated_namespace: "" + ) + + assert S3.get_file("test_image.jpg") == { + :ok, + {:url, "https://s3.amazonaws.com/test_image.jpg"} + } + end + + test "it returns path with bucket namespace when namespace is set" do + Config.put([Pleroma.Uploaders.S3], + bucket: "test_bucket", + public_endpoint: "https://s3.amazonaws.com", + bucket_namespace: "family" + ) + + assert S3.get_file("test_image.jpg") == { + :ok, + {:url, "https://s3.amazonaws.com/family:test_bucket/test_image.jpg"} + } + end + end + + describe "put_file/1" do + setup do + file_upload = %Pleroma.Upload{ + name: "image-tet.jpg", + content_type: "image/jpg", + path: "test_folder/image-tet.jpg", + tempfile: Path.absname("test/fixtures/image_tmp.jpg") + } + + [file_upload: file_upload] + end + + test "save file", %{file_upload: file_upload} do + with_mock ExAws, request: fn _ -> {:ok, :ok} end do + assert S3.put_file(file_upload) == {:ok, {:file, "test_folder/image-tet.jpg"}} + end + end + + test "returns error", %{file_upload: file_upload} do + with_mock ExAws, request: fn _ -> {:error, "S3 Upload failed"} end do + assert capture_log(fn -> + assert S3.put_file(file_upload) == {:error, "S3 Upload failed"} + end) =~ "Elixir.Pleroma.Uploaders.S3: {:error, \"S3 Upload failed\"}" + end + end + end +end