From 4e4b1016ab03d413b7b037333d1869935d7789be Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Mon, 27 Aug 2018 23:24:35 +0000 Subject: [PATCH 1/6] mix: add ex_aws and ex_aws_s3 dependencies --- mix.exs | 2 ++ mix.lock | 2 ++ 2 files changed, 4 insertions(+) diff --git a/mix.exs b/mix.exs index 941b9c149..6b95eeec3 100644 --- a/mix.exs +++ b/mix.exs @@ -46,6 +46,8 @@ defp deps do {:httpoison, "~> 1.2.0"}, {:jason, "~> 1.0"}, {:mogrify, "~> 0.6.1"}, + {:ex_aws, "~> 2.0"}, + {:ex_aws_s3, "~> 2.0"}, {:ex_machina, "~> 2.2", only: :test}, {:credo, "~> 0.9.3", only: [:dev, :test]}, {:mock, "~> 0.3.1", only: :test} diff --git a/mix.lock b/mix.lock index 6ee82301f..989a97add 100644 --- a/mix.lock +++ b/mix.lock @@ -12,6 +12,8 @@ "decimal": {:hex, :decimal, "1.5.0", "b0433a36d0e2430e3d50291b1c65f53c37d56f83665b43d79963684865beab68", [:mix], [], "hexpm"}, "ecto": {:hex, :ecto, "2.2.10", "e7366dc82f48f8dd78fcbf3ab50985ceeb11cb3dc93435147c6e13f2cda0992e", [:mix], [{:db_connection, "~> 1.1", [hex: :db_connection, repo: "hexpm", optional: true]}, {:decimal, "~> 1.2", [hex: :decimal, repo: "hexpm", optional: false]}, {:mariaex, "~> 0.8.0", [hex: :mariaex, repo: "hexpm", optional: true]}, {:poison, "~> 2.2 or ~> 3.0", [hex: :poison, repo: "hexpm", optional: true]}, {:poolboy, "~> 1.5", [hex: :poolboy, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.13.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:sbroker, "~> 1.0", [hex: :sbroker, repo: "hexpm", optional: true]}], "hexpm"}, "eternal": {:hex, :eternal, "1.2.0", "e2a6b6ce3b8c248f7dc31451aefca57e3bdf0e48d73ae5043229380a67614c41", [:mix], [], "hexpm"}, + "ex_aws": {:hex, :ex_aws, "2.1.0", "b92651527d6c09c479f9013caa9c7331f19cba38a650590d82ebf2c6c16a1d8a", [:mix], [{:configparser_ex, "~> 2.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "1.6.3 or 1.6.5 or 1.7.1 or 1.8.6 or ~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8", [hex: :jsx, repo: "hexpm", optional: true]}, {:poison, ">= 1.2.0", [hex: :poison, repo: "hexpm", optional: true]}, {:sweet_xml, "~> 0.6", [hex: :sweet_xml, repo: "hexpm", optional: true]}, {:xml_builder, "~> 0.1.0", [hex: :xml_builder, repo: "hexpm", optional: true]}], "hexpm"}, + "ex_aws_s3": {:hex, :ex_aws_s3, "2.0.1", "9e09366e77f25d3d88c5393824e613344631be8db0d1839faca49686e99b6704", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm"}, "ex_machina": {:hex, :ex_machina, "2.2.0", "fec496331e04fc2db2a1a24fe317c12c0c4a50d2beb8ebb3531ed1f0d84be0ed", [:mix], [{:ecto, "~> 2.1", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"}, "gettext": {:hex, :gettext, "0.15.0", "40a2b8ce33a80ced7727e36768499fc9286881c43ebafccae6bab731e2b2b8ce", [:mix], [], "hexpm"}, "hackney": {:hex, :hackney, "1.13.0", "24edc8cd2b28e1c652593833862435c80661834f6c9344e84b6a2255e7aeef03", [:rebar3], [{:certifi, "2.3.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "5.1.2", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "1.0.2", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"}, From e95d958b525cd0a448d38753099bbf7a4f30ba90 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Mon, 27 Aug 2018 23:30:53 +0000 Subject: [PATCH 2/6] sample config: show how amazon s3 support is activated, including third-party clones like wasabi --- config/config.exs | 4 +++- lib/mix/tasks/sample_config.eex | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/config/config.exs b/config/config.exs index eaf20e8f9..d234d23eb 100644 --- a/config/config.exs +++ b/config/config.exs @@ -12,7 +12,9 @@ config :pleroma, Pleroma.Upload, uploads: "uploads", - strip_exif: false + strip_exif: false, + use_s3: false, + s3_bucket: nil config :pleroma, :emoji, shortcode_globs: ["/emoji/custom/**/*.png"] diff --git a/lib/mix/tasks/sample_config.eex b/lib/mix/tasks/sample_config.eex index 6db36fa09..cfb5ef4c0 100644 --- a/lib/mix/tasks/sample_config.eex +++ b/lib/mix/tasks/sample_config.eex @@ -24,3 +24,20 @@ config :pleroma, Pleroma.Repo, database: "pleroma_dev", hostname: "localhost", pool_size: 10 + +# Configure S3 support if desired: +# +# config :pleroma, Pleroma.Upload, +# use_s3: true, +# bucket: "some-bucket" +# +# Configure S3 credentials: +# config :ex_aws, :s3, +# access_key_id: "xxxxxxxxxxxxx", +# secret_access_key: "yyyyyyyyyyyy", +# region: "us-east-1", +# scheme: "https://", +# +# For using third-party S3 clones like wasabi, also do: +# config :ex_aws, :s3, +# host: "s3.wasabisys.com" From 86c007ddd2a059e1d98780872abe30f92a24fbbf Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Mon, 27 Aug 2018 23:36:30 +0000 Subject: [PATCH 3/6] upload: strip exif data before finalizing the file path --- lib/pleroma/upload.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex index e0cb545b0..eaff8fbf3 100644 --- a/lib/pleroma/upload.ex +++ b/lib/pleroma/upload.ex @@ -9,6 +9,8 @@ def store(%Plug.Upload{} = file, should_dedupe) do upload_folder = get_upload_path(uuid, should_dedupe) url_path = get_url(name, uuid, should_dedupe) + strip_exif_data(content_type, file.path) + File.mkdir_p!(upload_folder) result_file = Path.join(upload_folder, name) @@ -18,8 +20,6 @@ def store(%Plug.Upload{} = file, should_dedupe) do File.cp!(file.path, result_file) end - strip_exif_data(content_type, result_file) - %{ "type" => "Document", "url" => [ From 03c35e579ba4804e176b9095186c2cdc0c244331 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Tue, 28 Aug 2018 00:18:24 +0000 Subject: [PATCH 4/6] sample config: add S3 public endpoint option --- lib/mix/tasks/sample_config.eex | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/mix/tasks/sample_config.eex b/lib/mix/tasks/sample_config.eex index cfb5ef4c0..2acf35ed9 100644 --- a/lib/mix/tasks/sample_config.eex +++ b/lib/mix/tasks/sample_config.eex @@ -25,18 +25,21 @@ config :pleroma, Pleroma.Repo, hostname: "localhost", pool_size: 10 -# Configure S3 support if desired: +# Configure S3 support if desired. +# The public S3 endpoint is different depending on region and provider, +# consult your S3 provider's documentation for details on what to use. # # config :pleroma, Pleroma.Upload, # use_s3: true, -# bucket: "some-bucket" +# bucket: "some-bucket", +# public_endpoint: "https://s3.amazonaws.com" # # Configure S3 credentials: # config :ex_aws, :s3, # access_key_id: "xxxxxxxxxxxxx", # secret_access_key: "yyyyyyyyyyyy", # region: "us-east-1", -# scheme: "https://", +# scheme: "https://" # # For using third-party S3 clones like wasabi, also do: # config :ex_aws, :s3, From 1596185ac62bb09db6e1739b9c24317a59bbbe43 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Tue, 28 Aug 2018 00:18:44 +0000 Subject: [PATCH 5/6] upload: add the S3 support itself --- lib/pleroma/upload.ex | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex index eaff8fbf3..6bed0e8ff 100644 --- a/lib/pleroma/upload.ex +++ b/lib/pleroma/upload.ex @@ -3,6 +3,9 @@ defmodule Pleroma.Upload do alias Pleroma.Web def store(%Plug.Upload{} = file, should_dedupe) do + settings = Application.get_env(:pleroma, Pleroma.Upload) + use_s3 = Keyword.fetch!(settings, :use_s3) + content_type = get_content_type(file.path) uuid = get_uuid(file, should_dedupe) name = get_name(file, uuid, content_type, should_dedupe) @@ -20,6 +23,13 @@ def store(%Plug.Upload{} = file, should_dedupe) do File.cp!(file.path, result_file) end + url_path = + if use_s3 do + put_s3_file(name, uuid, result_file, content_type) + else + url_path + end + %{ "type" => "Document", "url" => [ @@ -33,7 +43,11 @@ def store(%Plug.Upload{} = file, should_dedupe) do } end + # XXX: does this code actually work? i am skeptical. --kaniini def store(%{"img" => "data:image/" <> image_data}, should_dedupe) do + settings = Application.get_env(:pleroma, Pleroma.Upload) + use_s3 = Keyword.fetch!(settings, :use_s3) + parsed = Regex.named_captures(~r/(?jpeg|png|gif);base64,(?.*)/, image_data) data = Base.decode64!(parsed["data"], ignore: :whitespace) uuid = UUID.generate() @@ -71,6 +85,13 @@ def store(%{"img" => "data:image/" <> image_data}, should_dedupe) do strip_exif_data(content_type, result_file) + url_path = + if use_s3 do + put_s3_file(name, uuid, result_file, content_type) + else + url_path + end + %{ "type" => "Image", "url" => [ @@ -203,4 +224,24 @@ def get_content_type(file) do _e -> "application/octet-stream" end end + + defp put_s3_file(name, uuid, path, content_type) do + settings = Application.get_env(:pleroma, Pleroma.Upload) + bucket = Keyword.fetch!(settings, :bucket) + public_endpoint = Keyword.fetch!(settings, :public_endpoint) + + {:ok, file_data} = File.read(path) + + File.rm!(path) + + s3_name = "#{uuid}/#{name}" + + {:ok, result} = + ExAws.S3.put_object(bucket, s3_name, file_data, [ + {:acl, :public_read}, + {:content_type, content_type} + ]) |> ExAws.request() + + "#{public_endpoint}/#{bucket}/#{s3_name}" + end end From d22f66655b0693ebcf9291f7e443a7d7de2d28c6 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Tue, 28 Aug 2018 00:25:30 +0000 Subject: [PATCH 6/6] upload: formatting --- lib/pleroma/upload.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex index 6bed0e8ff..a744e6fd4 100644 --- a/lib/pleroma/upload.ex +++ b/lib/pleroma/upload.ex @@ -240,7 +240,8 @@ defp put_s3_file(name, uuid, path, content_type) do ExAws.S3.put_object(bucket, s3_name, file_data, [ {:acl, :public_read}, {:content_type, content_type} - ]) |> ExAws.request() + ]) + |> ExAws.request() "#{public_endpoint}/#{bucket}/#{s3_name}" end