From 6a42641b8d806f40f697303995fb12af39a93bd8 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sat, 10 Aug 2019 21:46:36 +0300 Subject: [PATCH 01/54] Add pack.toml loading --- lib/pleroma/emoji.ex | 41 ++++++++++++++++++++++++++++------------- mix.exs | 1 + mix.lock | 1 + 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/lib/pleroma/emoji.ex b/lib/pleroma/emoji.ex index 66e20f0e4..ede734a53 100644 --- a/lib/pleroma/emoji.ex +++ b/lib/pleroma/emoji.ex @@ -143,23 +143,38 @@ defmodule Pleroma.Emoji do defp load_pack(pack_dir, emoji_groups) do pack_name = Path.basename(pack_dir) - emoji_txt = Path.join(pack_dir, "emoji.txt") + pack_toml = Path.join(pack_dir, "pack.toml") - if File.exists?(emoji_txt) do - load_from_file(emoji_txt, emoji_groups) - else - extensions = Pleroma.Config.get([:emoji, :pack_extensions]) + if File.exists?(pack_toml) do + toml = Toml.decode_file!(pack_toml) - Logger.info( - "No emoji.txt found for pack \"#{pack_name}\", assuming all #{Enum.join(extensions, ", ")} files are emoji" - ) - - make_shortcode_to_file_map(pack_dir, extensions) - |> Enum.map(fn {shortcode, rel_file} -> + toml["files"] + |> Enum.map(fn {name, rel_file} -> filename = Path.join("/emoji/#{pack_name}", rel_file) - - {shortcode, filename, [to_string(match_extra(emoji_groups, filename))]} + {name, filename, pack_name} end) + else + # Load from emoji.txt / all files + emoji_txt = Path.join(pack_dir, "emoji.txt") + + if File.exists?(emoji_txt) do + load_from_file(emoji_txt, emoji_groups) + else + extensions = Pleroma.Config.get([:emoji, :pack_extensions]) + + Logger.info( + "No emoji.txt found for pack \"#{pack_name}\", assuming all #{ + Enum.join(extensions, ", ") + } files are emoji" + ) + + make_shortcode_to_file_map(pack_dir, extensions) + |> Enum.map(fn {shortcode, rel_file} -> + filename = Path.join("/emoji/#{pack_name}", rel_file) + + {shortcode, filename, [to_string(match_extra(emoji_groups, filename))]} + end) + end end end diff --git a/mix.exs b/mix.exs index f2635da24..172f3a940 100644 --- a/mix.exs +++ b/mix.exs @@ -157,6 +157,7 @@ defmodule Pleroma.Mixfile do {:ex_rated, "~> 1.3"}, {:ex_const, "~> 0.2"}, {:plug_static_index_html, "~> 1.0.0"}, + {:toml, "~> 0.5"}, {:excoveralls, "~> 0.11.1", only: :test}, {:mox, "~> 0.5", only: :test} ] ++ oauth_deps() diff --git a/mix.lock b/mix.lock index 24b34c09c..39b9fa930 100644 --- a/mix.lock +++ b/mix.lock @@ -92,6 +92,7 @@ "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, "tesla": {:hex, :tesla, "1.3.0", "f35d72f029e608f9cdc6f6d6fcc7c66cf6d6512a70cfef9206b21b8bd0203a30", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, "~> 1.3", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "~> 4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 0.4", [hex: :mint, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.3", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm"}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, + "toml": {:hex, :toml, "0.5.2", "e471388a8726d1ce51a6b32f864b8228a1eb8edc907a0edf2bb50eab9321b526", [:mix], [], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "tzdata": {:hex, :tzdata, "0.5.21", "8cbf3607fcce69636c672d5be2bbb08687fe26639a62bdcc283d267277db7cf0", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, "ueberauth": {:hex, :ueberauth, "0.6.1", "9e90d3337dddf38b1ca2753aca9b1e53d8a52b890191cdc55240247c89230412", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, From b791a0865641eb8210380e22e04a9fb680a79dcb Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 11 Aug 2019 00:39:21 +0300 Subject: [PATCH 02/54] Implement API actions on packs That incldues listing them and downloading them from other instances or from the remote url --- .../web/emoji_api/emoji_api_controller.ex | 171 ++++++++++++++++++ lib/pleroma/web/router.ex | 22 +++ 2 files changed, 193 insertions(+) create mode 100644 lib/pleroma/web/emoji_api/emoji_api_controller.ex diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex new file mode 100644 index 000000000..49d671518 --- /dev/null +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -0,0 +1,171 @@ +defmodule Pleroma.Web.EmojiAPI.EmojiAPIController do + use Pleroma.Web, :controller + + def reload(conn, _params) do + Pleroma.Emoji.reload() + + conn |> json("ok") + end + + @emoji_dir_path Path.join( + Pleroma.Config.get!([:instance, :static_dir]), + "emoji" + ) + + def list_packs(conn, _params) do + pack_infos = + case File.ls(@emoji_dir_path) do + {:error, _} -> + %{} + + {:ok, results} -> + results + |> Enum.filter(fn file -> + dir_path = Path.join(@emoji_dir_path, file) + # Filter to only use the pack.toml packs + File.dir?(dir_path) and File.exists?(Path.join(dir_path, "pack.toml")) + end) + |> Enum.map(fn pack_name -> + pack_path = Path.join(@emoji_dir_path, pack_name) + pack_file = Path.join(pack_path, "pack.toml") + + {pack_name, Toml.decode_file!(pack_file)} + end) + # Transform into a map of pack-name => pack-data + # Check if all the files are in place and can be sent + |> Enum.map(fn {name, pack} -> + pack_path = Path.join(@emoji_dir_path, name) + + archive_for_sha = make_archive(name, pack, pack_path) + archive_sha = :crypto.hash(:sha256, archive_for_sha) |> Base.encode16() + + {name, + pack + |> put_in(["pack", "can-download"], can_download?(pack, pack_path)) + |> put_in(["pack", "download-sha256"], archive_sha)} + end) + |> Enum.into(%{}) + end + + conn |> json(pack_infos) + end + + defp can_download?(pack, pack_path) do + # If the pack is set as shared, check if it can be downloaded + # That means that when asked, the pack can be packed and sent to the remote + # Otherwise, they'd have to download it from external-src + pack["pack"]["share-files"] and + Enum.all?(pack["files"], fn {_, path} -> + File.exists?(Path.join(pack_path, path)) + end) + end + + defp make_archive(name, pack, pack_dir) do + files = + ['pack.toml'] ++ + (pack["files"] |> Enum.map(fn {_, path} -> to_charlist(path) end)) + + {:ok, {_, zip_result}} = :zip.zip('#{name}.zip', files, [:memory, cwd: to_charlist(pack_dir)]) + + zip_result + end + + def download_shared(conn, %{"name" => name}) do + pack_dir = Path.join(@emoji_dir_path, name) + pack_toml = Path.join(pack_dir, "pack.toml") + + if File.exists?(pack_toml) do + pack = Toml.decode_file!(pack_toml) + + if can_download?(pack, pack_dir) do + zip_result = make_archive(name, pack, pack_dir) + + conn + |> send_download({:binary, zip_result}, filename: "#{name}.zip") + else + {:error, + conn + |> put_status(:forbidden) + |> json("Pack #{name} cannot be downloaded from this instance, either pack sharing\ + was disabled for this pack or some files are missing")} + end + else + {:error, + conn + |> put_status(:not_found) + |> json("Pack #{name} does not exist")} + end + end + + def download_from(conn, %{"instance_address" => address, "pack_name" => name} = data) do + list_uri = "#{address}/api/pleroma/emoji/packs/list" + + list = Tesla.get!(list_uri).body |> Jason.decode!() + full_pack = list[name] + pfiles = full_pack["files"] + pack = full_pack["pack"] + + pack_info_res = + cond do + pack["share-files"] && pack["can-download"] -> + {:ok, + %{ + sha: pack["download-sha256"], + uri: "#{address}/api/pleroma/emoji/packs/download_shared/#{name}" + }} + + pack["fallback-src"] -> + {:ok, + %{ + sha: pack["fallback-src-sha256"], + uri: pack["fallback-src"], + fallback: true + }} + + true -> + {:error, "The pack was not set as shared and the is no fallback url to download from"} + end + + case pack_info_res do + {:ok, %{sha: sha, uri: uri} = pinfo} -> + sha = Base.decode16!(sha) + emoji_archive = Tesla.get!(uri).body + + got_sha = :crypto.hash(:sha256, emoji_archive) + + if got_sha == sha do + local_name = data["as"] || name + pack_dir = Path.join(@emoji_dir_path, local_name) + File.mkdir_p!(pack_dir) + + files = + ['pack.toml'] ++ + (pfiles |> Enum.map(fn {_, path} -> to_charlist(path) end)) + + {:ok, _} = :zip.unzip(emoji_archive, cwd: to_charlist(pack_dir), file_list: files) + + # Fallback URL might not contain a pack.toml file, if that happens - fail (for now) + # FIXME: there seems to be a lack of any kind of encoders besides JSON. + erres = + if pinfo[:fallback] do + toml_path = Path.join(pack_dir, "pack.toml") + + unless File.exists?(toml_path) do + conn + |> put_status(:internal_server_error) + |> text("No pack.toml in falblack source") + end + end + + if not is_nil(erres), do: erres, else: conn |> text("ok") + else + conn + |> put_status(:internal_server_error) + |> text("SHA256 for the pack doesn't match the one sent by the server") + end + + {:error, e} -> + conn |> put_status(:internal_server_error) |> text(e) + end + end +end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index b9b85fd67..514446fb3 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -207,6 +207,28 @@ defmodule Pleroma.Web.Router do get("/moderation_log", AdminAPIController, :list_log) end + scope "/api/pleroma/emoji", Pleroma.Web.EmojiAPI do + scope [] do + pipe_through([:admin_api, :oauth_write]) + + post("/reload", EmojiAPIController, :reload) + end + + scope "/packs" do + # Modifying packs + pipe_through([:admin_api, :oauth_write]) + + post("/download_from", EmojiAPIController, :download_from) + end + + scope "/packs" do + # Pack info / downloading + get("/list", EmojiAPIController, :list_packs) + get("/download_shared/:name", EmojiAPIController, :download_shared) + get("/sha_of_shared/:name", EmojiAPIController, :sha_of_shared) + end + end + scope "/", Pleroma.Web.TwitterAPI do pipe_through(:pleroma_html) From 54b8e683bce13cf67f2674ea9f56b30604b28358 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 11 Aug 2019 22:32:15 +0300 Subject: [PATCH 03/54] Swap TOML for YAML to get YAML generation for packs from fallbacks If fallback url doesn't have a pack.yml file, one from the source will be used --- lib/pleroma/emoji.ex | 8 ++--- .../web/emoji_api/emoji_api_controller.ex | 36 +++++++++---------- lib/pleroma/web/router.ex | 1 - mix.exs | 2 +- mix.lock | 3 +- 5 files changed, 23 insertions(+), 27 deletions(-) diff --git a/lib/pleroma/emoji.ex b/lib/pleroma/emoji.ex index ede734a53..2a9f5f804 100644 --- a/lib/pleroma/emoji.ex +++ b/lib/pleroma/emoji.ex @@ -143,12 +143,12 @@ defmodule Pleroma.Emoji do defp load_pack(pack_dir, emoji_groups) do pack_name = Path.basename(pack_dir) - pack_toml = Path.join(pack_dir, "pack.toml") + pack_yaml = Path.join(pack_dir, "pack.yml") - if File.exists?(pack_toml) do - toml = Toml.decode_file!(pack_toml) + if File.exists?(pack_yaml) do + yaml = RelaxYaml.Decoder.read_from_file(pack_yaml) - toml["files"] + yaml["files"] |> Enum.map(fn {name, rel_file} -> filename = Path.join("/emoji/#{pack_name}", rel_file) {name, filename, pack_name} diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index 49d671518..7ef9b543d 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -22,14 +22,14 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIController do results |> Enum.filter(fn file -> dir_path = Path.join(@emoji_dir_path, file) - # Filter to only use the pack.toml packs - File.dir?(dir_path) and File.exists?(Path.join(dir_path, "pack.toml")) + # Filter to only use the pack.yml packs + File.dir?(dir_path) and File.exists?(Path.join(dir_path, "pack.yml")) end) |> Enum.map(fn pack_name -> pack_path = Path.join(@emoji_dir_path, pack_name) - pack_file = Path.join(pack_path, "pack.toml") + pack_file = Path.join(pack_path, "pack.yml") - {pack_name, Toml.decode_file!(pack_file)} + {pack_name, RelaxYaml.Decoder.read_from_file(pack_file)} end) # Transform into a map of pack-name => pack-data # Check if all the files are in place and can be sent @@ -62,7 +62,7 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIController do defp make_archive(name, pack, pack_dir) do files = - ['pack.toml'] ++ + ['pack.yml'] ++ (pack["files"] |> Enum.map(fn {_, path} -> to_charlist(path) end)) {:ok, {_, zip_result}} = :zip.zip('#{name}.zip', files, [:memory, cwd: to_charlist(pack_dir)]) @@ -72,10 +72,10 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIController do def download_shared(conn, %{"name" => name}) do pack_dir = Path.join(@emoji_dir_path, name) - pack_toml = Path.join(pack_dir, "pack.toml") + pack_yaml = Path.join(pack_dir, "pack.yml") - if File.exists?(pack_toml) do - pack = Toml.decode_file!(pack_toml) + if File.exists?(pack_yaml) do + pack = RelaxYaml.Decoder.read_from_file(pack_yaml) if can_download?(pack, pack_dir) do zip_result = make_archive(name, pack, pack_dir) @@ -139,25 +139,21 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIController do File.mkdir_p!(pack_dir) files = - ['pack.toml'] ++ + ['pack.yml'] ++ (pfiles |> Enum.map(fn {_, path} -> to_charlist(path) end)) {:ok, _} = :zip.unzip(emoji_archive, cwd: to_charlist(pack_dir), file_list: files) - # Fallback URL might not contain a pack.toml file, if that happens - fail (for now) - # FIXME: there seems to be a lack of any kind of encoders besides JSON. - erres = - if pinfo[:fallback] do - toml_path = Path.join(pack_dir, "pack.toml") + # Fallback URL might not contain a pack.yml file. Put on we have if there's none + if pinfo[:fallback] do + yaml_path = Path.join(pack_dir, "pack.yml") - unless File.exists?(toml_path) do - conn - |> put_status(:internal_server_error) - |> text("No pack.toml in falblack source") - end + unless File.exists?(yaml_path) do + File.write!(yaml_path, RelaxYaml.Encoder.encode(full_pack, [])) end + end - if not is_nil(erres), do: erres, else: conn |> text("ok") + conn |> text("ok") else conn |> put_status(:internal_server_error) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 514446fb3..1c781d750 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -225,7 +225,6 @@ defmodule Pleroma.Web.Router do # Pack info / downloading get("/list", EmojiAPIController, :list_packs) get("/download_shared/:name", EmojiAPIController, :download_shared) - get("/sha_of_shared/:name", EmojiAPIController, :sha_of_shared) end end diff --git a/mix.exs b/mix.exs index 172f3a940..e8356d564 100644 --- a/mix.exs +++ b/mix.exs @@ -157,7 +157,7 @@ defmodule Pleroma.Mixfile do {:ex_rated, "~> 1.3"}, {:ex_const, "~> 0.2"}, {:plug_static_index_html, "~> 1.0.0"}, - {:toml, "~> 0.5"}, + {:relax_yaml, "~> 0.1"}, {:excoveralls, "~> 0.11.1", only: :test}, {:mox, "~> 0.5", only: :test} ] ++ oauth_deps() diff --git a/mix.lock b/mix.lock index 39b9fa930..8852b5f65 100644 --- a/mix.lock +++ b/mix.lock @@ -84,6 +84,7 @@ "quantum": {:hex, :quantum, "2.3.4", "72a0e8855e2adc101459eac8454787cb74ab4169de6ca50f670e72142d4960e9", [:mix], [{:calendar, "~> 0.17", [hex: :calendar, repo: "hexpm", optional: true]}, {:crontab, "~> 1.1", [hex: :crontab, repo: "hexpm", optional: false]}, {:gen_stage, "~> 0.12", [hex: :gen_stage, repo: "hexpm", optional: false]}, {:swarm, "~> 3.3", [hex: :swarm, repo: "hexpm", optional: false]}, {:timex, "~> 3.1", [hex: :timex, repo: "hexpm", optional: true]}], "hexpm"}, "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm"}, "recon": {:git, "https://github.com/ferd/recon.git", "75d70c7c08926d2f24f1ee6de14ee50fe8a52763", [tag: "2.4.0"]}, + "relax_yaml": {:hex, :relax_yaml, "0.1.4", "99e55ae80b3bd1135f4288e1ba77b816ad7de05bcb4618a1a9f983ce7c89ff32", [:mix], [{:yamerl, "~> 0.4.0", [hex: :yamerl, repo: "hexpm", optional: false]}], "hexpm"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"}, "swarm": {:hex, :swarm, "3.4.0", "64f8b30055d74640d2186c66354b33b999438692a91be275bb89cdc7e401f448", [:mix], [{:gen_state_machine, "~> 2.0", [hex: :gen_state_machine, repo: "hexpm", optional: false]}, {:libring, "~> 1.0", [hex: :libring, repo: "hexpm", optional: false]}], "hexpm"}, "sweet_xml": {:hex, :sweet_xml, "0.6.6", "fc3e91ec5dd7c787b6195757fbcf0abc670cee1e4172687b45183032221b66b8", [:mix], [], "hexpm"}, @@ -92,7 +93,6 @@ "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, "tesla": {:hex, :tesla, "1.3.0", "f35d72f029e608f9cdc6f6d6fcc7c66cf6d6512a70cfef9206b21b8bd0203a30", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, "~> 1.3", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "~> 4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 0.4", [hex: :mint, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.3", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm"}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, - "toml": {:hex, :toml, "0.5.2", "e471388a8726d1ce51a6b32f864b8228a1eb8edc907a0edf2bb50eab9321b526", [:mix], [], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "tzdata": {:hex, :tzdata, "0.5.21", "8cbf3607fcce69636c672d5be2bbb08687fe26639a62bdcc283d267277db7cf0", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, "ueberauth": {:hex, :ueberauth, "0.6.1", "9e90d3337dddf38b1ca2753aca9b1e53d8a52b890191cdc55240247c89230412", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, @@ -100,4 +100,5 @@ "unsafe": {:hex, :unsafe, "1.0.1", "a27e1874f72ee49312e0a9ec2e0b27924214a05e3ddac90e91727bc76f8613d8", [:mix], [], "hexpm"}, "web_push_encryption": {:hex, :web_push_encryption, "0.2.1", "d42cecf73420d9dc0053ba3299cc8c8d6ff2be2487d67ca2a57265868e4d9a98", [:mix], [{:httpoison, "~> 1.0", [hex: :httpoison, repo: "hexpm", optional: false]}, {:jose, "~> 1.8", [hex: :jose, repo: "hexpm", optional: false]}, {:poison, "~> 3.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm"}, "websocket_client": {:git, "https://github.com/jeremyong/websocket_client.git", "9a6f65d05ebf2725d62fb19262b21f1805a59fbf", []}, + "yamerl": {:hex, :yamerl, "0.4.0", "ae215b1242810a9bc07716b88062f1bfe06f6bc7cf68372091f630baa536df79", [:rebar3], [], "hexpm"}, } From 7fb7dd9e0e0135af467477a66692990bdaecdbe9 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 11 Aug 2019 23:24:23 +0300 Subject: [PATCH 04/54] Only find SHA256 for packs that are shared --- .../web/emoji_api/emoji_api_controller.ex | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index 7ef9b543d..915059783 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -36,13 +36,19 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIController do |> Enum.map(fn {name, pack} -> pack_path = Path.join(@emoji_dir_path, name) - archive_for_sha = make_archive(name, pack, pack_path) - archive_sha = :crypto.hash(:sha256, archive_for_sha) |> Base.encode16() + if can_download?(pack, pack_path) do + archive_for_sha = make_archive(name, pack, pack_path) + archive_sha = :crypto.hash(:sha256, archive_for_sha) |> Base.encode16() - {name, - pack - |> put_in(["pack", "can-download"], can_download?(pack, pack_path)) - |> put_in(["pack", "download-sha256"], archive_sha)} + {name, + pack + |> put_in(["pack", "can-download"], true) + |> put_in(["pack", "download-sha256"], archive_sha)} + else + {name, + pack + |> put_in(["pack", "can-download"], false)} + end end) |> Enum.into(%{}) end From 7e4c8b56eab0e92b98efbf27e373d68758de540f Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 12 Aug 2019 10:35:34 +0300 Subject: [PATCH 05/54] Add tests for emoji pack sharing --- config/test.exs | 3 +- .../instance_static/emoji/test_pack/blank.png | Bin 0 -> 95 bytes test/instance_static/emoji/test_pack/pack.yml | 13 +++ .../emoji/test_pack_nonshared/pack.yml | 13 +++ test/web/emoji_api_controller_test.exs | 98 ++++++++++++++++++ 5 files changed, 126 insertions(+), 1 deletion(-) create mode 100644 test/instance_static/emoji/test_pack/blank.png create mode 100644 test/instance_static/emoji/test_pack/pack.yml create mode 100644 test/instance_static/emoji/test_pack_nonshared/pack.yml create mode 100644 test/web/emoji_api_controller_test.exs diff --git a/config/test.exs b/config/test.exs index df512b5d7..da2778aa7 100644 --- a/config/test.exs +++ b/config/test.exs @@ -30,7 +30,8 @@ config :pleroma, :instance, notify_email: "noreply@example.com", skip_thread_containment: false, federating: false, - external_user_synchronization: false + external_user_synchronization: false, + static_dir: "test/instance_static/" config :pleroma, :activitypub, sign_object_fetches: false diff --git a/test/instance_static/emoji/test_pack/blank.png b/test/instance_static/emoji/test_pack/blank.png new file mode 100644 index 0000000000000000000000000000000000000000..8f50fa02340e7e09e562f86e00b6e4bd6ad1d565 GIT binary patch literal 95 zcmeAS@N?(olHy`uVBq!ia0vp^4Is=2Bp6=1#-sr$rjj7PU get(emoji_api_path(conn, :list_packs)) |> json_response(200) + + assert Map.has_key?(resp, "test_pack") + + pack = resp["test_pack"] + + assert Map.has_key?(pack["pack"], "download-sha256") + assert pack["pack"]["can-download"] + + assert pack["files"] == %{"blank" => "blank.png"} + + # Non-shared pack + + assert Map.has_key?(resp, "test_pack_nonshared") + + pack = resp["test_pack_nonshared"] + + refute pack["pack"]["shared"] + refute pack["pack"]["can-download"] + end + + test "downloading a shared pack from download_shared" do + conn = build_conn() + + resp = + conn + |> get(emoji_api_path(conn, :download_shared, "test_pack")) + |> response(200) + + {:ok, arch} = :zip.unzip(resp, [:memory]) + + assert Enum.find(arch, fn {n, _} -> n == 'pack.yml' end) + assert Enum.find(arch, fn {n, _} -> n == 'blank.png' end) + end + + test "downloading a shared pack from another instance via download_from" do + on_exit(fn -> + File.rm_rf!("test/instance_static/emoji/test_pack2") + end) + + mock(fn + %{ + method: :get, + url: "https://example.com/api/pleroma/emoji/packs/list" + } -> + conn = build_conn() + + conn + |> get(emoji_api_path(conn, :list_packs)) + |> json_response(200) + |> json() + + %{ + method: :get, + url: "https://example.com/api/pleroma/emoji/packs/download_shared/test_pack" + } -> + conn = build_conn() + + conn + |> get(emoji_api_path(conn, :download_shared, "test_pack")) + |> response(200) + |> text() + end) + + admin = insert(:user, info: %{is_admin: true}) + + conn = build_conn() + + assert conn + |> put_req_header("content-type", "application/json") + |> assign(:user, admin) + |> post( + emoji_api_path( + conn, + :download_from + ), + %{ + instance_address: "https://example.com", + pack_name: "test_pack", + as: "test_pack2" + } + |> Jason.encode!() + ) + |> text_response(200) == "ok" + + assert File.exists?("test/instance_static/emoji/test_pack2/pack.yml") + assert File.exists?("test/instance_static/emoji/test_pack2/blank.png") + end +end From ee620ecbf11398277551ef603355a56a53690461 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 12 Aug 2019 13:13:01 +0300 Subject: [PATCH 06/54] Add caching for emoji pack sharing --- config/config.exs | 3 +- docs/config.md | 2 + lib/pleroma/application.ex | 6 ++- .../web/emoji_api/emoji_api_controller.ex | 42 ++++++++++++++++++- 4 files changed, 50 insertions(+), 3 deletions(-) diff --git a/config/config.exs b/config/config.exs index c7e0cf09f..4c758d4a0 100644 --- a/config/config.exs +++ b/config/config.exs @@ -122,7 +122,8 @@ config :pleroma, :emoji, # Put groups that have higher priority than defaults here. Example in `docs/config/custom_emoji.md` Custom: ["/emoji/*.png", "/emoji/**/*.png"] ], - default_manifest: "https://git.pleroma.social/pleroma/emoji-index/raw/master/index.json" + default_manifest: "https://git.pleroma.social/pleroma/emoji-index/raw/master/index.json", + shared_pack_cache_seconds_per_file: 60 config :pleroma, :uri_schemes, valid_schemes: [ diff --git a/docs/config.md b/docs/config.md index 3f37fa561..1179def56 100644 --- a/docs/config.md +++ b/docs/config.md @@ -707,6 +707,8 @@ Configure OAuth 2 provider capabilities: * `pack_extensions`: A list of file extensions for emojis, when no emoji.txt for a pack is present. Example `[".png", ".gif"]` * `groups`: Emojis are ordered in groups (tags). This is an array of key-value pairs where the key is the groupname and the value the location or array of locations. `*` can be used as a wildcard. Example `[Custom: ["/emoji/*.png", "/emoji/custom/*.png"]]` * `default_manifest`: Location of the JSON-manifest. This manifest contains information about the emoji-packs you can download. Currently only one manifest can be added (no arrays). +* `shared_pack_cache_seconds_per_file`: When an emoji pack is shared, the archive is created and cached in + memory for this amount of seconds multiplied by the number of files. ## Database options diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index dabce771d..a339e2c48 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -102,10 +102,14 @@ defmodule Pleroma.Application do build_cachex("rich_media", default_ttl: :timer.minutes(120), limit: 5000), build_cachex("scrubber", limit: 2500), build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500), - build_cachex("web_resp", limit: 2500) + build_cachex("web_resp", limit: 2500), + build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10) ] end + defp emoji_packs_expiration, + do: expiration(default: :timer.seconds(5 * 60), interval: :timer.seconds(60)) + defp idempotency_expiration, do: expiration(default: :timer.seconds(6 * 60 * 60), interval: :timer.seconds(60)) diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index 915059783..8219eaaa1 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -1,6 +1,8 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIController do use Pleroma.Web, :controller + require Logger + def reload(conn, _params) do Pleroma.Emoji.reload() @@ -12,6 +14,8 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIController do "emoji" ) + @cache_seconds_per_file Pleroma.Config.get!([:emoji, :shared_pack_cache_seconds_per_file]) + def list_packs(conn, _params) do pack_infos = case File.ls(@emoji_dir_path) do @@ -66,13 +70,49 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIController do end) end - defp make_archive(name, pack, pack_dir) do + defp create_archive_and_cache(name, pack, pack_dir, md5) do files = ['pack.yml'] ++ (pack["files"] |> Enum.map(fn {_, path} -> to_charlist(path) end)) {:ok, {_, zip_result}} = :zip.zip('#{name}.zip', files, [:memory, cwd: to_charlist(pack_dir)]) + cache_ms = :timer.seconds(@cache_seconds_per_file * Enum.count(files)) + + Cachex.put!( + :emoji_packs_cache, + name, + # if pack.yml MD5 changes, the cache is not valid anymore + %{pack_yml_md5: md5, pack_data: zip_result}, + # Add a minute to cache time for every file in the pack + ttl: cache_ms + ) + + Logger.debug("Create an archive for the '#{name}' shared emoji pack, \ +keeping it in cache for #{div(cache_ms, 1000)}s") + + zip_result + end + + defp make_archive(name, pack, pack_dir) do + # Having a different pack.yml md5 invalidates cache + pack_yml_md5 = :crypto.hash(:md5, File.read!(Path.join(pack_dir, "pack.yml"))) + + maybe_cached_pack = Cachex.get!(:emoji_packs_cache, name) + + zip_result = + if is_nil(maybe_cached_pack) do + create_archive_and_cache(name, pack, pack_dir, pack_yml_md5) + else + if maybe_cached_pack[:pack_yml_md5] == pack_yml_md5 do + Logger.debug("Using cache for the '#{name}' shared emoji pack") + + maybe_cached_pack[:pack_data] + else + create_archive_and_cache(name, pack, pack_dir, pack_yml_md5) + end + end + zip_result end From 7a0c755d0a69157868e245b35b48ed07a7dfd3c7 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 12 Aug 2019 16:43:28 +0300 Subject: [PATCH 07/54] Send ok for emoji reloading as text, not as json --- lib/pleroma/web/emoji_api/emoji_api_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index 8219eaaa1..72daccc8c 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -6,7 +6,7 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIController do def reload(conn, _params) do Pleroma.Emoji.reload() - conn |> json("ok") + conn |> text("ok") end @emoji_dir_path Path.join( From 3a8669b48771ac4203b6abf2a372c6960d36345a Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 12 Aug 2019 17:35:25 +0300 Subject: [PATCH 08/54] Fix responses for emoji pack controlller --- lib/pleroma/web/emoji_api/emoji_api_controller.ex | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index 72daccc8c..f2b1e8a8d 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -88,7 +88,7 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIController do ttl: cache_ms ) - Logger.debug("Create an archive for the '#{name}' shared emoji pack, \ + Logger.debug("Create an archive for the '#{name}' emoji pack, \ keeping it in cache for #{div(cache_ms, 1000)}s") zip_result @@ -132,14 +132,14 @@ keeping it in cache for #{div(cache_ms, 1000)}s") {:error, conn |> put_status(:forbidden) - |> json("Pack #{name} cannot be downloaded from this instance, either pack sharing\ + |> text("Pack #{name} cannot be downloaded from this instance, either pack sharing\ was disabled for this pack or some files are missing")} end else {:error, conn |> put_status(:not_found) - |> json("Pack #{name} does not exist")} + |> text("Pack #{name} does not exist")} end end @@ -169,7 +169,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") }} true -> - {:error, "The pack was not set as shared and the is no fallback url to download from"} + {:error, "The pack was not set as shared and there is no fallback src to download from"} end case pack_info_res do From 2d4b8f3d20c4dbf60e52e95e77f2e77766974402 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 12 Aug 2019 18:03:59 +0300 Subject: [PATCH 09/54] Add an endpoint for deleting emoji packs --- lib/pleroma/web/emoji_api/emoji_api_controller.ex | 12 ++++++++++++ lib/pleroma/web/router.ex | 1 + test/web/emoji_api_controller_test.exs | 9 ++++++++- 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index f2b1e8a8d..49d970277 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -210,4 +210,16 @@ keeping it in cache for #{div(cache_ms, 1000)}s") conn |> put_status(:internal_server_error) |> text(e) end end + + def delete(conn, %{"name" => name}) do + pack_dir = Path.join(@emoji_dir_path, name) + + case File.rm_rf(pack_dir) do + {:ok, _} -> + conn |> text("ok") + + {:error, _} -> + conn |> put_status(:internal_server_error) |> text("Couldn't delete the pack #{name}") + end + end end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 1c781d750..4df0ca3c3 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -218,6 +218,7 @@ defmodule Pleroma.Web.Router do # Modifying packs pipe_through([:admin_api, :oauth_write]) + delete("/delete/:name", EmojiAPIController, :delete) post("/download_from", EmojiAPIController, :download_from) end diff --git a/test/web/emoji_api_controller_test.exs b/test/web/emoji_api_controller_test.exs index c037883ee..13a34d38d 100644 --- a/test/web/emoji_api_controller_test.exs +++ b/test/web/emoji_api_controller_test.exs @@ -42,7 +42,7 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIControllerTest do assert Enum.find(arch, fn {n, _} -> n == 'blank.png' end) end - test "downloading a shared pack from another instance via download_from" do + test "downloading a shared pack from another instance via download_from, deleting it" do on_exit(fn -> File.rm_rf!("test/instance_static/emoji/test_pack2") end) @@ -94,5 +94,12 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIControllerTest do assert File.exists?("test/instance_static/emoji/test_pack2/pack.yml") assert File.exists?("test/instance_static/emoji/test_pack2/blank.png") + + assert conn + |> assign(:user, admin) + |> delete(emoji_api_path(conn, :delete, "test_pack2")) + |> response(200) == "ok" + + refute File.exists?("test/instance_static/emoji/test_pack2") end end From b0ecd412f5c499773cdc462c50d6c8104a819550 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 12 Aug 2019 18:28:05 +0300 Subject: [PATCH 10/54] Clean out old emojis on reload --- lib/pleroma/emoji.ex | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/pleroma/emoji.ex b/lib/pleroma/emoji.ex index 2a9f5f804..f56b26da2 100644 --- a/lib/pleroma/emoji.ex +++ b/lib/pleroma/emoji.ex @@ -122,6 +122,9 @@ defmodule Pleroma.Emoji do fn pack -> load_pack(Path.join(emoji_dir_path, pack), emoji_groups) end ) + # Clear out old emojis + :ets.delete_all_objects(@ets) + true = :ets.insert(@ets, emojis) end From 2a94eca096f67a908410ffdd82f5bace8a3df88c Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Thu, 15 Aug 2019 11:39:39 +0300 Subject: [PATCH 11/54] Change YAML to JSON --- lib/pleroma/emoji.ex | 8 ++-- .../web/emoji_api/emoji_api_controller.ex | 40 +++++++++---------- mix.exs | 1 - mix.lock | 1 - .../instance_static/emoji/test_pack/pack.json | 16 ++++++++ test/instance_static/emoji/test_pack/pack.yml | 13 ------ .../emoji/test_pack_nonshared/pack.json | 16 ++++++++ .../emoji/test_pack_nonshared/pack.yml | 13 ------ test/web/emoji_api_controller_test.exs | 4 +- 9 files changed, 58 insertions(+), 54 deletions(-) create mode 100644 test/instance_static/emoji/test_pack/pack.json delete mode 100644 test/instance_static/emoji/test_pack/pack.yml create mode 100644 test/instance_static/emoji/test_pack_nonshared/pack.json delete mode 100644 test/instance_static/emoji/test_pack_nonshared/pack.yml diff --git a/lib/pleroma/emoji.ex b/lib/pleroma/emoji.ex index f56b26da2..170a7d098 100644 --- a/lib/pleroma/emoji.ex +++ b/lib/pleroma/emoji.ex @@ -146,12 +146,12 @@ defmodule Pleroma.Emoji do defp load_pack(pack_dir, emoji_groups) do pack_name = Path.basename(pack_dir) - pack_yaml = Path.join(pack_dir, "pack.yml") + pack_file = Path.join(pack_dir, "pack.json") - if File.exists?(pack_yaml) do - yaml = RelaxYaml.Decoder.read_from_file(pack_yaml) + if File.exists?(pack_file) do + contents = Jason.decode!(File.read!(pack_file)) - yaml["files"] + contents["files"] |> Enum.map(fn {name, rel_file} -> filename = Path.join("/emoji/#{pack_name}", rel_file) {name, filename, pack_name} diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index 49d970277..aedc70372 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -26,14 +26,14 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIController do results |> Enum.filter(fn file -> dir_path = Path.join(@emoji_dir_path, file) - # Filter to only use the pack.yml packs - File.dir?(dir_path) and File.exists?(Path.join(dir_path, "pack.yml")) + # Filter to only use the pack.json packs + File.dir?(dir_path) and File.exists?(Path.join(dir_path, "pack.json")) end) |> Enum.map(fn pack_name -> pack_path = Path.join(@emoji_dir_path, pack_name) - pack_file = Path.join(pack_path, "pack.yml") + pack_file = Path.join(pack_path, "pack.json") - {pack_name, RelaxYaml.Decoder.read_from_file(pack_file)} + {pack_name, Jason.decode!(File.read!(pack_file))} end) # Transform into a map of pack-name => pack-data # Check if all the files are in place and can be sent @@ -72,7 +72,7 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIController do defp create_archive_and_cache(name, pack, pack_dir, md5) do files = - ['pack.yml'] ++ + ['pack.json'] ++ (pack["files"] |> Enum.map(fn {_, path} -> to_charlist(path) end)) {:ok, {_, zip_result}} = :zip.zip('#{name}.zip', files, [:memory, cwd: to_charlist(pack_dir)]) @@ -82,8 +82,8 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIController do Cachex.put!( :emoji_packs_cache, name, - # if pack.yml MD5 changes, the cache is not valid anymore - %{pack_yml_md5: md5, pack_data: zip_result}, + # if pack.json MD5 changes, the cache is not valid anymore + %{pack_json_md5: md5, pack_data: zip_result}, # Add a minute to cache time for every file in the pack ttl: cache_ms ) @@ -95,21 +95,21 @@ keeping it in cache for #{div(cache_ms, 1000)}s") end defp make_archive(name, pack, pack_dir) do - # Having a different pack.yml md5 invalidates cache - pack_yml_md5 = :crypto.hash(:md5, File.read!(Path.join(pack_dir, "pack.yml"))) + # Having a different pack.json md5 invalidates cache + pack_file_md5 = :crypto.hash(:md5, File.read!(Path.join(pack_dir, "pack.json"))) maybe_cached_pack = Cachex.get!(:emoji_packs_cache, name) zip_result = if is_nil(maybe_cached_pack) do - create_archive_and_cache(name, pack, pack_dir, pack_yml_md5) + create_archive_and_cache(name, pack, pack_dir, pack_file_md5) else - if maybe_cached_pack[:pack_yml_md5] == pack_yml_md5 do + if maybe_cached_pack[:pack_file_md5] == pack_file_md5 do Logger.debug("Using cache for the '#{name}' shared emoji pack") maybe_cached_pack[:pack_data] else - create_archive_and_cache(name, pack, pack_dir, pack_yml_md5) + create_archive_and_cache(name, pack, pack_dir, pack_file_md5) end end @@ -118,10 +118,10 @@ keeping it in cache for #{div(cache_ms, 1000)}s") def download_shared(conn, %{"name" => name}) do pack_dir = Path.join(@emoji_dir_path, name) - pack_yaml = Path.join(pack_dir, "pack.yml") + pack_file = Path.join(pack_dir, "pack.json") - if File.exists?(pack_yaml) do - pack = RelaxYaml.Decoder.read_from_file(pack_yaml) + if File.exists?(pack_file) do + pack = Jason.decode!(File.read!(pack_file)) if can_download?(pack, pack_dir) do zip_result = make_archive(name, pack, pack_dir) @@ -185,17 +185,17 @@ keeping it in cache for #{div(cache_ms, 1000)}s") File.mkdir_p!(pack_dir) files = - ['pack.yml'] ++ + ['pack.json'] ++ (pfiles |> Enum.map(fn {_, path} -> to_charlist(path) end)) {:ok, _} = :zip.unzip(emoji_archive, cwd: to_charlist(pack_dir), file_list: files) - # Fallback URL might not contain a pack.yml file. Put on we have if there's none + # Fallback URL might not contain a pack.json file. Put on we have if there's none if pinfo[:fallback] do - yaml_path = Path.join(pack_dir, "pack.yml") + pack_file_path = Path.join(pack_dir, "pack.json") - unless File.exists?(yaml_path) do - File.write!(yaml_path, RelaxYaml.Encoder.encode(full_pack, [])) + unless File.exists?(pack_file_path) do + File.write!(pack_file_path, Jason.encode!(full_pack)) end end diff --git a/mix.exs b/mix.exs index e8356d564..f2635da24 100644 --- a/mix.exs +++ b/mix.exs @@ -157,7 +157,6 @@ defmodule Pleroma.Mixfile do {:ex_rated, "~> 1.3"}, {:ex_const, "~> 0.2"}, {:plug_static_index_html, "~> 1.0.0"}, - {:relax_yaml, "~> 0.1"}, {:excoveralls, "~> 0.11.1", only: :test}, {:mox, "~> 0.5", only: :test} ] ++ oauth_deps() diff --git a/mix.lock b/mix.lock index 8852b5f65..d27041b96 100644 --- a/mix.lock +++ b/mix.lock @@ -84,7 +84,6 @@ "quantum": {:hex, :quantum, "2.3.4", "72a0e8855e2adc101459eac8454787cb74ab4169de6ca50f670e72142d4960e9", [:mix], [{:calendar, "~> 0.17", [hex: :calendar, repo: "hexpm", optional: true]}, {:crontab, "~> 1.1", [hex: :crontab, repo: "hexpm", optional: false]}, {:gen_stage, "~> 0.12", [hex: :gen_stage, repo: "hexpm", optional: false]}, {:swarm, "~> 3.3", [hex: :swarm, repo: "hexpm", optional: false]}, {:timex, "~> 3.1", [hex: :timex, repo: "hexpm", optional: true]}], "hexpm"}, "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm"}, "recon": {:git, "https://github.com/ferd/recon.git", "75d70c7c08926d2f24f1ee6de14ee50fe8a52763", [tag: "2.4.0"]}, - "relax_yaml": {:hex, :relax_yaml, "0.1.4", "99e55ae80b3bd1135f4288e1ba77b816ad7de05bcb4618a1a9f983ce7c89ff32", [:mix], [{:yamerl, "~> 0.4.0", [hex: :yamerl, repo: "hexpm", optional: false]}], "hexpm"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"}, "swarm": {:hex, :swarm, "3.4.0", "64f8b30055d74640d2186c66354b33b999438692a91be275bb89cdc7e401f448", [:mix], [{:gen_state_machine, "~> 2.0", [hex: :gen_state_machine, repo: "hexpm", optional: false]}, {:libring, "~> 1.0", [hex: :libring, repo: "hexpm", optional: false]}], "hexpm"}, "sweet_xml": {:hex, :sweet_xml, "0.6.6", "fc3e91ec5dd7c787b6195757fbcf0abc670cee1e4172687b45183032221b66b8", [:mix], [], "hexpm"}, diff --git a/test/instance_static/emoji/test_pack/pack.json b/test/instance_static/emoji/test_pack/pack.json new file mode 100644 index 000000000..1b260f0f7 --- /dev/null +++ b/test/instance_static/emoji/test_pack/pack.json @@ -0,0 +1,16 @@ +{ + "pack": { + "license": "Test license", + "homepage": "https://pleroma.social", + "description": "Test description", + + "fallblack-src": "https://example.com", + "fallback-src-sha256": "65CDCCBCA9388A68023519F997367783BE69ED42864398CAC568E56F65CE0E75", + + "share-files": true + }, + + "files": { + "blank": "blank.png" + } +} diff --git a/test/instance_static/emoji/test_pack/pack.yml b/test/instance_static/emoji/test_pack/pack.yml deleted file mode 100644 index 851b06d17..000000000 --- a/test/instance_static/emoji/test_pack/pack.yml +++ /dev/null @@ -1,13 +0,0 @@ -pack: - license: Test license - homepage: https://pleroma.social - description: Test description - - fallblack-src: https://example.com - # SHA256 of the fallback-src - fallback-src-sha256: 65CDCCBCA9388A68023519F997367783BE69ED42864398CAC568E56F65CE0E75 - - share-files: true - -files: - blank: blank.png diff --git a/test/instance_static/emoji/test_pack_nonshared/pack.json b/test/instance_static/emoji/test_pack_nonshared/pack.json new file mode 100644 index 000000000..b49b1efe7 --- /dev/null +++ b/test/instance_static/emoji/test_pack_nonshared/pack.json @@ -0,0 +1,16 @@ +{ + "pack": { + "license": "Test license", + "homepage": "https://pleroma.social", + "description": "Test description", + + "fallblack-src": "https://example.com", + "fallback-src-sha256": "65CDCCBCA9388A68023519F997367783BE69ED42864398CAC568E56F65CE0E75", + + "share-files": false + }, + + "files": { + "blank": "blank.png" + } +} diff --git a/test/instance_static/emoji/test_pack_nonshared/pack.yml b/test/instance_static/emoji/test_pack_nonshared/pack.yml deleted file mode 100644 index 45c340415..000000000 --- a/test/instance_static/emoji/test_pack_nonshared/pack.yml +++ /dev/null @@ -1,13 +0,0 @@ -pack: - license: Test license - homepage: https://pleroma.social - description: Test description - - fallblack-src: https://example.com - # SHA256 of the fallback-src - fallback-src-sha256: 65CDCCBCA9388A68023519F997367783BE69ED42864398CAC568E56F65CE0E75 - - share-files: false - -files: - blank: blank.png diff --git a/test/web/emoji_api_controller_test.exs b/test/web/emoji_api_controller_test.exs index 13a34d38d..bf56c1516 100644 --- a/test/web/emoji_api_controller_test.exs +++ b/test/web/emoji_api_controller_test.exs @@ -38,7 +38,7 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIControllerTest do {:ok, arch} = :zip.unzip(resp, [:memory]) - assert Enum.find(arch, fn {n, _} -> n == 'pack.yml' end) + assert Enum.find(arch, fn {n, _} -> n == 'pack.json' end) assert Enum.find(arch, fn {n, _} -> n == 'blank.png' end) end @@ -92,7 +92,7 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIControllerTest do ) |> text_response(200) == "ok" - assert File.exists?("test/instance_static/emoji/test_pack2/pack.yml") + assert File.exists?("test/instance_static/emoji/test_pack2/pack.json") assert File.exists?("test/instance_static/emoji/test_pack2/blank.png") assert conn From b78973d27f0c9225104914c79cf93bf3589fe7cc Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Thu, 15 Aug 2019 11:46:03 +0300 Subject: [PATCH 12/54] fallback can't have pack.json, reflect that in code having pacj.json and sha256 in a fallback pack would cause a circular dependency of itself --- lib/pleroma/web/emoji_api/emoji_api_controller.ex | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index aedc70372..3b9eab8b8 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -184,19 +184,19 @@ keeping it in cache for #{div(cache_ms, 1000)}s") pack_dir = Path.join(@emoji_dir_path, local_name) File.mkdir_p!(pack_dir) + # Fallback cannot contain a pack.json file files = - ['pack.json'] ++ + unless(pinfo[:fallback], do: ['pack.json'], else: []) ++ (pfiles |> Enum.map(fn {_, path} -> to_charlist(path) end)) {:ok, _} = :zip.unzip(emoji_archive, cwd: to_charlist(pack_dir), file_list: files) - # Fallback URL might not contain a pack.json file. Put on we have if there's none + # Fallback can't contain a pack.json file, since that would cause the fallback-src-sha256 + # in it to depend on itself if pinfo[:fallback] do pack_file_path = Path.join(pack_dir, "pack.json") - unless File.exists?(pack_file_path) do - File.write!(pack_file_path, Jason.encode!(full_pack)) - end + File.write!(pack_file_path, Jason.encode!(full_pack)) end conn |> text("ok") From adf31d596e77ef71e2ffe80d9dc41988f6c1cfb5 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Thu, 15 Aug 2019 12:07:51 +0300 Subject: [PATCH 13/54] Add tests for downloading from fallback url --- .../emoji/test_pack_nonshared/nonshared.zip | Bin 0 -> 256 bytes .../emoji/test_pack_nonshared/pack.json | 4 +- test/web/emoji_api_controller_test.exs | 40 +++++++++++++++++- 3 files changed, 41 insertions(+), 3 deletions(-) create mode 100644 test/instance_static/emoji/test_pack_nonshared/nonshared.zip diff --git a/test/instance_static/emoji/test_pack_nonshared/nonshared.zip b/test/instance_static/emoji/test_pack_nonshared/nonshared.zip new file mode 100644 index 0000000000000000000000000000000000000000..148446c642ea24b494bc3e25ccd772faaf2f2a13 GIT binary patch literal 256 zcmWIWW@Zs#U|`^2I2p(9A0OT*8Uf_R12HFq3`0^*VqUghL0)=j2qy#cF4@r7Q$So= z!Og(P@`9Ox0ZhE+`B41)>7++V2?-CrektH&y2Pt+hC@XnZuhYzjGD_PDeO;RYuj`( zUAMu8(_j4f1g>LGSdR&<=@xdWn#IJs;|^bzfkATSK6P%elQ2Vo rHzSiAGcLzT0G$W{OBz8ml2chBPDOKOfHx}}NFgH-`UC0NAPxfnZrnv? literal 0 HcmV?d00001 diff --git a/test/instance_static/emoji/test_pack_nonshared/pack.json b/test/instance_static/emoji/test_pack_nonshared/pack.json index b49b1efe7..b96781f81 100644 --- a/test/instance_static/emoji/test_pack_nonshared/pack.json +++ b/test/instance_static/emoji/test_pack_nonshared/pack.json @@ -4,8 +4,8 @@ "homepage": "https://pleroma.social", "description": "Test description", - "fallblack-src": "https://example.com", - "fallback-src-sha256": "65CDCCBCA9388A68023519F997367783BE69ED42864398CAC568E56F65CE0E75", + "fallback-src": "https://nonshared-pack", + "fallback-src-sha256": "74409E2674DAA06C072729C6C8426C4CB3B7E0B85ED77792DB7A436E11D76DAF", "share-files": false }, diff --git a/test/web/emoji_api_controller_test.exs b/test/web/emoji_api_controller_test.exs index bf56c1516..aa30e3058 100644 --- a/test/web/emoji_api_controller_test.exs +++ b/test/web/emoji_api_controller_test.exs @@ -42,9 +42,10 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIControllerTest do assert Enum.find(arch, fn {n, _} -> n == 'blank.png' end) end - test "downloading a shared pack from another instance via download_from, deleting it" do + test "downloading shared & unshared packs from another instance via download_from, deleting them" do on_exit(fn -> File.rm_rf!("test/instance_static/emoji/test_pack2") + File.rm_rf!("test/instance_static/emoji/test_pack_nonshared2") end) mock(fn @@ -69,6 +70,12 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIControllerTest do |> get(emoji_api_path(conn, :download_shared, "test_pack")) |> response(200) |> text() + + %{ + method: :get, + url: "https://nonshared-pack" + } -> + text(File.read!("test/instance_static/emoji/test_pack_nonshared/nonshared.zip")) end) admin = insert(:user, info: %{is_admin: true}) @@ -101,5 +108,36 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIControllerTest do |> response(200) == "ok" refute File.exists?("test/instance_static/emoji/test_pack2") + + # non-shared, downloaded from the fallback URL + + conn = build_conn() + + assert conn + |> put_req_header("content-type", "application/json") + |> assign(:user, admin) + |> post( + emoji_api_path( + conn, + :download_from + ), + %{ + instance_address: "https://example.com", + pack_name: "test_pack_nonshared", + as: "test_pack_nonshared2" + } + |> Jason.encode!() + ) + |> text_response(200) == "ok" + + assert File.exists?("test/instance_static/emoji/test_pack_nonshared2/pack.json") + assert File.exists?("test/instance_static/emoji/test_pack_nonshared2/blank.png") + + assert conn + |> assign(:user, admin) + |> delete(emoji_api_path(conn, :delete, "test_pack_nonshared2")) + |> response(200) == "ok" + + refute File.exists?("test/instance_static/emoji/test_pack_nonshared2") end end From bcc0bfd0c54784fe6a7ccd88fc083bd09dca41af Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Thu, 15 Aug 2019 19:55:58 +0300 Subject: [PATCH 14/54] Add an endpoint for emoji pack metadata updating --- .../web/emoji_api/emoji_api_controller.ex | 49 ++++++++++++++++++- lib/pleroma/web/router.ex | 1 + 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index 3b9eab8b8..4096ccbed 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -196,7 +196,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") if pinfo[:fallback] do pack_file_path = Path.join(pack_dir, "pack.json") - File.write!(pack_file_path, Jason.encode!(full_pack)) + File.write!(pack_file_path, Jason.encode!(full_pack, pretty: true)) end conn |> text("ok") @@ -222,4 +222,51 @@ keeping it in cache for #{div(cache_ms, 1000)}s") conn |> put_status(:internal_server_error) |> text("Couldn't delete the pack #{name}") end end + + def update_metadata(conn, %{"name" => name, "new_data" => new_data}) do + pack_dir = Path.join(@emoji_dir_path, name) + pack_file_p = Path.join(pack_dir, "pack.json") + + full_pack = Jason.decode!(File.read!(pack_file_p)) + + new_data = + if not is_nil(new_data["fallback-src"]) and is_nil(new_data["fallback-src-sha256"]) do + pack_arch = Tesla.get!(new_data["fallback-src"]).body + + {:ok, flist} = :zip.unzip(pack_arch, [:memory]) + + # Check if all files from the pack.json are in the archive + has_all_files = + Enum.all?(full_pack["files"], fn {_, from_manifest} -> + Enum.find(flist, fn {from_archive, _} -> + to_string(from_archive) == from_manifest + end) + end) + + unless has_all_files do + {:error, + conn + |> put_status(:bad_request) + |> text("The fallback archive does not have all files specified in pack.json")} + else + fallback_sha = :crypto.hash(:sha256, pack_arch) |> Base.encode16() + + {:ok, new_data |> Map.put("fallback-src-sha256", fallback_sha)} + end + else + {:ok, new_data} + end + + case new_data do + {:ok, new_data} -> + full_pack = Map.put(full_pack, "pack", new_data) + File.write!(pack_file_p, Jason.encode!(full_pack, pretty: true)) + + # Send new data back with fallback sha filled + conn |> json(new_data) + + {:error, e} -> + e + end + end end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 4df0ca3c3..471d09c43 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -218,6 +218,7 @@ defmodule Pleroma.Web.Router do # Modifying packs pipe_through([:admin_api, :oauth_write]) + post("/update_metadata/:name", EmojiAPIController, :update_metadata) delete("/delete/:name", EmojiAPIController, :delete) post("/download_from", EmojiAPIController, :download_from) end From 9dc9689144a54f3e5513dd26de61ec43421d6d50 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Fri, 16 Aug 2019 13:22:14 +0300 Subject: [PATCH 15/54] Add tests for pack metadata updating --- .../instance_static/emoji/test_pack/pack.json | 3 - test/web/emoji_api_controller_test.exs | 118 ++++++++++++++++-- 2 files changed, 109 insertions(+), 12 deletions(-) diff --git a/test/instance_static/emoji/test_pack/pack.json b/test/instance_static/emoji/test_pack/pack.json index 1b260f0f7..5a8ee75f9 100644 --- a/test/instance_static/emoji/test_pack/pack.json +++ b/test/instance_static/emoji/test_pack/pack.json @@ -4,9 +4,6 @@ "homepage": "https://pleroma.social", "description": "Test description", - "fallblack-src": "https://example.com", - "fallback-src-sha256": "65CDCCBCA9388A68023519F997367783BE69ED42864398CAC568E56F65CE0E75", - "share-files": true }, diff --git a/test/web/emoji_api_controller_test.exs b/test/web/emoji_api_controller_test.exs index aa30e3058..759a4dc04 100644 --- a/test/web/emoji_api_controller_test.exs +++ b/test/web/emoji_api_controller_test.exs @@ -5,6 +5,11 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIControllerTest do import Pleroma.Factory + @emoji_dir_path Path.join( + Pleroma.Config.get!([:instance, :static_dir]), + "emoji" + ) + test "shared & non-shared pack information in list_packs is ok" do conn = build_conn() resp = conn |> get(emoji_api_path(conn, :list_packs)) |> json_response(200) @@ -44,8 +49,8 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIControllerTest do test "downloading shared & unshared packs from another instance via download_from, deleting them" do on_exit(fn -> - File.rm_rf!("test/instance_static/emoji/test_pack2") - File.rm_rf!("test/instance_static/emoji/test_pack_nonshared2") + File.rm_rf!("#{@emoji_dir_path}/test_pack2") + File.rm_rf!("#{@emoji_dir_path}/test_pack_nonshared2") end) mock(fn @@ -75,7 +80,7 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIControllerTest do method: :get, url: "https://nonshared-pack" } -> - text(File.read!("test/instance_static/emoji/test_pack_nonshared/nonshared.zip")) + text(File.read!("#{@emoji_dir_path}/test_pack_nonshared/nonshared.zip")) end) admin = insert(:user, info: %{is_admin: true}) @@ -99,15 +104,15 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIControllerTest do ) |> text_response(200) == "ok" - assert File.exists?("test/instance_static/emoji/test_pack2/pack.json") - assert File.exists?("test/instance_static/emoji/test_pack2/blank.png") + assert File.exists?("#{@emoji_dir_path}/test_pack2/pack.json") + assert File.exists?("#{@emoji_dir_path}/test_pack2/blank.png") assert conn |> assign(:user, admin) |> delete(emoji_api_path(conn, :delete, "test_pack2")) |> response(200) == "ok" - refute File.exists?("test/instance_static/emoji/test_pack2") + refute File.exists?("#{@emoji_dir_path}/test_pack2") # non-shared, downloaded from the fallback URL @@ -130,14 +135,109 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIControllerTest do ) |> text_response(200) == "ok" - assert File.exists?("test/instance_static/emoji/test_pack_nonshared2/pack.json") - assert File.exists?("test/instance_static/emoji/test_pack_nonshared2/blank.png") + assert File.exists?("#{@emoji_dir_path}/test_pack_nonshared2/pack.json") + assert File.exists?("#{@emoji_dir_path}/test_pack_nonshared2/blank.png") assert conn |> assign(:user, admin) |> delete(emoji_api_path(conn, :delete, "test_pack_nonshared2")) |> response(200) == "ok" - refute File.exists?("test/instance_static/emoji/test_pack_nonshared2") + refute File.exists?("#{@emoji_dir_path}/test_pack_nonshared2") + end + + describe "updating pack metadata" do + setup do + pack_file = "#{@emoji_dir_path}/test_pack/pack.json" + original_content = File.read!(pack_file) + + on_exit(fn -> + File.write!(pack_file, original_content) + end) + + {:ok, + admin: insert(:user, info: %{is_admin: true}), + pack_file: pack_file, + new_data: %{ + "license" => "Test license changed", + "homepage" => "https://pleroma.social", + "description" => "Test description", + "share-files" => false + }} + end + + test "for a pack without a fallback source", ctx do + conn = build_conn() + + assert conn + |> assign(:user, ctx[:admin]) + |> post( + emoji_api_path(conn, :update_metadata, "test_pack"), + %{ + "new_data" => ctx[:new_data] + } + ) + |> json_response(200) == ctx[:new_data] + + assert Jason.decode!(File.read!(ctx[:pack_file]))["pack"] == ctx[:new_data] + end + + test "for a pack with a fallback source", ctx do + mock(fn + %{ + method: :get, + url: "https://nonshared-pack" + } -> + text(File.read!("#{@emoji_dir_path}/test_pack_nonshared/nonshared.zip")) + end) + + new_data = Map.put(ctx[:new_data], "fallback-src", "https://nonshared-pack") + + new_data_with_sha = + Map.put( + new_data, + "fallback-src-sha256", + "74409E2674DAA06C072729C6C8426C4CB3B7E0B85ED77792DB7A436E11D76DAF" + ) + + conn = build_conn() + + assert conn + |> assign(:user, ctx[:admin]) + |> post( + emoji_api_path(conn, :update_metadata, "test_pack"), + %{ + "new_data" => new_data + } + ) + |> json_response(200) == new_data_with_sha + + assert Jason.decode!(File.read!(ctx[:pack_file]))["pack"] == new_data_with_sha + end + + test "when the fallback source doesn't have all the files", ctx do + mock(fn + %{ + method: :get, + url: "https://nonshared-pack" + } -> + {:ok, {'empty.zip', empty_arch}} = :zip.zip('empty.zip', [], [:memory]) + text(empty_arch) + end) + + new_data = Map.put(ctx[:new_data], "fallback-src", "https://nonshared-pack") + + conn = build_conn() + + assert conn + |> assign(:user, ctx[:admin]) + |> post( + emoji_api_path(conn, :update_metadata, "test_pack"), + %{ + "new_data" => new_data + } + ) + |> text_response(:bad_request) =~ "does not have all" + end end end From 261d92f9c2605c720e7fce8b05025e5ac452e5c9 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Fri, 16 Aug 2019 13:30:14 +0300 Subject: [PATCH 16/54] Update the pack fallback-src sha generation condition The old one would not regenerate sha when fallback src changed --- lib/pleroma/web/emoji_api/emoji_api_controller.ex | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index 4096ccbed..4873129c4 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -229,8 +229,13 @@ keeping it in cache for #{div(cache_ms, 1000)}s") full_pack = Jason.decode!(File.read!(pack_file_p)) + # The new fallback-src is in the new data and it's not the same as it was in the old data + should_update_fb_sha = + not is_nil(new_data["fallback-src"]) and + new_data["fallback-src"] != full_pack["pack"]["fallback-src"] + new_data = - if not is_nil(new_data["fallback-src"]) and is_nil(new_data["fallback-src-sha256"]) do + if should_update_fb_sha do pack_arch = Tesla.get!(new_data["fallback-src"]).body {:ok, flist} = :zip.unzip(pack_arch, [:memory]) From 9afe7258dd5ca1e5a6333a5a9f93d9ab43d4aaf4 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 18 Aug 2019 22:05:38 +0300 Subject: [PATCH 17/54] Implememt emoji pack file updating + write tests --- .../web/emoji_api/emoji_api_controller.ex | 132 +++++++++++++++++- lib/pleroma/web/router.ex | 3 +- test/web/emoji_api_controller_test.exs | 69 ++++++++- 3 files changed, 196 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index 4873129c4..dc3dcf1ea 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -223,7 +223,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") end end - def update_metadata(conn, %{"name" => name, "new_data" => new_data}) do + def update_metadata(conn, %{"pack_name" => name, "new_data" => new_data}) do pack_dir = Path.join(@emoji_dir_path, name) pack_file_p = Path.join(pack_dir, "pack.json") @@ -274,4 +274,134 @@ keeping it in cache for #{div(cache_ms, 1000)}s") e end end + + def update_file( + conn, + %{"pack_name" => pack_name, "action" => action, "shortcode" => shortcode} = params + ) do + pack_dir = Path.join(@emoji_dir_path, pack_name) + pack_file_p = Path.join(pack_dir, "pack.json") + + full_pack = Jason.decode!(File.read!(pack_file_p)) + + res = + case action do + "add" -> + unless Map.has_key?(full_pack["files"], shortcode) do + with %{"file" => %Plug.Upload{filename: filename, path: upload_path}} <- params do + # If there was a file name provided with the request, use it, otherwise just use the + # uploaded file name + filename = + if Map.has_key?(params, "filename") do + params["filename"] + else + filename + end + + file_path = Path.join(pack_dir, filename) + + # If the name contains directories, create them + if String.contains?(file_path, "/") do + File.mkdir_p!(Path.dirname(file_path)) + end + + # Copy the uploaded file from the temporary directory + File.copy!(upload_path, file_path) + + updated_full_pack = put_in(full_pack, ["files", shortcode], filename) + + {:ok, updated_full_pack} + else + _ -> {:error, conn |> put_status(:bad_request) |> text("\"file\" not provided")} + end + else + {:error, + conn + |> put_status(:conflict) + |> text("An emoji with the \"#{shortcode}\" shortcode already exists")} + end + + "remove" -> + if Map.has_key?(full_pack["files"], shortcode) do + {emoji_file_path, updated_full_pack} = pop_in(full_pack, ["files", shortcode]) + + emoji_file_path = Path.join(pack_dir, emoji_file_path) + + # Delete the emoji file + File.rm!(emoji_file_path) + + # If the old directory has no more files, remove it + if String.contains?(emoji_file_path, "/") do + dir = Path.dirname(emoji_file_path) + + if Enum.empty?(File.ls!(dir)) do + File.rmdir!(dir) + end + end + + {:ok, updated_full_pack} + else + {:error, + conn |> put_status(:bad_request) |> text("Emoji \"#{shortcode}\" does not exist")} + end + + "update" -> + if Map.has_key?(full_pack["files"], shortcode) do + with %{"new_shortcode" => new_shortcode, "new_filename" => new_filename} <- params do + # First, remove the old shortcode, saving the old path + {old_emoji_file_path, updated_full_pack} = pop_in(full_pack, ["files", shortcode]) + old_emoji_file_path = Path.join(pack_dir, old_emoji_file_path) + new_emoji_file_path = Path.join(pack_dir, new_filename) + + # If the name contains directories, create them + if String.contains?(new_emoji_file_path, "/") do + File.mkdir_p!(Path.dirname(new_emoji_file_path)) + end + + # Move/Rename the old filename to a new filename + # These are probably on the same filesystem, so just rename should work + :ok = File.rename(old_emoji_file_path, new_emoji_file_path) + + # If the old directory has no more files, remove it + if String.contains?(old_emoji_file_path, "/") do + dir = Path.dirname(old_emoji_file_path) + + if Enum.empty?(File.ls!(dir)) do + File.rmdir!(dir) + end + end + + # Then, put in the new shortcode with the new path + updated_full_pack = + put_in(updated_full_pack, ["files", new_shortcode], new_filename) + + {:ok, updated_full_pack} + else + _ -> + {:error, + conn + |> put_status(:bad_request) + |> text("new_shortcode or new_file were not specified")} + end + else + {:error, + conn |> put_status(:bad_request) |> text("Emoji \"#{shortcode}\" does not exist")} + end + + _ -> + {:error, conn |> put_status(:bad_request) |> text("Unknown action: #{action}")} + end + + case res do + {:ok, updated_full_pack} -> + # Write the emoji pack file + File.write!(pack_file_p, Jason.encode!(updated_full_pack, pretty: true)) + + # Return the modified file list + conn |> json(updated_full_pack["files"]) + + {:error, e} -> + e + end + end end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 471d09c43..acd6f740b 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -218,7 +218,8 @@ defmodule Pleroma.Web.Router do # Modifying packs pipe_through([:admin_api, :oauth_write]) - post("/update_metadata/:name", EmojiAPIController, :update_metadata) + post("/update_file/:pack_name", EmojiAPIController, :update_file) + post("/update_metadata/:pack_name", EmojiAPIController, :update_metadata) delete("/delete/:name", EmojiAPIController, :delete) post("/download_from", EmojiAPIController, :download_from) end diff --git a/test/web/emoji_api_controller_test.exs b/test/web/emoji_api_controller_test.exs index 759a4dc04..6d3603da5 100644 --- a/test/web/emoji_api_controller_test.exs +++ b/test/web/emoji_api_controller_test.exs @@ -85,11 +85,10 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIControllerTest do admin = insert(:user, info: %{is_admin: true}) - conn = build_conn() + conn = build_conn() |> assign(:user, admin) assert conn |> put_req_header("content-type", "application/json") - |> assign(:user, admin) |> post( emoji_api_path( conn, @@ -108,7 +107,6 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIControllerTest do assert File.exists?("#{@emoji_dir_path}/test_pack2/blank.png") assert conn - |> assign(:user, admin) |> delete(emoji_api_path(conn, :delete, "test_pack2")) |> response(200) == "ok" @@ -116,11 +114,10 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIControllerTest do # non-shared, downloaded from the fallback URL - conn = build_conn() + conn = build_conn() |> assign(:user, admin) assert conn |> put_req_header("content-type", "application/json") - |> assign(:user, admin) |> post( emoji_api_path( conn, @@ -139,7 +136,6 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIControllerTest do assert File.exists?("#{@emoji_dir_path}/test_pack_nonshared2/blank.png") assert conn - |> assign(:user, admin) |> delete(emoji_api_path(conn, :delete, "test_pack_nonshared2")) |> response(200) == "ok" @@ -240,4 +236,65 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIControllerTest do |> text_response(:bad_request) =~ "does not have all" end end + + test "updating pack files" do + pack_file = "#{@emoji_dir_path}/test_pack/pack.json" + original_content = File.read!(pack_file) + + on_exit(fn -> + File.write!(pack_file, original_content) + + File.rm_rf!("#{@emoji_dir_path}/test_pack/dir") + File.rm_rf!("#{@emoji_dir_path}/test_pack/dir_2") + end) + + admin = insert(:user, info: %{is_admin: true}) + + conn = build_conn() + + same_name = %{ + "action" => "add", + "shortcode" => "blank", + "filename" => "dir/blank.png", + "file" => %Plug.Upload{ + filename: "blank.png", + path: "#{@emoji_dir_path}/test_pack/blank.png" + } + } + + different_name = %{same_name | "shortcode" => "blank_2"} + + conn = conn |> assign(:user, admin) + + assert conn + |> post(emoji_api_path(conn, :update_file, "test_pack"), same_name) + |> text_response(:conflict) =~ "already exists" + + assert conn + |> post(emoji_api_path(conn, :update_file, "test_pack"), different_name) + |> json_response(200) == %{"blank" => "blank.png", "blank_2" => "dir/blank.png"} + + assert File.exists?("#{@emoji_dir_path}/test_pack/dir/blank.png") + + assert conn + |> post(emoji_api_path(conn, :update_file, "test_pack"), %{ + "action" => "update", + "shortcode" => "blank_2", + "new_shortcode" => "blank_3", + "new_filename" => "dir_2/blank_3.png" + }) + |> json_response(200) == %{"blank" => "blank.png", "blank_3" => "dir_2/blank_3.png"} + + refute File.exists?("#{@emoji_dir_path}/test_pack/dir/") + assert File.exists?("#{@emoji_dir_path}/test_pack/dir_2/blank_3.png") + + assert conn + |> post(emoji_api_path(conn, :update_file, "test_pack"), %{ + "action" => "remove", + "shortcode" => "blank_3" + }) + |> json_response(200) == %{"blank" => "blank.png"} + + refute File.exists?("#{@emoji_dir_path}/test_pack/dir_2/") + end end From 16edfef12e6781971e2056a80a0ac38dcc254b1b Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 19 Aug 2019 19:26:15 +0300 Subject: [PATCH 18/54] Handle empty shortcode/filename/new_shortcode/new_filename --- .../web/emoji_api/emoji_api_controller.ex | 88 +++++++++++-------- 1 file changed, 52 insertions(+), 36 deletions(-) diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index dc3dcf1ea..fdecbb700 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -298,19 +298,27 @@ keeping it in cache for #{div(cache_ms, 1000)}s") filename end - file_path = Path.join(pack_dir, filename) + unless String.trim(shortcode) |> String.length() == 0 or + String.trim(filename) |> String.length() == 0 do + file_path = Path.join(pack_dir, filename) - # If the name contains directories, create them - if String.contains?(file_path, "/") do - File.mkdir_p!(Path.dirname(file_path)) + # If the name contains directories, create them + if String.contains?(file_path, "/") do + File.mkdir_p!(Path.dirname(file_path)) + end + + # Copy the uploaded file from the temporary directory + File.copy!(upload_path, file_path) + + updated_full_pack = put_in(full_pack, ["files", shortcode], filename) + + {:ok, updated_full_pack} + else + {:error, + conn + |> put_status(:bad_request) + |> text("shortcode or filename cannot be empty")} end - - # Copy the uploaded file from the temporary directory - File.copy!(upload_path, file_path) - - updated_full_pack = put_in(full_pack, ["files", shortcode], filename) - - {:ok, updated_full_pack} else _ -> {:error, conn |> put_status(:bad_request) |> text("\"file\" not provided")} end @@ -348,34 +356,42 @@ keeping it in cache for #{div(cache_ms, 1000)}s") "update" -> if Map.has_key?(full_pack["files"], shortcode) do with %{"new_shortcode" => new_shortcode, "new_filename" => new_filename} <- params do - # First, remove the old shortcode, saving the old path - {old_emoji_file_path, updated_full_pack} = pop_in(full_pack, ["files", shortcode]) - old_emoji_file_path = Path.join(pack_dir, old_emoji_file_path) - new_emoji_file_path = Path.join(pack_dir, new_filename) + unless String.trim(new_shortcode) |> String.length() == 0 or + String.trim(new_filename) |> String.length() == 0 do + # First, remove the old shortcode, saving the old path + {old_emoji_file_path, updated_full_pack} = pop_in(full_pack, ["files", shortcode]) + old_emoji_file_path = Path.join(pack_dir, old_emoji_file_path) + new_emoji_file_path = Path.join(pack_dir, new_filename) - # If the name contains directories, create them - if String.contains?(new_emoji_file_path, "/") do - File.mkdir_p!(Path.dirname(new_emoji_file_path)) - end - - # Move/Rename the old filename to a new filename - # These are probably on the same filesystem, so just rename should work - :ok = File.rename(old_emoji_file_path, new_emoji_file_path) - - # If the old directory has no more files, remove it - if String.contains?(old_emoji_file_path, "/") do - dir = Path.dirname(old_emoji_file_path) - - if Enum.empty?(File.ls!(dir)) do - File.rmdir!(dir) + # If the name contains directories, create them + if String.contains?(new_emoji_file_path, "/") do + File.mkdir_p!(Path.dirname(new_emoji_file_path)) end + + # Move/Rename the old filename to a new filename + # These are probably on the same filesystem, so just rename should work + :ok = File.rename(old_emoji_file_path, new_emoji_file_path) + + # If the old directory has no more files, remove it + if String.contains?(old_emoji_file_path, "/") do + dir = Path.dirname(old_emoji_file_path) + + if Enum.empty?(File.ls!(dir)) do + File.rmdir!(dir) + end + end + + # Then, put in the new shortcode with the new path + updated_full_pack = + put_in(updated_full_pack, ["files", new_shortcode], new_filename) + + {:ok, updated_full_pack} + else + {:error, + conn + |> put_status(:bad_request) + |> text("new_shortcode or new_filename cannot be empty")} end - - # Then, put in the new shortcode with the new path - updated_full_pack = - put_in(updated_full_pack, ["files", new_shortcode], new_filename) - - {:ok, updated_full_pack} else _ -> {:error, From 8dbdd5c280d15fde4712989001d4ddee1cd37cff Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 20 Aug 2019 14:52:36 +0300 Subject: [PATCH 19/54] Allow uploading new emojis to packs from URLs --- .../web/emoji_api/emoji_api_controller.ex | 65 ++++++++++--------- test/web/emoji_api_controller_test.exs | 34 ++++++++++ 2 files changed, 69 insertions(+), 30 deletions(-) diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index fdecbb700..87ae0e092 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -288,39 +288,44 @@ keeping it in cache for #{div(cache_ms, 1000)}s") case action do "add" -> unless Map.has_key?(full_pack["files"], shortcode) do - with %{"file" => %Plug.Upload{filename: filename, path: upload_path}} <- params do - # If there was a file name provided with the request, use it, otherwise just use the - # uploaded file name - filename = - if Map.has_key?(params, "filename") do - params["filename"] - else - filename - end - - unless String.trim(shortcode) |> String.length() == 0 or - String.trim(filename) |> String.length() == 0 do - file_path = Path.join(pack_dir, filename) - - # If the name contains directories, create them - if String.contains?(file_path, "/") do - File.mkdir_p!(Path.dirname(file_path)) - end - - # Copy the uploaded file from the temporary directory - File.copy!(upload_path, file_path) - - updated_full_pack = put_in(full_pack, ["files", shortcode], filename) - - {:ok, updated_full_pack} + filename = + if Map.has_key?(params, "filename") do + params["filename"] else - {:error, - conn - |> put_status(:bad_request) - |> text("shortcode or filename cannot be empty")} + case params["file"] do + %Plug.Upload{filename: filename} -> filename + url when is_binary(url) -> Path.basename(url) + end end + + unless String.trim(shortcode) |> String.length() == 0 or + String.trim(filename) |> String.length() == 0 do + file_path = Path.join(pack_dir, filename) + + # If the name contains directories, create them + if String.contains?(file_path, "/") do + File.mkdir_p!(Path.dirname(file_path)) + end + + case params["file"] do + %Plug.Upload{path: upload_path} -> + # Copy the uploaded file from the temporary directory + File.copy!(upload_path, file_path) + + url when is_binary(url) -> + # Download and write the file + file_contents = Tesla.get!(url).body + File.write!(file_path, file_contents) + end + + updated_full_pack = put_in(full_pack, ["files", shortcode], filename) + + {:ok, updated_full_pack} else - _ -> {:error, conn |> put_status(:bad_request) |> text("\"file\" not provided")} + {:error, + conn + |> put_status(:bad_request) + |> text("shortcode or filename cannot be empty")} end else {:error, diff --git a/test/web/emoji_api_controller_test.exs b/test/web/emoji_api_controller_test.exs index 6d3603da5..c1aece691 100644 --- a/test/web/emoji_api_controller_test.exs +++ b/test/web/emoji_api_controller_test.exs @@ -244,6 +244,7 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIControllerTest do on_exit(fn -> File.write!(pack_file, original_content) + File.rm_rf!("#{@emoji_dir_path}/test_pack/blank_url.png") File.rm_rf!("#{@emoji_dir_path}/test_pack/dir") File.rm_rf!("#{@emoji_dir_path}/test_pack/dir_2") end) @@ -296,5 +297,38 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIControllerTest do |> json_response(200) == %{"blank" => "blank.png"} refute File.exists?("#{@emoji_dir_path}/test_pack/dir_2/") + + mock(fn + %{ + method: :get, + url: "https://test-blank/blank_url.png" + } -> + text(File.read!("#{@emoji_dir_path}/test_pack/blank.png")) + end) + + # The name should be inferred from the URL ending + from_url = %{ + "action" => "add", + "shortcode" => "blank_url", + "file" => "https://test-blank/blank_url.png" + } + + assert conn + |> post(emoji_api_path(conn, :update_file, "test_pack"), from_url) + |> json_response(200) == %{ + "blank" => "blank.png", + "blank_url" => "blank_url.png" + } + + assert File.exists?("#{@emoji_dir_path}/test_pack/blank_url.png") + + assert conn + |> post(emoji_api_path(conn, :update_file, "test_pack"), %{ + "action" => "remove", + "shortcode" => "blank_url" + }) + |> json_response(200) == %{"blank" => "blank.png"} + + refute File.exists?("#{@emoji_dir_path}/test_pack/blank_url.png") end end From 6b4a144e4d9fa17db6fbda800511f7f41ae1c731 Mon Sep 17 00:00:00 2001 From: vaartis Date: Sat, 24 Aug 2019 21:58:21 +0000 Subject: [PATCH 20/54] Remove unused yaml dependency from mix.lock --- mix.lock | 1 - 1 file changed, 1 deletion(-) diff --git a/mix.lock b/mix.lock index d27041b96..24b34c09c 100644 --- a/mix.lock +++ b/mix.lock @@ -99,5 +99,4 @@ "unsafe": {:hex, :unsafe, "1.0.1", "a27e1874f72ee49312e0a9ec2e0b27924214a05e3ddac90e91727bc76f8613d8", [:mix], [], "hexpm"}, "web_push_encryption": {:hex, :web_push_encryption, "0.2.1", "d42cecf73420d9dc0053ba3299cc8c8d6ff2be2487d67ca2a57265868e4d9a98", [:mix], [{:httpoison, "~> 1.0", [hex: :httpoison, repo: "hexpm", optional: false]}, {:jose, "~> 1.8", [hex: :jose, repo: "hexpm", optional: false]}, {:poison, "~> 3.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm"}, "websocket_client": {:git, "https://github.com/jeremyong/websocket_client.git", "9a6f65d05ebf2725d62fb19262b21f1805a59fbf", []}, - "yamerl": {:hex, :yamerl, "0.4.0", "ae215b1242810a9bc07716b88062f1bfe06f6bc7cf68372091f630baa536df79", [:rebar3], [], "hexpm"}, } From f5131540dc9bbf8038e6625f4524ca01b52abbbf Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Wed, 28 Aug 2019 19:29:01 +0300 Subject: [PATCH 21/54] Add a way to create emoji packs via an endpoint --- .../web/emoji_api/emoji_api_controller.ex | 21 ++++++++++++ lib/pleroma/web/router.ex | 1 + test/web/emoji_api_controller_test.exs | 34 +++++++++++++++++++ 3 files changed, 56 insertions(+) diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index 87ae0e092..0bd9cd207 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -211,6 +211,27 @@ keeping it in cache for #{div(cache_ms, 1000)}s") end end + def create(conn, %{"name" => name}) do + pack_dir = Path.join(@emoji_dir_path, name) + + unless File.exists?(pack_dir) do + File.mkdir_p!(pack_dir) + + pack_file_p = Path.join(pack_dir, "pack.json") + + File.write!( + pack_file_p, + Jason.encode!(%{pack: %{}, files: %{}}) + ) + + conn |> text("ok") + else + conn + |> put_status(:conflict) + |> text("A pack named \"#{name}\" already exists") + end + end + def delete(conn, %{"name" => name}) do pack_dir = Path.join(@emoji_dir_path, name) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index acd6f740b..a21fefc70 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -220,6 +220,7 @@ defmodule Pleroma.Web.Router do post("/update_file/:pack_name", EmojiAPIController, :update_file) post("/update_metadata/:pack_name", EmojiAPIController, :update_metadata) + post("/create/:name", EmojiAPIController, :create) delete("/delete/:name", EmojiAPIController, :delete) post("/download_from", EmojiAPIController, :download_from) end diff --git a/test/web/emoji_api_controller_test.exs b/test/web/emoji_api_controller_test.exs index c1aece691..fa194a26c 100644 --- a/test/web/emoji_api_controller_test.exs +++ b/test/web/emoji_api_controller_test.exs @@ -331,4 +331,38 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIControllerTest do refute File.exists?("#{@emoji_dir_path}/test_pack/blank_url.png") end + + test "creating and deleting a pack" do + on_exit(fn -> + File.rm_rf!("#{@emoji_dir_path}/test_created") + end) + + admin = insert(:user, info: %{is_admin: true}) + + conn = build_conn() |> assign(:user, admin) + + assert conn + |> put_req_header("content-type", "application/json") + |> post( + emoji_api_path( + conn, + :create, + "test_created" + ) + ) + |> text_response(200) == "ok" + + assert File.exists?("#{@emoji_dir_path}/test_created/pack.json") + + assert Jason.decode!(File.read!("#{@emoji_dir_path}/test_created/pack.json")) == %{ + "pack" => %{}, + "files" => %{} + } + + assert conn + |> delete(emoji_api_path(conn, :delete, "test_created")) + |> response(200) == "ok" + + refute File.exists?("#{@emoji_dir_path}/test_created/pack.json") + end end From 13cd93a0d314238427c217ec0ab8f59f329321f5 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 1 Sep 2019 15:38:45 +0300 Subject: [PATCH 22/54] Use && insted of "and" for checking shared-files for packs share-files can be nil and "and" does not like that --- lib/pleroma/web/emoji_api/emoji_api_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index 0bd9cd207..f34a4e08c 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -64,7 +64,7 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIController do # If the pack is set as shared, check if it can be downloaded # That means that when asked, the pack can be packed and sent to the remote # Otherwise, they'd have to download it from external-src - pack["pack"]["share-files"] and + pack["pack"]["share-files"] && Enum.all?(pack["files"], fn {_, path} -> File.exists?(Path.join(pack_path, path)) end) From 9eb2ee4df0478daec1172eec2289868105b72756 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 10 Sep 2019 21:16:30 +0300 Subject: [PATCH 23/54] Allow importing old (emoji.txt / plain) packs from the filesystem --- .../web/emoji_api/emoji_api_controller.ex | 66 ++++++++++++++++++ lib/pleroma/web/router.ex | 2 + .../emoji/test_pack_for_import/blank.png | Bin 0 -> 95 bytes test/web/emoji_api_controller_test.exs | 41 +++++++++++ 4 files changed, 109 insertions(+) create mode 100644 test/instance_static/emoji/test_pack_for_import/blank.png diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index f34a4e08c..dffb91b0f 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -446,4 +446,70 @@ keeping it in cache for #{div(cache_ms, 1000)}s") e end end + + def import_from_fs(conn, _params) do + case File.ls(@emoji_dir_path) do + {:error, _} -> + conn + |> put_status(:internal_server_error) + |> text("Error accessing emoji pack directory") + + {:ok, results} -> + imported_pack_names = + results + |> Enum.filter(fn file -> + dir_path = Path.join(@emoji_dir_path, file) + # Find the directories that do NOT have pack.json + File.dir?(dir_path) and not File.exists?(Path.join(dir_path, "pack.json")) + end) + |> Enum.map(fn dir -> + dir_path = Path.join(@emoji_dir_path, dir) + emoji_txt_path = Path.join(dir_path, "emoji.txt") + + files_for_pack = + if File.exists?(emoji_txt_path) do + # There's an emoji.txt file, it's likely from a pack installed by the pack manager. + # Make a pack.json file from the contents of that emoji.txt fileh + + # FIXME: Copy-pasted from Pleroma.Emoji/load_from_file_stream/2 + + # Create a map of shortcodes to filenames from emoji.txt + + File.read!(emoji_txt_path) + |> String.split("\n") + |> Enum.map(&String.trim/1) + |> Enum.map(fn line -> + case String.split(line, ~r/,\s*/) do + # This matches both strings with and without tags and we don't care about tags here + [name, file | _] -> + {name, file} + + _ -> + nil + end + end) + |> Enum.filter(fn x -> not is_nil(x) end) + |> Enum.into(%{}) + else + # If there's no emoji.txt, assume all files that are of certain extensions from the config + # are emojis and import them all + Pleroma.Emoji.make_shortcode_to_file_map( + dir_path, + Pleroma.Config.get!([:emoji, :pack_extensions]) + ) + end + + pack_json_contents = Jason.encode!(%{pack: %{}, files: files_for_pack}) + + File.write!( + Path.join(dir_path, "pack.json"), + pack_json_contents + ) + + dir + end) + + conn |> json(imported_pack_names) + end + end end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index a21fefc70..1252048f0 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -218,6 +218,8 @@ defmodule Pleroma.Web.Router do # Modifying packs pipe_through([:admin_api, :oauth_write]) + post("/import_from_fs", EmojiAPIController, :import_from_fs) + post("/update_file/:pack_name", EmojiAPIController, :update_file) post("/update_metadata/:pack_name", EmojiAPIController, :update_metadata) post("/create/:name", EmojiAPIController, :create) diff --git a/test/instance_static/emoji/test_pack_for_import/blank.png b/test/instance_static/emoji/test_pack_for_import/blank.png new file mode 100644 index 0000000000000000000000000000000000000000..8f50fa02340e7e09e562f86e00b6e4bd6ad1d565 GIT binary patch literal 95 zcmeAS@N?(olHy`uVBq!ia0vp^4Is=2Bp6=1#-sr$rjj7PU + File.rm!("#{@emoji_dir_path}/test_pack_for_import/emoji.txt") + File.rm!("#{@emoji_dir_path}/test_pack_for_import/pack.json") + end) + + conn = build_conn() + resp = conn |> get(emoji_api_path(conn, :list_packs)) |> json_response(200) + + refute Map.has_key?(resp, "test_pack_for_import") + + admin = insert(:user, info: %{is_admin: true}) + + assert conn + |> assign(:user, admin) + |> post(emoji_api_path(conn, :import_from_fs)) + |> json_response(200) == ["test_pack_for_import"] + + resp = conn |> get(emoji_api_path(conn, :list_packs)) |> json_response(200) + assert resp["test_pack_for_import"]["files"] == %{"blank" => "blank.png"} + + File.rm!("#{@emoji_dir_path}/test_pack_for_import/pack.json") + refute File.exists?("#{@emoji_dir_path}/test_pack_for_import/pack.json") + + emoji_txt_content = "blank, blank.png, Fun\n\nblank2, blank.png" + + File.write!("#{@emoji_dir_path}/test_pack_for_import/emoji.txt", emoji_txt_content) + + assert conn + |> assign(:user, admin) + |> post(emoji_api_path(conn, :import_from_fs)) + |> json_response(200) == ["test_pack_for_import"] + + resp = conn |> get(emoji_api_path(conn, :list_packs)) |> json_response(200) + + assert resp["test_pack_for_import"]["files"] == %{ + "blank" => "blank.png", + "blank2" => "blank.png" + } + end end From 87057101b0e14eb51ff9367dfe9c5522ea933161 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 10 Sep 2019 21:34:57 +0300 Subject: [PATCH 24/54] Add documentation for the emoji api endpoints --- .../web/emoji_api/emoji_api_controller.ex | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index dffb91b0f..dc676b00f 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -16,6 +16,12 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIController do @cache_seconds_per_file Pleroma.Config.get!([:emoji, :shared_pack_cache_seconds_per_file]) + @doc """ + Lists the packs available on the instance as JSON. + + The information is public and does not require authentification. The format is + a map of "pack directory name" to pack.json contents. + """ def list_packs(conn, _params) do pack_infos = case File.ls(@emoji_dir_path) do @@ -116,6 +122,10 @@ keeping it in cache for #{div(cache_ms, 1000)}s") zip_result end + @doc """ + An endpoint for other instances (via admin UI) or users (via browser) + to download packs that the instance shares. + """ def download_shared(conn, %{"name" => name}) do pack_dir = Path.join(@emoji_dir_path, name) pack_file = Path.join(pack_dir, "pack.json") @@ -143,6 +153,13 @@ keeping it in cache for #{div(cache_ms, 1000)}s") end end + @doc """ + An admin endpoint to request downloading a pack named `pack_name` from the instance + `instance_address`. + + If the requested instance's admin chose to share the pack, it will be downloaded + from that instance, otherwise it will be downloaded from the fallback source, if there is one. + """ def download_from(conn, %{"instance_address" => address, "pack_name" => name} = data) do list_uri = "#{address}/api/pleroma/emoji/packs/list" @@ -211,6 +228,9 @@ keeping it in cache for #{div(cache_ms, 1000)}s") end end + @doc """ + Creates an empty pack named `name` which then can be updated via the admin UI. + """ def create(conn, %{"name" => name}) do pack_dir = Path.join(@emoji_dir_path, name) @@ -232,6 +252,9 @@ keeping it in cache for #{div(cache_ms, 1000)}s") end end + @doc """ + Deletes the pack `name` and all it's files. + """ def delete(conn, %{"name" => name}) do pack_dir = Path.join(@emoji_dir_path, name) @@ -244,6 +267,11 @@ keeping it in cache for #{div(cache_ms, 1000)}s") end end + @doc """ + An endpoint to update `pack_names`'s metadata. + + `new_data` is the new metadata for the pack, that will replace the old metadata. + """ def update_metadata(conn, %{"pack_name" => name, "new_data" => new_data}) do pack_dir = Path.join(@emoji_dir_path, name) pack_file_p = Path.join(pack_dir, "pack.json") @@ -296,6 +324,20 @@ keeping it in cache for #{div(cache_ms, 1000)}s") end end + @doc """ + Updates a file in a pack. + + Updating can mean three things: + + - `add` adds an emoji named `shortcode` to the pack `pack_name`, + that means that the emoji file needs to be uploaded with the request + (thus requiring it to be a multipart request) and be named `file`. + There can also be an optional `filename` that will be the new emoji file name + (if it's not there, the name will be taken from the uploaded file). + - `update` changes emoji shortcode (from `shortcode` to `new_shortcode` or moves the file + (from the current filename to `new_filename`) + - `remove` removes the emoji named `shortcode` and it's associated file + """ def update_file( conn, %{"pack_name" => pack_name, "action" => action, "shortcode" => shortcode} = params @@ -447,6 +489,16 @@ keeping it in cache for #{div(cache_ms, 1000)}s") end end + @doc """ + Imports emoji from the filesystem. + + Importing means checking all the directories in the + `$instance_static/emoji/` for directories which do not have + `pack.json`. If one has an emoji.txt file, that file will be used + to create a `pack.json` file with it's contents. If the directory has + neither, all the files with specific configured extenstions will be + assumed to be emojis and stored in the new `pack.json` file. + """ def import_from_fs(conn, _params) do case File.ls(@emoji_dir_path) do {:error, _} -> From f6d4acc87181c94fa202ff5673f741ae9cb45b14 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 10 Sep 2019 22:09:20 +0300 Subject: [PATCH 25/54] Fix credo warnings --- lib/pleroma/web/emoji_api/emoji_api_controller.ex | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index dc676b00f..cbd237519 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -532,7 +532,8 @@ keeping it in cache for #{div(cache_ms, 1000)}s") |> Enum.map(&String.trim/1) |> Enum.map(fn line -> case String.split(line, ~r/,\s*/) do - # This matches both strings with and without tags and we don't care about tags here + # This matches both strings with and without tags + # and we don't care about tags here [name, file | _] -> {name, file} @@ -543,8 +544,8 @@ keeping it in cache for #{div(cache_ms, 1000)}s") |> Enum.filter(fn x -> not is_nil(x) end) |> Enum.into(%{}) else - # If there's no emoji.txt, assume all files that are of certain extensions from the config - # are emojis and import them all + # If there's no emoji.txt, assume all files + # that are of certain extensions from the config are emojis and import them all Pleroma.Emoji.make_shortcode_to_file_map( dir_path, Pleroma.Config.get!([:emoji, :pack_extensions]) From 163082de6f789044b4fcb0c69f5b4cfd89731903 Mon Sep 17 00:00:00 2001 From: vaartis Date: Wed, 11 Sep 2019 09:07:19 +0000 Subject: [PATCH 26/54] Apply suggestion to lib/pleroma/web/emoji_api/emoji_api_controller.ex --- .../web/emoji_api/emoji_api_controller.ex | 20 ++++++------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index cbd237519..499802fa5 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -104,22 +104,14 @@ keeping it in cache for #{div(cache_ms, 1000)}s") # Having a different pack.json md5 invalidates cache pack_file_md5 = :crypto.hash(:md5, File.read!(Path.join(pack_dir, "pack.json"))) - maybe_cached_pack = Cachex.get!(:emoji_packs_cache, name) + case Cachex.get!(:emoji_packs_cache, name) do + %{pack_file_md5: ^pack_file_md5, pack_data: zip_result} -> + Logger.debug("Using cache for the '#{name}' shared emoji pack") + zip_result - zip_result = - if is_nil(maybe_cached_pack) do + _ -> create_archive_and_cache(name, pack, pack_dir, pack_file_md5) - else - if maybe_cached_pack[:pack_file_md5] == pack_file_md5 do - Logger.debug("Using cache for the '#{name}' shared emoji pack") - - maybe_cached_pack[:pack_data] - else - create_archive_and_cache(name, pack, pack_dir, pack_file_md5) - end - end - - zip_result + end end @doc """ From c049c32270b8f70ae679e739730a3f63cdbd7d95 Mon Sep 17 00:00:00 2001 From: vaartis Date: Wed, 11 Sep 2019 09:12:22 +0000 Subject: [PATCH 27/54] Fixed a typo in create_archive_and_cache --- lib/pleroma/web/emoji_api/emoji_api_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index 499802fa5..51620a3eb 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -94,7 +94,7 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIController do ttl: cache_ms ) - Logger.debug("Create an archive for the '#{name}' emoji pack, \ + Logger.debug("Created an archive for the '#{name}' emoji pack, \ keeping it in cache for #{div(cache_ms, 1000)}s") zip_result From f251225caeede08869b472886337afea0cd47d51 Mon Sep 17 00:00:00 2001 From: vaartis Date: Wed, 11 Sep 2019 15:32:54 +0000 Subject: [PATCH 28/54] Apply suggestions to emoji_api_controller.ex --- .../web/emoji_api/emoji_api_controller.ex | 201 +++++++++--------- 1 file changed, 95 insertions(+), 106 deletions(-) diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index 51620a3eb..0c3da6740 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -153,31 +153,32 @@ keeping it in cache for #{div(cache_ms, 1000)}s") from that instance, otherwise it will be downloaded from the fallback source, if there is one. """ def download_from(conn, %{"instance_address" => address, "pack_name" => name} = data) do - list_uri = "#{address}/api/pleroma/emoji/packs/list" - - list = Tesla.get!(list_uri).body |> Jason.decode!() - full_pack = list[name] + full_pack = + "#{address}/api/pleroma/emoji/packs/list" + |> Tesla.get!() + |> Map.get(:body) + |> Jason.decode!() + |> Map.get(name) pfiles = full_pack["files"] - pack = full_pack["pack"] pack_info_res = - cond do - pack["share-files"] && pack["can-download"] -> + case full_pack["pack"] do + %{"share-files" => true, "can-download" => true, "download-sha256" => sha} -> {:ok, %{ - sha: pack["download-sha256"], + sha: sha, uri: "#{address}/api/pleroma/emoji/packs/download_shared/#{name}" }} - pack["fallback-src"] -> + %{"fallback-src" => src, "fallback-src-sha256" => sha} when is_binary(src) -> {:ok, %{ - sha: pack["fallback-src-sha256"], - uri: pack["fallback-src"], + sha: sha, + uri: src, fallback: true }} - true -> + _ -> {:error, "The pack was not set as shared and there is no fallback src to download from"} end @@ -194,9 +195,9 @@ keeping it in cache for #{div(cache_ms, 1000)}s") File.mkdir_p!(pack_dir) # Fallback cannot contain a pack.json file - files = - unless(pinfo[:fallback], do: ['pack.json'], else: []) ++ - (pfiles |> Enum.map(fn {_, path} -> to_charlist(path) end)) + files = Enum.map(full_pack["files"], fn {_, path} -> to_charlist(path) end) + # Fallback cannot contain a pack.json file + files = if pinfo[:fallback], do: files, else: ['pack.json'] ++ files {:ok, _} = :zip.unzip(emoji_archive, cwd: to_charlist(pack_dir), file_list: files) @@ -226,7 +227,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") def create(conn, %{"name" => name}) do pack_dir = Path.join(@emoji_dir_path, name) - unless File.exists?(pack_dir) do + if not File.exists?(pack_dir) do File.mkdir_p!(pack_dir) pack_file_p = Path.join(pack_dir, "pack.json") @@ -265,8 +266,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") `new_data` is the new metadata for the pack, that will replace the old metadata. """ def update_metadata(conn, %{"pack_name" => name, "new_data" => new_data}) do - pack_dir = Path.join(@emoji_dir_path, name) - pack_file_p = Path.join(pack_dir, "pack.json") + pack_file_p = Path.join([@emoji_dir_path, name, "pack.json"]) full_pack = Jason.decode!(File.read!(pack_file_p)) @@ -275,47 +275,42 @@ keeping it in cache for #{div(cache_ms, 1000)}s") not is_nil(new_data["fallback-src"]) and new_data["fallback-src"] != full_pack["pack"]["fallback-src"] - new_data = - if should_update_fb_sha do - pack_arch = Tesla.get!(new_data["fallback-src"]).body + with {_, true} <- {:should_update?, should_update_fb_sha}, + %{body: pack_arch} <- Tesla.get!(new_data["fallback-src"]), + {:ok, flist} <- :zip.unzip(pack_arch, [:memory]), + {_, true} <- {:has_all_files?, has_all_files?(full_pack, flist)} do + fallback_sha = :crypto.hash(:sha256, pack_arch) |> Base.encode16() - {:ok, flist} = :zip.unzip(pack_arch, [:memory]) + new_data = Map.put(new_data, "fallback-src-sha256", fallback_sha) + update_metadata_and_send(conn, full_pack, new_data, pack_file_p) + else + {:should_update?, _} -> + update_metadata_and_send(conn, full_pack, new_data, pack_file_p) - # Check if all files from the pack.json are in the archive - has_all_files = - Enum.all?(full_pack["files"], fn {_, from_manifest} -> - Enum.find(flist, fn {from_archive, _} -> - to_string(from_archive) == from_manifest - end) - end) - - unless has_all_files do - {:error, - conn - |> put_status(:bad_request) - |> text("The fallback archive does not have all files specified in pack.json")} - else - fallback_sha = :crypto.hash(:sha256, pack_arch) |> Base.encode16() - - {:ok, new_data |> Map.put("fallback-src-sha256", fallback_sha)} - end - else - {:ok, new_data} - end - - case new_data do - {:ok, new_data} -> - full_pack = Map.put(full_pack, "pack", new_data) - File.write!(pack_file_p, Jason.encode!(full_pack, pretty: true)) - - # Send new data back with fallback sha filled - conn |> json(new_data) - - {:error, e} -> - e + {:has_all_files?, _} -> + conn + |> put_status(:bad_request) + |> text("The fallback archive does not have all files specified in pack.json") end end + # Check if all files from the pack.json are in the archive + defp has_all_files?(%{"files" => files}, flist) do + Enum.all?(files, fn {_, from_manifest} -> + Enum.find(flist, fn {from_archive, _} -> + to_string(from_archive) == from_manifest + end) + end) + end + + defp update_metadata_and_send(conn, full_pack, new_data, pack_file_p) do + full_pack = Map.put(full_pack, "pack", new_data) + File.write!(pack_file_p, Jason.encode!(full_pack, pretty: true)) + + # Send new data back with fallback sha filled + json(conn, new_data) + end + @doc """ Updates a file in a pack. @@ -492,69 +487,63 @@ keeping it in cache for #{div(cache_ms, 1000)}s") assumed to be emojis and stored in the new `pack.json` file. """ def import_from_fs(conn, _params) do - case File.ls(@emoji_dir_path) do + with {:ok, results} <- File.ls(@emoji_dir_path) do + imported_pack_names = + results + |> Enum.filter(fn file -> + dir_path = Path.join(@emoji_dir_path, file) + # Find the directories that do NOT have pack.json + File.dir?(dir_path) and not File.exists?(Path.join(dir_path, "pack.json")) + end) + |> Enum.map(&write_pack_json_contents/1) + + json(conn, imported_pack_names) + else {:error, _} -> conn |> put_status(:internal_server_error) |> text("Error accessing emoji pack directory") + end + end - {:ok, results} -> - imported_pack_names = - results - |> Enum.filter(fn file -> - dir_path = Path.join(@emoji_dir_path, file) - # Find the directories that do NOT have pack.json - File.dir?(dir_path) and not File.exists?(Path.join(dir_path, "pack.json")) - end) - |> Enum.map(fn dir -> - dir_path = Path.join(@emoji_dir_path, dir) - emoji_txt_path = Path.join(dir_path, "emoji.txt") + defp write_pack_json_contents(dir) do + dir_path = Path.join(@emoji_dir_path, dir) + emoji_txt_path = Path.join(dir_path, "emoji.txt") - files_for_pack = - if File.exists?(emoji_txt_path) do - # There's an emoji.txt file, it's likely from a pack installed by the pack manager. - # Make a pack.json file from the contents of that emoji.txt fileh + files_for_pack = files_for_pack(emoji_txt_path, dir_path) + pack_json_contents = Jason.encode!(%{pack: %{}, files: files_for_pack}) - # FIXME: Copy-pasted from Pleroma.Emoji/load_from_file_stream/2 + File.write!(Path.join(dir_path, "pack.json"), pack_json_contents) - # Create a map of shortcodes to filenames from emoji.txt + dir + end - File.read!(emoji_txt_path) - |> String.split("\n") - |> Enum.map(&String.trim/1) - |> Enum.map(fn line -> - case String.split(line, ~r/,\s*/) do - # This matches both strings with and without tags - # and we don't care about tags here - [name, file | _] -> - {name, file} + defp files_for_pack(emoji_txt_path, dir_path) do + if File.exists?(emoji_txt_path) do + # There's an emoji.txt file, it's likely from a pack installed by the pack manager. + # Make a pack.json file from the contents of that emoji.txt fileh - _ -> - nil - end - end) - |> Enum.filter(fn x -> not is_nil(x) end) - |> Enum.into(%{}) - else - # If there's no emoji.txt, assume all files - # that are of certain extensions from the config are emojis and import them all - Pleroma.Emoji.make_shortcode_to_file_map( - dir_path, - Pleroma.Config.get!([:emoji, :pack_extensions]) - ) - end + # FIXME: Copy-pasted from Pleroma.Emoji/load_from_file_stream/2 - pack_json_contents = Jason.encode!(%{pack: %{}, files: files_for_pack}) - - File.write!( - Path.join(dir_path, "pack.json"), - pack_json_contents - ) - - dir - end) - - conn |> json(imported_pack_names) + # Create a map of shortcodes to filenames from emoji.txt + File.read!(emoji_txt_path) + |> String.split("\n") + |> Enum.map(&String.trim/1) + |> Enum.map(fn line -> + case String.split(line, ~r/,\s*/) do + # This matches both strings with and without tags + # and we don't care about tags here + [name, file | _] -> {name, file} + _ -> nil + end + end) + |> Enum.filter(fn x -> not is_nil(x) end) + |> Enum.into(%{}) + else + # If there's no emoji.txt, assume all files + # that are of certain extensions from the config are emojis and import them all + pack_extensions = Pleroma.Config.get!([:emoji, :pack_extensions]) + Pleroma.Emoji.make_shortcode_to_file_map(dir_path, pack_extensions) end end end From b8a214b0ab264a64ca287e40e99acd401810ef58 Mon Sep 17 00:00:00 2001 From: vaartis Date: Wed, 11 Sep 2019 15:48:51 +0000 Subject: [PATCH 29/54] Split list_packs --- .../web/emoji_api/emoji_api_controller.ex | 75 ++++++++++--------- 1 file changed, 39 insertions(+), 36 deletions(-) diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index 0c3da6740..22619f4d7 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -23,47 +23,49 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIController do a map of "pack directory name" to pack.json contents. """ def list_packs(conn, _params) do - pack_infos = - case File.ls(@emoji_dir_path) do - {:error, _} -> - %{} + with {:ok, results} <- File.ls(@emoji_dir_path) do + pack_infos = + results + |> Enum.filter(&has_pack_json?/1) + |> Enum.map(&load_pack/1) + # Check if all the files are in place and can be sent + |> Enum.map(&validate_pack/1) + # Transform into a map of pack-name => pack-data + |> Enum.into(%{}) - {:ok, results} -> - results - |> Enum.filter(fn file -> - dir_path = Path.join(@emoji_dir_path, file) - # Filter to only use the pack.json packs - File.dir?(dir_path) and File.exists?(Path.join(dir_path, "pack.json")) - end) - |> Enum.map(fn pack_name -> - pack_path = Path.join(@emoji_dir_path, pack_name) - pack_file = Path.join(pack_path, "pack.json") + json(conn, pack_infos) + end + end - {pack_name, Jason.decode!(File.read!(pack_file))} - end) - # Transform into a map of pack-name => pack-data - # Check if all the files are in place and can be sent - |> Enum.map(fn {name, pack} -> - pack_path = Path.join(@emoji_dir_path, name) + defp has_pack_json?(file) do + dir_path = Path.join(@emoji_dir_path, file) + # Filter to only use the pack.json packs + File.dir?(dir_path) and File.exists?(Path.join(dir_path, "pack.json")) + end - if can_download?(pack, pack_path) do - archive_for_sha = make_archive(name, pack, pack_path) - archive_sha = :crypto.hash(:sha256, archive_for_sha) |> Base.encode16() + defp load_pack(pack_name) do + pack_path = Path.join(@emoji_dir_path, pack_name) + pack_file = Path.join(pack_path, "pack.json") - {name, - pack - |> put_in(["pack", "can-download"], true) - |> put_in(["pack", "download-sha256"], archive_sha)} - else - {name, - pack - |> put_in(["pack", "can-download"], false)} - end - end) - |> Enum.into(%{}) - end + {pack_name, Jason.decode!(File.read!(pack_file))} + end - conn |> json(pack_infos) + defp validate_pack({name, pack}) do + pack_path = Path.join(@emoji_dir_path, name) + + if can_download?(pack, pack_path) do + archive_for_sha = make_archive(name, pack, pack_path) + archive_sha = :crypto.hash(:sha256, archive_for_sha) |> Base.encode16() + + pack = + pack + |> put_in(["pack", "can-download"], true) + |> put_in(["pack", "download-sha256"], archive_sha) + + {name, pack} + else + {name, put_in(pack, ["pack", "can-download"], false)} + end end defp can_download?(pack, pack_path) do @@ -159,6 +161,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") |> Map.get(:body) |> Jason.decode!() |> Map.get(name) + pfiles = full_pack["files"] pack_info_res = From 8790365fef9d5f76b7ac1c94933e2ee218e76285 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Wed, 11 Sep 2019 18:52:21 +0300 Subject: [PATCH 30/54] Remove unused variable --- lib/pleroma/web/emoji_api/emoji_api_controller.ex | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index 22619f4d7..8ef6ae71f 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -162,8 +162,6 @@ keeping it in cache for #{div(cache_ms, 1000)}s") |> Jason.decode!() |> Map.get(name) - pfiles = full_pack["files"] - pack_info_res = case full_pack["pack"] do %{"share-files" => true, "can-download" => true, "download-sha256" => sha} -> From 8f509e6d1ee8955fc430d1f4ed7929ba0d91177c Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Wed, 11 Sep 2019 18:59:31 +0300 Subject: [PATCH 31/54] Use with w/ pack_info_res --- .../web/emoji_api/emoji_api_controller.ex | 52 ++++++++----------- 1 file changed, 23 insertions(+), 29 deletions(-) diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index 8ef6ae71f..9e0ff0b28 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -183,42 +183,36 @@ keeping it in cache for #{div(cache_ms, 1000)}s") {:error, "The pack was not set as shared and there is no fallback src to download from"} end - case pack_info_res do - {:ok, %{sha: sha, uri: uri} = pinfo} -> - sha = Base.decode16!(sha) - emoji_archive = Tesla.get!(uri).body + with {:ok, %{sha: sha, uri: uri} = pinfo} <- pack_info_res, + %{body: emoji_archive} <- Tesla.get!(uri), + {_, true} <- {:sha, Base.decode16!(sha) == :crypto.hash(:sha256, emoji_archive)} do + local_name = data["as"] || name + pack_dir = Path.join(@emoji_dir_path, local_name) + File.mkdir_p!(pack_dir) - got_sha = :crypto.hash(:sha256, emoji_archive) + files = Enum.map(full_pack["files"], fn {_, path} -> to_charlist(path) end) + # Fallback cannot contain a pack.json file + files = if pinfo[:fallback], do: files, else: ['pack.json'] ++ files - if got_sha == sha do - local_name = data["as"] || name - pack_dir = Path.join(@emoji_dir_path, local_name) - File.mkdir_p!(pack_dir) + {:ok, _} = :zip.unzip(emoji_archive, cwd: to_charlist(pack_dir), file_list: files) - # Fallback cannot contain a pack.json file - files = Enum.map(full_pack["files"], fn {_, path} -> to_charlist(path) end) - # Fallback cannot contain a pack.json file - files = if pinfo[:fallback], do: files, else: ['pack.json'] ++ files + # Fallback can't contain a pack.json file, since that would cause the fallback-src-sha256 + # in it to depend on itself + if pinfo[:fallback] do + pack_file_path = Path.join(pack_dir, "pack.json") - {:ok, _} = :zip.unzip(emoji_archive, cwd: to_charlist(pack_dir), file_list: files) - - # Fallback can't contain a pack.json file, since that would cause the fallback-src-sha256 - # in it to depend on itself - if pinfo[:fallback] do - pack_file_path = Path.join(pack_dir, "pack.json") - - File.write!(pack_file_path, Jason.encode!(full_pack, pretty: true)) - end - - conn |> text("ok") - else - conn - |> put_status(:internal_server_error) - |> text("SHA256 for the pack doesn't match the one sent by the server") - end + File.write!(pack_file_path, Jason.encode!(full_pack, pretty: true)) + end + text(conn, "ok") + else {:error, e} -> conn |> put_status(:internal_server_error) |> text(e) + + {:sha, _} -> + conn + |> put_status(:internal_server_error) + |> text("SHA256 for the pack doesn't match the one sent by the server") end end From cb125ffaf7f744e60fc134ef6b7b847d3838922a Mon Sep 17 00:00:00 2001 From: vaartis Date: Wed, 11 Sep 2019 16:00:48 +0000 Subject: [PATCH 32/54] Apply suggestion to lib/pleroma/web/emoji_api/emoji_api_controller.ex --- .../web/emoji_api/emoji_api_controller.ex | 34 ++++++++----------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index 9e0ff0b28..28eaf5ae3 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -124,26 +124,22 @@ keeping it in cache for #{div(cache_ms, 1000)}s") pack_dir = Path.join(@emoji_dir_path, name) pack_file = Path.join(pack_dir, "pack.json") - if File.exists?(pack_file) do - pack = Jason.decode!(File.read!(pack_file)) - - if can_download?(pack, pack_dir) do - zip_result = make_archive(name, pack, pack_dir) - - conn - |> send_download({:binary, zip_result}, filename: "#{name}.zip") - else - {:error, - conn - |> put_status(:forbidden) - |> text("Pack #{name} cannot be downloaded from this instance, either pack sharing\ - was disabled for this pack or some files are missing")} - end + with {_, true} <- {:exists?, File.exists?(pack_file)}, + pack = Jason.decode!(File.read!(pack_file)), + {_, true} <- {:can_download?, can_download?(pack, pack_dir)} do + zip_result = make_archive(name, pack, pack_dir) + send_download(conn, {:binary, zip_result}, filename: "#{name}.zip") else - {:error, - conn - |> put_status(:not_found) - |> text("Pack #{name} does not exist")} + {:can_download?, _} -> + conn + |> put_status(:forbidden) + |> text("Pack #{name} cannot be downloaded from this instance, either pack sharing\ + was disabled for this pack or some files are missing") + + {:exists?, _} -> + conn + |> put_status(:not_found) + |> text("Pack #{name} does not exist") end end From f24731788ef9dcbeb29c9dc5db9270a5787caff6 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Wed, 11 Sep 2019 19:01:21 +0300 Subject: [PATCH 33/54] Move emoji pack list from /list to / --- lib/pleroma/web/router.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 1252048f0..17f7406fd 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -229,7 +229,7 @@ defmodule Pleroma.Web.Router do scope "/packs" do # Pack info / downloading - get("/list", EmojiAPIController, :list_packs) + get("/", EmojiAPIController, :list_packs) get("/download_shared/:name", EmojiAPIController, :download_shared) end end From 7c784128fd8016e133c59e9c5076fa2d77a9bdee Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Wed, 11 Sep 2019 19:39:47 +0300 Subject: [PATCH 34/54] Change emoji api responses to JSON --- .../web/emoji_api/emoji_api_controller.ex | 316 ++++++++++-------- test/web/emoji_api_controller_test.exs | 36 +- 2 files changed, 186 insertions(+), 166 deletions(-) diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index 28eaf5ae3..1c5b7c687 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -6,7 +6,7 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIController do def reload(conn, _params) do Pleroma.Emoji.reload() - conn |> text("ok") + conn |> json("ok") end @emoji_dir_path Path.join( @@ -133,13 +133,15 @@ keeping it in cache for #{div(cache_ms, 1000)}s") {:can_download?, _} -> conn |> put_status(:forbidden) - |> text("Pack #{name} cannot be downloaded from this instance, either pack sharing\ - was disabled for this pack or some files are missing") + |> json(%{ + error: "Pack #{name} cannot be downloaded from this instance, either pack sharing\ + was disabled for this pack or some files are missing" + }) {:exists?, _} -> conn |> put_status(:not_found) - |> text("Pack #{name} does not exist") + |> json(%{error: "Pack #{name} does not exist"}) end end @@ -200,15 +202,15 @@ keeping it in cache for #{div(cache_ms, 1000)}s") File.write!(pack_file_path, Jason.encode!(full_pack, pretty: true)) end - text(conn, "ok") + json(conn, "ok") else {:error, e} -> - conn |> put_status(:internal_server_error) |> text(e) + conn |> put_status(:internal_server_error) |> json(%{error: e}) {:sha, _} -> conn |> put_status(:internal_server_error) - |> text("SHA256 for the pack doesn't match the one sent by the server") + |> json(%{error: "SHA256 for the pack doesn't match the one sent by the server"}) end end @@ -228,11 +230,11 @@ keeping it in cache for #{div(cache_ms, 1000)}s") Jason.encode!(%{pack: %{}, files: %{}}) ) - conn |> text("ok") + conn |> json("ok") else conn |> put_status(:conflict) - |> text("A pack named \"#{name}\" already exists") + |> json(%{error: "A pack named \"#{name}\" already exists"}) end end @@ -244,10 +246,12 @@ keeping it in cache for #{div(cache_ms, 1000)}s") case File.rm_rf(pack_dir) do {:ok, _} -> - conn |> text("ok") + conn |> json("ok") {:error, _} -> - conn |> put_status(:internal_server_error) |> text("Couldn't delete the pack #{name}") + conn + |> put_status(:internal_server_error) + |> json(%{error: "Couldn't delete the pack #{name}"}) end end @@ -281,7 +285,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") {:has_all_files?, _} -> conn |> put_status(:bad_request) - |> text("The fallback archive does not have all files specified in pack.json") + |> json(%{error: "The fallback archive does not have all files specified in pack.json"}) end end @@ -302,6 +306,25 @@ keeping it in cache for #{div(cache_ms, 1000)}s") json(conn, new_data) end + defp get_filename(%{"filename" => filename}), do: filename + + defp get_filename(%{"file" => file}) do + case file do + %Plug.Upload{filename: filename} -> filename + url when is_binary(url) -> Path.basename(url) + end + end + + defp empty?(str), do: String.trim(str) == "" + + defp update_file_and_send(conn, updated_full_pack, pack_file_p) do + # Write the emoji pack file + File.write!(pack_file_p, Jason.encode!(updated_full_pack, pretty: true)) + + # Return the modified file list + json(conn, updated_full_pack["files"]) + end + @doc """ Updates a file in a pack. @@ -316,157 +339,154 @@ keeping it in cache for #{div(cache_ms, 1000)}s") (from the current filename to `new_filename`) - `remove` removes the emoji named `shortcode` and it's associated file """ + + # Add def update_file( conn, - %{"pack_name" => pack_name, "action" => action, "shortcode" => shortcode} = params + %{"pack_name" => pack_name, "action" => "add", "shortcode" => shortcode} = params ) do pack_dir = Path.join(@emoji_dir_path, pack_name) pack_file_p = Path.join(pack_dir, "pack.json") full_pack = Jason.decode!(File.read!(pack_file_p)) - res = - case action do - "add" -> - unless Map.has_key?(full_pack["files"], shortcode) do - filename = - if Map.has_key?(params, "filename") do - params["filename"] - else - case params["file"] do - %Plug.Upload{filename: filename} -> filename - url when is_binary(url) -> Path.basename(url) - end - end + with {_, false} <- {:has_shortcode, Map.has_key?(full_pack["files"], shortcode)}, + filename <- get_filename(params), + false <- empty?(shortcode), + false <- empty?(filename) do + file_path = Path.join(pack_dir, filename) - unless String.trim(shortcode) |> String.length() == 0 or - String.trim(filename) |> String.length() == 0 do - file_path = Path.join(pack_dir, filename) - - # If the name contains directories, create them - if String.contains?(file_path, "/") do - File.mkdir_p!(Path.dirname(file_path)) - end - - case params["file"] do - %Plug.Upload{path: upload_path} -> - # Copy the uploaded file from the temporary directory - File.copy!(upload_path, file_path) - - url when is_binary(url) -> - # Download and write the file - file_contents = Tesla.get!(url).body - File.write!(file_path, file_contents) - end - - updated_full_pack = put_in(full_pack, ["files", shortcode], filename) - - {:ok, updated_full_pack} - else - {:error, - conn - |> put_status(:bad_request) - |> text("shortcode or filename cannot be empty")} - end - else - {:error, - conn - |> put_status(:conflict) - |> text("An emoji with the \"#{shortcode}\" shortcode already exists")} - end - - "remove" -> - if Map.has_key?(full_pack["files"], shortcode) do - {emoji_file_path, updated_full_pack} = pop_in(full_pack, ["files", shortcode]) - - emoji_file_path = Path.join(pack_dir, emoji_file_path) - - # Delete the emoji file - File.rm!(emoji_file_path) - - # If the old directory has no more files, remove it - if String.contains?(emoji_file_path, "/") do - dir = Path.dirname(emoji_file_path) - - if Enum.empty?(File.ls!(dir)) do - File.rmdir!(dir) - end - end - - {:ok, updated_full_pack} - else - {:error, - conn |> put_status(:bad_request) |> text("Emoji \"#{shortcode}\" does not exist")} - end - - "update" -> - if Map.has_key?(full_pack["files"], shortcode) do - with %{"new_shortcode" => new_shortcode, "new_filename" => new_filename} <- params do - unless String.trim(new_shortcode) |> String.length() == 0 or - String.trim(new_filename) |> String.length() == 0 do - # First, remove the old shortcode, saving the old path - {old_emoji_file_path, updated_full_pack} = pop_in(full_pack, ["files", shortcode]) - old_emoji_file_path = Path.join(pack_dir, old_emoji_file_path) - new_emoji_file_path = Path.join(pack_dir, new_filename) - - # If the name contains directories, create them - if String.contains?(new_emoji_file_path, "/") do - File.mkdir_p!(Path.dirname(new_emoji_file_path)) - end - - # Move/Rename the old filename to a new filename - # These are probably on the same filesystem, so just rename should work - :ok = File.rename(old_emoji_file_path, new_emoji_file_path) - - # If the old directory has no more files, remove it - if String.contains?(old_emoji_file_path, "/") do - dir = Path.dirname(old_emoji_file_path) - - if Enum.empty?(File.ls!(dir)) do - File.rmdir!(dir) - end - end - - # Then, put in the new shortcode with the new path - updated_full_pack = - put_in(updated_full_pack, ["files", new_shortcode], new_filename) - - {:ok, updated_full_pack} - else - {:error, - conn - |> put_status(:bad_request) - |> text("new_shortcode or new_filename cannot be empty")} - end - else - _ -> - {:error, - conn - |> put_status(:bad_request) - |> text("new_shortcode or new_file were not specified")} - end - else - {:error, - conn |> put_status(:bad_request) |> text("Emoji \"#{shortcode}\" does not exist")} - end - - _ -> - {:error, conn |> put_status(:bad_request) |> text("Unknown action: #{action}")} + # If the name contains directories, create them + if String.contains?(file_path, "/") do + File.mkdir_p!(Path.dirname(file_path)) end - case res do - {:ok, updated_full_pack} -> - # Write the emoji pack file - File.write!(pack_file_p, Jason.encode!(updated_full_pack, pretty: true)) + case params["file"] do + %Plug.Upload{path: upload_path} -> + # Copy the uploaded file from the temporary directory + File.copy!(upload_path, file_path) - # Return the modified file list - conn |> json(updated_full_pack["files"]) + url when is_binary(url) -> + # Download and write the file + file_contents = Tesla.get!(url).body + File.write!(file_path, file_contents) + end - {:error, e} -> - e + updated_full_pack = put_in(full_pack, ["files", shortcode], filename) + update_file_and_send(conn, updated_full_pack, pack_file_p) + else + {:has_shortcode, _} -> + conn + |> put_status(:conflict) + |> json(%{error: "An emoji with the \"#{shortcode}\" shortcode already exists"}) + + true -> + conn + |> put_status(:bad_request) + |> json(%{error: "shortcode or filename cannot be empty"}) end end + # Remove + def update_file(conn, %{ + "pack_name" => pack_name, + "action" => "remove", + "shortcode" => shortcode + }) do + pack_dir = Path.join(@emoji_dir_path, pack_name) + pack_file_p = Path.join(pack_dir, "pack.json") + + full_pack = Jason.decode!(File.read!(pack_file_p)) + + if Map.has_key?(full_pack["files"], shortcode) do + {emoji_file_path, updated_full_pack} = pop_in(full_pack, ["files", shortcode]) + + emoji_file_path = Path.join(pack_dir, emoji_file_path) + + # Delete the emoji file + File.rm!(emoji_file_path) + + # If the old directory has no more files, remove it + if String.contains?(emoji_file_path, "/") do + dir = Path.dirname(emoji_file_path) + + if Enum.empty?(File.ls!(dir)) do + File.rmdir!(dir) + end + end + + update_file_and_send(conn, updated_full_pack, pack_file_p) + else + conn + |> put_status(:bad_request) + |> json(%{error: "Emoji \"#{shortcode}\" does not exist"}) + end + end + + # Update + def update_file( + conn, + %{"pack_name" => pack_name, "action" => "update", "shortcode" => shortcode} = params + ) do + pack_dir = Path.join(@emoji_dir_path, pack_name) + pack_file_p = Path.join(pack_dir, "pack.json") + + full_pack = Jason.decode!(File.read!(pack_file_p)) + + with {_, true} <- {:has_shortcode, Map.has_key?(full_pack["files"], shortcode)}, + %{"new_shortcode" => new_shortcode, "new_filename" => new_filename} <- params, + false <- empty?(new_shortcode), + false <- empty?(new_filename) do + # First, remove the old shortcode, saving the old path + {old_emoji_file_path, updated_full_pack} = pop_in(full_pack, ["files", shortcode]) + old_emoji_file_path = Path.join(pack_dir, old_emoji_file_path) + new_emoji_file_path = Path.join(pack_dir, new_filename) + + # If the name contains directories, create them + if String.contains?(new_emoji_file_path, "/") do + File.mkdir_p!(Path.dirname(new_emoji_file_path)) + end + + # Move/Rename the old filename to a new filename + # These are probably on the same filesystem, so just rename should work + :ok = File.rename(old_emoji_file_path, new_emoji_file_path) + + # If the old directory has no more files, remove it + if String.contains?(old_emoji_file_path, "/") do + dir = Path.dirname(old_emoji_file_path) + + if Enum.empty?(File.ls!(dir)) do + File.rmdir!(dir) + end + end + + # Then, put in the new shortcode with the new path + updated_full_pack = put_in(updated_full_pack, ["files", new_shortcode], new_filename) + update_file_and_send(conn, updated_full_pack, pack_file_p) + else + {:has_shortcode, _} -> + conn + |> put_status(:bad_request) + |> json(%{error: "Emoji \"#{shortcode}\" does not exist"}) + + true -> + conn + |> put_status(:bad_request) + |> json(%{error: "new_shortcode or new_filename cannot be empty"}) + + _ -> + conn + |> put_status(:bad_request) + |> json(%{error: "new_shortcode or new_file were not specified"}) + end + end + + def update_file(conn, %{"action" => action}) do + conn + |> put_status(:bad_request) + |> json(%{error: "Unknown action: #{action}"}) + end + @doc """ Imports emoji from the filesystem. @@ -493,7 +513,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") {:error, _} -> conn |> put_status(:internal_server_error) - |> text("Error accessing emoji pack directory") + |> json(%{error: "Error accessing emoji pack directory"}) end end diff --git a/test/web/emoji_api_controller_test.exs b/test/web/emoji_api_controller_test.exs index 8b2a942ce..7942a7b01 100644 --- a/test/web/emoji_api_controller_test.exs +++ b/test/web/emoji_api_controller_test.exs @@ -101,14 +101,14 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIControllerTest do } |> Jason.encode!() ) - |> text_response(200) == "ok" + |> json_response(200) == "ok" assert File.exists?("#{@emoji_dir_path}/test_pack2/pack.json") assert File.exists?("#{@emoji_dir_path}/test_pack2/blank.png") assert conn |> delete(emoji_api_path(conn, :delete, "test_pack2")) - |> response(200) == "ok" + |> json_response(200) == "ok" refute File.exists?("#{@emoji_dir_path}/test_pack2") @@ -130,14 +130,14 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIControllerTest do } |> Jason.encode!() ) - |> text_response(200) == "ok" + |> json_response(200) == "ok" assert File.exists?("#{@emoji_dir_path}/test_pack_nonshared2/pack.json") assert File.exists?("#{@emoji_dir_path}/test_pack_nonshared2/blank.png") assert conn |> delete(emoji_api_path(conn, :delete, "test_pack_nonshared2")) - |> response(200) == "ok" + |> json_response(200) == "ok" refute File.exists?("#{@emoji_dir_path}/test_pack_nonshared2") end @@ -225,15 +225,15 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIControllerTest do conn = build_conn() - assert conn - |> assign(:user, ctx[:admin]) - |> post( - emoji_api_path(conn, :update_metadata, "test_pack"), - %{ - "new_data" => new_data - } - ) - |> text_response(:bad_request) =~ "does not have all" + assert (conn + |> assign(:user, ctx[:admin]) + |> post( + emoji_api_path(conn, :update_metadata, "test_pack"), + %{ + "new_data" => new_data + } + ) + |> json_response(:bad_request))["error"] =~ "does not have all" end end @@ -267,9 +267,9 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIControllerTest do conn = conn |> assign(:user, admin) - assert conn - |> post(emoji_api_path(conn, :update_file, "test_pack"), same_name) - |> text_response(:conflict) =~ "already exists" + assert (conn + |> post(emoji_api_path(conn, :update_file, "test_pack"), same_name) + |> json_response(:conflict))["error"] =~ "already exists" assert conn |> post(emoji_api_path(conn, :update_file, "test_pack"), different_name) @@ -350,7 +350,7 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIControllerTest do "test_created" ) ) - |> text_response(200) == "ok" + |> json_response(200) == "ok" assert File.exists?("#{@emoji_dir_path}/test_created/pack.json") @@ -361,7 +361,7 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIControllerTest do assert conn |> delete(emoji_api_path(conn, :delete, "test_created")) - |> response(200) == "ok" + |> json_response(200) == "ok" refute File.exists?("#{@emoji_dir_path}/test_created/pack.json") end From 3971bf9c5f00d12a0a2048eb3676069d58a9f243 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Wed, 11 Sep 2019 21:43:16 +0300 Subject: [PATCH 35/54] Change :sha to :checksum --- lib/pleroma/web/emoji_api/emoji_api_controller.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index 1c5b7c687..0d4a17c61 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -183,7 +183,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") with {:ok, %{sha: sha, uri: uri} = pinfo} <- pack_info_res, %{body: emoji_archive} <- Tesla.get!(uri), - {_, true} <- {:sha, Base.decode16!(sha) == :crypto.hash(:sha256, emoji_archive)} do + {_, true} <- {:checksum, Base.decode16!(sha) == :crypto.hash(:sha256, emoji_archive)} do local_name = data["as"] || name pack_dir = Path.join(@emoji_dir_path, local_name) File.mkdir_p!(pack_dir) @@ -207,7 +207,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") {:error, e} -> conn |> put_status(:internal_server_error) |> json(%{error: e}) - {:sha, _} -> + {:checksum, _} -> conn |> put_status(:internal_server_error) |> json(%{error: "SHA256 for the pack doesn't match the one sent by the server"}) From 6cd651a38be898456c06d8fee7fd15f1b406848c Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Wed, 11 Sep 2019 21:50:55 +0300 Subject: [PATCH 36/54] Make the emoji controller api more RESTy --- lib/pleroma/web/router.ex | 10 +++++----- test/web/emoji_api_controller_test.exs | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 17f7406fd..bae25c60a 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -220,17 +220,17 @@ defmodule Pleroma.Web.Router do post("/import_from_fs", EmojiAPIController, :import_from_fs) - post("/update_file/:pack_name", EmojiAPIController, :update_file) - post("/update_metadata/:pack_name", EmojiAPIController, :update_metadata) - post("/create/:name", EmojiAPIController, :create) - delete("/delete/:name", EmojiAPIController, :delete) + post("/:pack_name/update_file", EmojiAPIController, :update_file) + post("/:pack_name/update_metadata", EmojiAPIController, :update_metadata) + put("/:name", EmojiAPIController, :create) + delete("/:name", EmojiAPIController, :delete) post("/download_from", EmojiAPIController, :download_from) end scope "/packs" do # Pack info / downloading get("/", EmojiAPIController, :list_packs) - get("/download_shared/:name", EmojiAPIController, :download_shared) + get("/:name/download_shared/", EmojiAPIController, :download_shared) end end diff --git a/test/web/emoji_api_controller_test.exs b/test/web/emoji_api_controller_test.exs index 7942a7b01..e92e92f74 100644 --- a/test/web/emoji_api_controller_test.exs +++ b/test/web/emoji_api_controller_test.exs @@ -343,7 +343,7 @@ defmodule Pleroma.Web.EmojiAPI.EmojiAPIControllerTest do assert conn |> put_req_header("content-type", "application/json") - |> post( + |> put( emoji_api_path( conn, :create, From dd818bdd487149b75295abd351e3dee3e7378dd7 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Wed, 11 Sep 2019 22:39:26 +0300 Subject: [PATCH 37/54] Add documentation for the emoji endpoints --- docs/api/pleroma_api.md | 66 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/docs/api/pleroma_api.md b/docs/api/pleroma_api.md index 30fac77da..a7e7fbe25 100644 --- a/docs/api/pleroma_api.md +++ b/docs/api/pleroma_api.md @@ -365,3 +365,69 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa * Params: * `recipients`: A list of ids of users that should receive posts to this conversation. This will replace the current list of recipients, so submit the full list. The owner of owner of the conversation will always be part of the set of recipients, though. * Response: JSON, statuses (200 - healthy, 503 unhealthy) + + +## `POST /api/pleroma/emoji/reload` +### Reload the instance's custom emoji +* Method `POST` +* Authentication: required +* Params: None +* Response: JSON, "ok" and 200 status + +## `PUT /api/pleroma/emoji/packs/:name` +### Creates an empty custom emoji pack +* Method `PUT` +* Authentication: required +* Params: None +* Response: JSON, "ok" and 200 status or 409 if the pack with that name already exists + +## `DELETE /api/pleroma/emoji/packs/:name` +### Delete a custom emoji pack +* Method `DELETE` +* Authentication: required +* Params: None +* Response: JSON, "ok" and 200 status or 500 if there was an error deleting the pack + +## `POST /api/pleroma/emoji/packs/:name/update_file` +### Update a file in a custom emoji pack +* Method `POST` +* Authentication: required +* Params: + * if the `action` is `add`, adds an emoji named `shortcode` to the pack `pack_name`, + that means that the emoji file needs to be uploaded with the request + (thus requiring it to be a multipart request) and be named `file`. + There can also be an optional `filename` that will be the new emoji file name + (if it's not there, the name will be taken from the uploaded file). + * if the `action` is `update`, changes emoji shortcode + (from `shortcode` to `new_shortcode` or moves the file (from the current filename to `new_filename`) + * if the `action` is `remove`, removes the emoji named `shortcode` and it's associated file +* Response: JSON, updated "files" section of the pack and 200 status, 409 if the trying to use a shortcode + that is already taken, 400 if there was an error with the shortcode, filename or file (additional info + in the "error" part of the response JSON) + +## `POST /api/pleroma/emoji/packs/:name/update_metadata` +### Updates (replaces) pack metadata +* Method `POST` +* Authentication: required +* Params: + * `new_data`: new metadata to replace the old one +* Response: JSON, updated "metadata" section of the pack and 200 status or 400 if there was a + problem with the new metadata (the error is specified in the "error" part of the response JSON) + +## `POST /api/pleroma/emoji/packs/download_from` +### Requests the instance to download the pack from another instance +* Method `POST` +* Authentication: required +* Params: + * `instance_address`: the address of the instance to download from + * `pack_name`: the pack to download from that instance +* Response: JSON, "ok" and 200 status if the pack was downloaded, or 500 if there were + errors downloading the pack + +## `GET /api/pleroma/emoji/packs/:name/download_shared` +### Requests the instance to download the pack from another instance +* Method `GET` +* Authentication: not requires +* Params: None +* Response: the archive of the pack with a 200 status code, 403 if the pack is not set as shared, + 404 if the pack does not exist From 74fb6d864760ccaa18b9a20d148c590254779454 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Wed, 11 Sep 2019 22:43:00 +0300 Subject: [PATCH 38/54] Move EmojiAPIController from EmojiAPI to PleromaAPI --- lib/pleroma/web/emoji_api/emoji_api_controller.ex | 2 +- lib/pleroma/web/router.ex | 2 +- test/web/emoji_api_controller_test.exs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index 0d4a17c61..a83f8af57 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -1,4 +1,4 @@ -defmodule Pleroma.Web.EmojiAPI.EmojiAPIController do +defmodule Pleroma.Web.PleromaAPI.EmojiAPIController do use Pleroma.Web, :controller require Logger diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index bae25c60a..715e4ba68 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -207,7 +207,7 @@ defmodule Pleroma.Web.Router do get("/moderation_log", AdminAPIController, :list_log) end - scope "/api/pleroma/emoji", Pleroma.Web.EmojiAPI do + scope "/api/pleroma/emoji", Pleroma.Web.PleromaAPI do scope [] do pipe_through([:admin_api, :oauth_write]) diff --git a/test/web/emoji_api_controller_test.exs b/test/web/emoji_api_controller_test.exs index e92e92f74..38d11cdce 100644 --- a/test/web/emoji_api_controller_test.exs +++ b/test/web/emoji_api_controller_test.exs @@ -1,4 +1,4 @@ -defmodule Pleroma.Web.EmojiAPI.EmojiAPIControllerTest do +defmodule Pleroma.Web.PleromaAPI.EmojiAPIControllerTest do use Pleroma.Web.ConnCase import Tesla.Mock From 36f2275dc9f6c58163e4e07f8ace9d75e96033c7 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Wed, 11 Sep 2019 22:58:55 +0300 Subject: [PATCH 39/54] A feature for shareable emoji packs, use it in download_from & tests --- .../web/emoji_api/emoji_api_controller.ex | 115 ++++++++++-------- .../web/nodeinfo/nodeinfo_controller.ex | 1 + test/web/emoji_api_controller_test.exs | 22 ++++ 3 files changed, 88 insertions(+), 50 deletions(-) diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/emoji_api/emoji_api_controller.ex index a83f8af57..36ca2c804 100644 --- a/lib/pleroma/web/emoji_api/emoji_api_controller.ex +++ b/lib/pleroma/web/emoji_api/emoji_api_controller.ex @@ -153,64 +153,79 @@ keeping it in cache for #{div(cache_ms, 1000)}s") from that instance, otherwise it will be downloaded from the fallback source, if there is one. """ def download_from(conn, %{"instance_address" => address, "pack_name" => name} = data) do - full_pack = - "#{address}/api/pleroma/emoji/packs/list" + shareable_packs_available = + "#{address}/nodeinfo/2.1.json" |> Tesla.get!() |> Map.get(:body) |> Jason.decode!() - |> Map.get(name) + |> Map.get("features") + |> Enum.member?("shareable_emoji_packs") - pack_info_res = - case full_pack["pack"] do - %{"share-files" => true, "can-download" => true, "download-sha256" => sha} -> - {:ok, - %{ - sha: sha, - uri: "#{address}/api/pleroma/emoji/packs/download_shared/#{name}" - }} + if shareable_packs_available do + full_pack = + "#{address}/api/pleroma/emoji/packs/list" + |> Tesla.get!() + |> Map.get(:body) + |> Jason.decode!() + |> Map.get(name) - %{"fallback-src" => src, "fallback-src-sha256" => sha} when is_binary(src) -> - {:ok, - %{ - sha: sha, - uri: src, - fallback: true - }} + pack_info_res = + case full_pack["pack"] do + %{"share-files" => true, "can-download" => true, "download-sha256" => sha} -> + {:ok, + %{ + sha: sha, + uri: "#{address}/api/pleroma/emoji/packs/download_shared/#{name}" + }} - _ -> - {:error, "The pack was not set as shared and there is no fallback src to download from"} + %{"fallback-src" => src, "fallback-src-sha256" => sha} when is_binary(src) -> + {:ok, + %{ + sha: sha, + uri: src, + fallback: true + }} + + _ -> + {:error, + "The pack was not set as shared and there is no fallback src to download from"} + end + + with {:ok, %{sha: sha, uri: uri} = pinfo} <- pack_info_res, + %{body: emoji_archive} <- Tesla.get!(uri), + {_, true} <- {:checksum, Base.decode16!(sha) == :crypto.hash(:sha256, emoji_archive)} do + local_name = data["as"] || name + pack_dir = Path.join(@emoji_dir_path, local_name) + File.mkdir_p!(pack_dir) + + files = Enum.map(full_pack["files"], fn {_, path} -> to_charlist(path) end) + # Fallback cannot contain a pack.json file + files = if pinfo[:fallback], do: files, else: ['pack.json'] ++ files + + {:ok, _} = :zip.unzip(emoji_archive, cwd: to_charlist(pack_dir), file_list: files) + + # Fallback can't contain a pack.json file, since that would cause the fallback-src-sha256 + # in it to depend on itself + if pinfo[:fallback] do + pack_file_path = Path.join(pack_dir, "pack.json") + + File.write!(pack_file_path, Jason.encode!(full_pack, pretty: true)) + end + + json(conn, "ok") + else + {:error, e} -> + conn |> put_status(:internal_server_error) |> json(%{error: e}) + + {:checksum, _} -> + conn + |> put_status(:internal_server_error) + |> json(%{error: "SHA256 for the pack doesn't match the one sent by the server"}) end - - with {:ok, %{sha: sha, uri: uri} = pinfo} <- pack_info_res, - %{body: emoji_archive} <- Tesla.get!(uri), - {_, true} <- {:checksum, Base.decode16!(sha) == :crypto.hash(:sha256, emoji_archive)} do - local_name = data["as"] || name - pack_dir = Path.join(@emoji_dir_path, local_name) - File.mkdir_p!(pack_dir) - - files = Enum.map(full_pack["files"], fn {_, path} -> to_charlist(path) end) - # Fallback cannot contain a pack.json file - files = if pinfo[:fallback], do: files, else: ['pack.json'] ++ files - - {:ok, _} = :zip.unzip(emoji_archive, cwd: to_charlist(pack_dir), file_list: files) - - # Fallback can't contain a pack.json file, since that would cause the fallback-src-sha256 - # in it to depend on itself - if pinfo[:fallback] do - pack_file_path = Path.join(pack_dir, "pack.json") - - File.write!(pack_file_path, Jason.encode!(full_pack, pretty: true)) - end - - json(conn, "ok") else - {:error, e} -> - conn |> put_status(:internal_server_error) |> json(%{error: e}) - - {:checksum, _} -> - conn - |> put_status(:internal_server_error) - |> json(%{error: "SHA256 for the pack doesn't match the one sent by the server"}) + conn + |> put_status(:internal_server_error) + |> json(%{error: "The requested instance does not support sharing emoji packs"}) end end diff --git a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex index ee14cfd6b..192984242 100644 --- a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex +++ b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex @@ -57,6 +57,7 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do "mastodon_api_streaming", "polls", "pleroma_explicit_addressing", + "shareable_emoji_packs", if Config.get([:media_proxy, :enabled]) do "media_proxy" end, diff --git a/test/web/emoji_api_controller_test.exs b/test/web/emoji_api_controller_test.exs index 38d11cdce..1af4d3720 100644 --- a/test/web/emoji_api_controller_test.exs +++ b/test/web/emoji_api_controller_test.exs @@ -54,6 +54,12 @@ defmodule Pleroma.Web.PleromaAPI.EmojiAPIControllerTest do end) mock(fn + %{method: :get, url: "https://old-instance/nodeinfo/2.1.json"} -> + json(%{features: []}) + + %{method: :get, url: "https://example.com/nodeinfo/2.1.json"} -> + json(%{features: ["shareable_emoji_packs"]}) + %{ method: :get, url: "https://example.com/api/pleroma/emoji/packs/list" @@ -87,6 +93,22 @@ defmodule Pleroma.Web.PleromaAPI.EmojiAPIControllerTest do conn = build_conn() |> assign(:user, admin) + assert (conn + |> put_req_header("content-type", "application/json") + |> post( + emoji_api_path( + conn, + :download_from + ), + %{ + instance_address: "https://old-instance", + pack_name: "test_pack", + as: "test_pack2" + } + |> Jason.encode!() + ) + |> json_response(500))["error"] =~ "does not support" + assert conn |> put_req_header("content-type", "application/json") |> post( From 7680aec17d6690ccf7383354572456c2118a8750 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Thu, 12 Sep 2019 00:00:28 +0300 Subject: [PATCH 40/54] Move emoji api to pleroma api dir --- .../web/{emoji_api => pleroma_api}/emoji_api_controller.ex | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename lib/pleroma/web/{emoji_api => pleroma_api}/emoji_api_controller.ex (100%) diff --git a/lib/pleroma/web/emoji_api/emoji_api_controller.ex b/lib/pleroma/web/pleroma_api/emoji_api_controller.ex similarity index 100% rename from lib/pleroma/web/emoji_api/emoji_api_controller.ex rename to lib/pleroma/web/pleroma_api/emoji_api_controller.ex From d51e5e447ee944e77646b15a7aabc0214e99c351 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Thu, 12 Sep 2019 20:38:57 +0300 Subject: [PATCH 41/54] Move emoji reloading to admin api --- docs/api/admin_api.md | 7 +++++++ docs/api/pleroma_api.md | 8 -------- lib/pleroma/web/admin_api/admin_api_controller.ex | 6 ++++++ lib/pleroma/web/pleroma_api/emoji_api_controller.ex | 6 ------ lib/pleroma/web/router.ex | 8 ++------ 5 files changed, 15 insertions(+), 20 deletions(-) diff --git a/docs/api/admin_api.md b/docs/api/admin_api.md index 7637fa0d4..0377ea655 100644 --- a/docs/api/admin_api.md +++ b/docs/api/admin_api.md @@ -733,3 +733,10 @@ Compile time settings (need instance reboot): } ] ``` + +## `POST /api/pleroma/admin/reload_emoji` +### Reload the instance's custom emoji +* Method `POST` +* Authentication: required +* Params: None +* Response: JSON, "ok" and 200 status diff --git a/docs/api/pleroma_api.md b/docs/api/pleroma_api.md index a7e7fbe25..05a4e6fcc 100644 --- a/docs/api/pleroma_api.md +++ b/docs/api/pleroma_api.md @@ -366,14 +366,6 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa * `recipients`: A list of ids of users that should receive posts to this conversation. This will replace the current list of recipients, so submit the full list. The owner of owner of the conversation will always be part of the set of recipients, though. * Response: JSON, statuses (200 - healthy, 503 unhealthy) - -## `POST /api/pleroma/emoji/reload` -### Reload the instance's custom emoji -* Method `POST` -* Authentication: required -* Params: None -* Response: JSON, "ok" and 200 status - ## `PUT /api/pleroma/emoji/packs/:name` ### Creates an empty custom emoji pack * Method `PUT` diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 8a8091daa..4d4e862dd 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -599,6 +599,12 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do |> render("index.json", %{configs: updated}) end + def reload_emoji(conn, _params) do + Pleroma.Emoji.reload() + + conn |> json("ok") + end + def errors(conn, {:error, :not_found}) do conn |> put_status(:not_found) diff --git a/lib/pleroma/web/pleroma_api/emoji_api_controller.ex b/lib/pleroma/web/pleroma_api/emoji_api_controller.ex index 36ca2c804..bc1639095 100644 --- a/lib/pleroma/web/pleroma_api/emoji_api_controller.ex +++ b/lib/pleroma/web/pleroma_api/emoji_api_controller.ex @@ -3,12 +3,6 @@ defmodule Pleroma.Web.PleromaAPI.EmojiAPIController do require Logger - def reload(conn, _params) do - Pleroma.Emoji.reload() - - conn |> json("ok") - end - @emoji_dir_path Path.join( Pleroma.Config.get!([:instance, :static_dir]), "emoji" diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 715e4ba68..71ef382c5 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -205,15 +205,11 @@ defmodule Pleroma.Web.Router do get("/config/migrate_from_db", AdminAPIController, :migrate_from_db) get("/moderation_log", AdminAPIController, :list_log) + + post("/reload_emoji", AdminAPIController, :reload_emoji) end scope "/api/pleroma/emoji", Pleroma.Web.PleromaAPI do - scope [] do - pipe_through([:admin_api, :oauth_write]) - - post("/reload", EmojiAPIController, :reload) - end - scope "/packs" do # Modifying packs pipe_through([:admin_api, :oauth_write]) From 8aed05ac1518a10fb30532429984e02a05180ec3 Mon Sep 17 00:00:00 2001 From: vaartis Date: Fri, 13 Sep 2019 12:32:23 +0000 Subject: [PATCH 42/54] Apply suggestion to docs/api/pleroma_api.md --- docs/api/pleroma_api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/api/pleroma_api.md b/docs/api/pleroma_api.md index 05a4e6fcc..e76bf0caf 100644 --- a/docs/api/pleroma_api.md +++ b/docs/api/pleroma_api.md @@ -417,7 +417,7 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa errors downloading the pack ## `GET /api/pleroma/emoji/packs/:name/download_shared` -### Requests the instance to download the pack from another instance +### Requests a local pack from the instance * Method `GET` * Authentication: not requires * Params: None From 43022c347f9001d9cb8de976dd521a1e5f1c1318 Mon Sep 17 00:00:00 2001 From: vaartis Date: Fri, 13 Sep 2019 12:32:40 +0000 Subject: [PATCH 43/54] Apply suggestion to docs/api/pleroma_api.md --- docs/api/pleroma_api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/api/pleroma_api.md b/docs/api/pleroma_api.md index e76bf0caf..faf6e3acd 100644 --- a/docs/api/pleroma_api.md +++ b/docs/api/pleroma_api.md @@ -419,7 +419,7 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa ## `GET /api/pleroma/emoji/packs/:name/download_shared` ### Requests a local pack from the instance * Method `GET` -* Authentication: not requires +* Authentication: not required * Params: None * Response: the archive of the pack with a 200 status code, 403 if the pack is not set as shared, 404 if the pack does not exist From 86795d5ac2604e08654b872927678d3e05a68e85 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Fri, 13 Sep 2019 21:00:28 +0300 Subject: [PATCH 44/54] Document emoji pack listing in the api docs --- docs/api/pleroma_api.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/api/pleroma_api.md b/docs/api/pleroma_api.md index faf6e3acd..a469ddfbf 100644 --- a/docs/api/pleroma_api.md +++ b/docs/api/pleroma_api.md @@ -366,6 +366,13 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa * `recipients`: A list of ids of users that should receive posts to this conversation. This will replace the current list of recipients, so submit the full list. The owner of owner of the conversation will always be part of the set of recipients, though. * Response: JSON, statuses (200 - healthy, 503 unhealthy) +## `GET /api/pleroma/emoji/packs` +### Lists the custom emoji packs on the server +* Method `GET` +* Authentication: not required +* Params: None +* Response: JSON, "ok" and 200 status and the JSON hashmap of "pack name" to "pack contents" + ## `PUT /api/pleroma/emoji/packs/:name` ### Creates an empty custom emoji pack * Method `PUT` From a1325d5fd9b540017cbffbb73db85ee9fa9f12d0 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Wed, 18 Sep 2019 18:09:57 +0300 Subject: [PATCH 45/54] Change path from nodeinfo to metadata->features --- lib/pleroma/web/pleroma_api/emoji_api_controller.ex | 2 +- test/web/emoji_api_controller_test.exs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/pleroma_api/emoji_api_controller.ex b/lib/pleroma/web/pleroma_api/emoji_api_controller.ex index bc1639095..391c317e7 100644 --- a/lib/pleroma/web/pleroma_api/emoji_api_controller.ex +++ b/lib/pleroma/web/pleroma_api/emoji_api_controller.ex @@ -152,7 +152,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") |> Tesla.get!() |> Map.get(:body) |> Jason.decode!() - |> Map.get("features") + |> get_in(["metadata", "features"]) |> Enum.member?("shareable_emoji_packs") if shareable_packs_available do diff --git a/test/web/emoji_api_controller_test.exs b/test/web/emoji_api_controller_test.exs index 1af4d3720..297dc092f 100644 --- a/test/web/emoji_api_controller_test.exs +++ b/test/web/emoji_api_controller_test.exs @@ -55,10 +55,10 @@ defmodule Pleroma.Web.PleromaAPI.EmojiAPIControllerTest do mock(fn %{method: :get, url: "https://old-instance/nodeinfo/2.1.json"} -> - json(%{features: []}) + json(%{metadata: %{features: []}}) %{method: :get, url: "https://example.com/nodeinfo/2.1.json"} -> - json(%{features: ["shareable_emoji_packs"]}) + json(%{metadata: %{features: ["shareable_emoji_packs"]}}) %{ method: :get, From b585134c9092b49e7b5c24e04d6d6315d45dd0a2 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Wed, 18 Sep 2019 19:48:25 +0300 Subject: [PATCH 46/54] Get the nodeinfo address from the well-known --- lib/pleroma/web/pleroma_api/emoji_api_controller.ex | 8 +++++++- test/web/emoji_api_controller_test.exs | 6 ++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/pleroma_api/emoji_api_controller.ex b/lib/pleroma/web/pleroma_api/emoji_api_controller.ex index 391c317e7..6beca426a 100644 --- a/lib/pleroma/web/pleroma_api/emoji_api_controller.ex +++ b/lib/pleroma/web/pleroma_api/emoji_api_controller.ex @@ -148,7 +148,13 @@ keeping it in cache for #{div(cache_ms, 1000)}s") """ def download_from(conn, %{"instance_address" => address, "pack_name" => name} = data) do shareable_packs_available = - "#{address}/nodeinfo/2.1.json" + "#{address}/.well-known/nodeinfo" + |> Tesla.get!() + |> Map.get(:body) + |> Jason.decode!() + |> List.last() + |> Map.get("href") + # Get the actual nodeinfo address and fetch it |> Tesla.get!() |> Map.get(:body) |> Jason.decode!() diff --git a/test/web/emoji_api_controller_test.exs b/test/web/emoji_api_controller_test.exs index 297dc092f..c5a553692 100644 --- a/test/web/emoji_api_controller_test.exs +++ b/test/web/emoji_api_controller_test.exs @@ -54,9 +54,15 @@ defmodule Pleroma.Web.PleromaAPI.EmojiAPIControllerTest do end) mock(fn + %{method: :get, url: "https://old-instance/.well-known/nodeinfo"} -> + json([%{href: "https://old-instance/nodeinfo/2.1.json"}]) + %{method: :get, url: "https://old-instance/nodeinfo/2.1.json"} -> json(%{metadata: %{features: []}}) + %{method: :get, url: "https://example.com/.well-known/nodeinfo"} -> + json([%{href: "https://example.com/nodeinfo/2.1.json"}]) + %{method: :get, url: "https://example.com/nodeinfo/2.1.json"} -> json(%{metadata: %{features: ["shareable_emoji_packs"]}}) From c3856bed0c4c177c3e6716d06d615a928d95f69c Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Wed, 18 Sep 2019 23:17:15 +0200 Subject: [PATCH 47/54] docs/clients.md: Update source code urls --- docs/clients.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/clients.md b/docs/clients.md index 9029361f8..6c6180f7a 100644 --- a/docs/clients.md +++ b/docs/clients.md @@ -39,7 +39,7 @@ Feel free to contact us to be added to this list! ### Nekonium - Homepage: [F-Droid Repository](https://repo.gdgd.jp.net/), [Google Play](https://play.google.com/store/apps/details?id=com.apps.nekonium), [Amazon](https://www.amazon.co.jp/dp/B076FXPRBC/) -- Source: +- Source: - Contact: [@lin@pleroma.gdgd.jp.net](https://pleroma.gdgd.jp.net/users/lin) - Platforms: Android - Features: Streaming Ready @@ -67,7 +67,7 @@ Feel free to contact us to be added to this list! ## Alternative Web Interfaces ### Brutaldon - Homepage: -- Source Code: +- Source Code: - Contact: [@gcupc@glitch.social](https://glitch.social/users/gcupc) - Features: No Streaming From 3e972c0456a6f556bd1ee9118116f347d774df61 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Thu, 19 Sep 2019 00:21:16 +0300 Subject: [PATCH 48/54] Add :shared_pack_cache_seconds_per_file to description.exs --- config/description.exs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/config/description.exs b/config/description.exs index 65ea6bf01..5dc8dc364 100644 --- a/config/description.exs +++ b/config/description.exs @@ -2256,6 +2256,14 @@ config :pleroma, :config_description, [ "Location of the JSON-manifest. This manifest contains information about the emoji-packs you can download." <> " Currently only one manifest can be added (no arrays)", suggestions: ["https://git.pleroma.social/pleroma/emoji-index/raw/master/index.json"] + }, + %{ + key: :shared_pack_cache_seconds_per_file, + type: :integer, + descpiption: + "When an emoji pack is shared, the archive is created and cached in memory" <> + " for this amount of seconds multiplied by the number of files.", + suggestions: [60] } ] }, From fe5e0b784604b1352e98e7915c3c67d59ac4f709 Mon Sep 17 00:00:00 2001 From: eugenijm Date: Thu, 19 Sep 2019 08:27:55 +0300 Subject: [PATCH 49/54] Mastodon API: Return `pleroma.direct_conversation_id` when creating direct messages (`POST /api/v1/statuses`) --- CHANGELOG.md | 1 + .../mastodon_api/controllers/mastodon_api_controller.ex | 7 ++++++- test/web/mastodon_api/mastodon_api_controller_test.exs | 4 +++- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 906aa985e..84b64e2b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Replaced [pleroma_job_queue](https://git.pleroma.social/pleroma/pleroma_job_queue) and `Pleroma.Web.Federator.RetryQueue` with [Oban](https://github.com/sorentwo/oban) (see [`docs/config.md`](docs/config.md) on migrating customized worker / retry settings) - Introduced [quantum](https://github.com/quantum-elixir/quantum-core) job scheduler - Admin API: Return `total` when querying for reports +- Mastodon API: Return `pleroma.direct_conversation_id` when creating a direct message (`POST /api/v1/statuses`) ## [1.1.0] - 2019-??-?? ### Security diff --git a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex index 37eeb2ac3..6704ee7e8 100644 --- a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex @@ -611,7 +611,12 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do {:ok, activity} -> conn |> put_view(StatusView) - |> try_render("status.json", %{activity: activity, for: user, as: :activity}) + |> try_render("status.json", %{ + activity: activity, + for: user, + as: :activity, + with_direct_conversation_id: true + }) end end end diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index fb04748bb..35a0d3fe1 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -296,7 +296,9 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do conn |> post("api/v1/statuses", %{"status" => content, "visibility" => "direct"}) - assert %{"id" => id, "visibility" => "direct"} = json_response(conn, 200) + assert %{"id" => id} = response = json_response(conn, 200) + assert response["visibility"] == "direct" + assert response["pleroma"]["direct_conversation_id"] assert activity = Activity.get_by_id(id) assert activity.recipients == [user2.ap_id, conn.assigns[:user].ap_id] assert activity.data["to"] == [user2.ap_id] From 0e6085da106cb966c340fac2d307d9e8e26e91ed Mon Sep 17 00:00:00 2001 From: D Anzorge Date: Thu, 19 Sep 2019 16:09:07 +0200 Subject: [PATCH 50/54] Fix pagination in AP outbox.json --- lib/pleroma/web/activity_pub/views/user_view.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index 164b973d0..a2f73e140 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -227,11 +227,12 @@ defmodule Pleroma.Web.ActivityPub.UserView do activities = ActivityPub.fetch_user_activities(user, nil, params) + # this is sorted chronologically, so first activity is the newest (max) {max_id, min_id, collection} = if length(activities) > 0 do { - Enum.at(Enum.reverse(activities), 0).id, Enum.at(activities, 0).id, + Enum.at(Enum.reverse(activities), 0).id, Enum.map(activities, fn act -> {:ok, data} = Transmogrifier.prepare_outgoing(act.data) data From 9aca2cc95d0d8886d35be17e5cdd683004b425d9 Mon Sep 17 00:00:00 2001 From: D Anzorge Date: Thu, 19 Sep 2019 16:09:24 +0200 Subject: [PATCH 51/54] Add test for correct AP outbox pagination --- .../web/activity_pub/views/user_view_test.exs | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/test/web/activity_pub/views/user_view_test.exs b/test/web/activity_pub/views/user_view_test.exs index 2b4a04afd..eda95e3ea 100644 --- a/test/web/activity_pub/views/user_view_test.exs +++ b/test/web/activity_pub/views/user_view_test.exs @@ -142,4 +142,27 @@ defmodule Pleroma.Web.ActivityPub.UserViewTest do assert %{"totalItems" => 1} = UserView.render("following.json", %{user: user}) end end + + test "outbox paginates correctly" do + user = insert(:user) + + posts = + for i <- 0..25 do + {:ok, activity} = CommonAPI.post(user, %{"status" => "post #{i}"}) + activity + end + + # outbox sorts chronologically, newest first, with ten per page + posts = Enum.reverse(posts) + + %{"first" => %{"next" => next_url}} = + UserView.render("outbox.json", %{user: user, max_id: nil}) + + next_id = Enum.at(posts, 9).id + assert next_url =~ next_id + + %{"next" => next_url} = UserView.render("outbox.json", %{user: user, max_id: next_id}) + next_id = Enum.at(posts, 19).id + assert next_url =~ next_id + end end From fe4db3b94e71bafb913044de543472764671cd1a Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 19 Sep 2019 21:01:05 +0200 Subject: [PATCH 52/54] API Docs: Document conversation ids. --- docs/api/differences_in_mastoapi_responses.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/api/differences_in_mastoapi_responses.md b/docs/api/differences_in_mastoapi_responses.md index 3c7f5dad7..d007a69c3 100644 --- a/docs/api/differences_in_mastoapi_responses.md +++ b/docs/api/differences_in_mastoapi_responses.md @@ -21,7 +21,8 @@ Adding the parameter `with_muted=true` to the timeline queries will also return Has these additional fields under the `pleroma` object: - `local`: true if the post was made on the local instance -- `conversation_id`: the ID of the conversation the status is associated with (if any) +- `conversation_id`: the ID of the AP context the status is associated with (if any) +- `direct_conversation_id`: the ID of the Mastodon direct message conversation the status is associated with (if any) - `in_reply_to_account_acct`: the `acct` property of User entity for replied user (if any) - `content`: a map consisting of alternate representations of the `content` property with the key being it's mimetype. Currently the only alternate representation supported is `text/plain` - `spoiler_text`: a map consisting of alternate representations of the `spoiler_text` property with the key being it's mimetype. Currently the only alternate representation supported is `text/plain` From df3feb9412f1a6b9962aa5ad4a45e73aabc486d7 Mon Sep 17 00:00:00 2001 From: feld Date: Fri, 20 Sep 2019 13:21:07 +0000 Subject: [PATCH 53/54] Make it obvious how to support dual stack for MongooseIM --- installation/pleroma-mongooseim.cfg | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/installation/pleroma-mongooseim.cfg b/installation/pleroma-mongooseim.cfg index d7567321f..576f83541 100755 --- a/installation/pleroma-mongooseim.cfg +++ b/installation/pleroma-mongooseim.cfg @@ -215,7 +215,9 @@ ]} ]}, - { 5222, ejabberd_c2s, [ + %% If you want dual stack, you have to clone this entire config stanza + %% and change the bind to "::" + { {5222, "0.0.0.0"}, ejabberd_c2s, [ %% %% If TLS is compiled in and you installed a SSL @@ -246,7 +248,9 @@ %% {max_stanza_size, 65536} %% ]}, - { 5269, ejabberd_s2s_in, [ + %% If you want dual stack, you have to clone this entire config stanza + %% and change the bind to "::" + { {5269, "0.0.0.0"}, ejabberd_s2s_in, [ {shaper, s2s_shaper}, {max_stanza_size, 131072}, {protocol_options, ["no_sslv3"]} From 6b3d5ed6db6a3c73eb1f8373ebd670427aa8849d Mon Sep 17 00:00:00 2001 From: rinpatch Date: Mon, 23 Sep 2019 21:14:51 +0300 Subject: [PATCH 54/54] Emoji API Controller: Follow phoenix directory structure --- .../web/pleroma_api/{ => controllers}/emoji_api_controller.ex | 0 .../web/pleroma_api/{ => controllers}/pleroma_api_controller.ex | 0 test/web/{ => pleroma_api}/emoji_api_controller_test.exs | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename lib/pleroma/web/pleroma_api/{ => controllers}/emoji_api_controller.ex (100%) rename lib/pleroma/web/pleroma_api/{ => controllers}/pleroma_api_controller.ex (100%) rename test/web/{ => pleroma_api}/emoji_api_controller_test.exs (100%) diff --git a/lib/pleroma/web/pleroma_api/emoji_api_controller.ex b/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex similarity index 100% rename from lib/pleroma/web/pleroma_api/emoji_api_controller.ex rename to lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex diff --git a/lib/pleroma/web/pleroma_api/pleroma_api_controller.ex b/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex similarity index 100% rename from lib/pleroma/web/pleroma_api/pleroma_api_controller.ex rename to lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex diff --git a/test/web/emoji_api_controller_test.exs b/test/web/pleroma_api/emoji_api_controller_test.exs similarity index 100% rename from test/web/emoji_api_controller_test.exs rename to test/web/pleroma_api/emoji_api_controller_test.exs