From cdbe7cd37ab8209acc8979ff5a3132b71feda869 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 24 Sep 2019 09:27:34 +0300 Subject: [PATCH 1/6] When listing emoji packs, be sure to create the directory --- .../controllers/emoji_api_controller.ex | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex b/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex index 370bee9c3..be1f187ec 100644 --- a/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex @@ -17,7 +17,10 @@ defmodule Pleroma.Web.PleromaAPI.EmojiAPIController do a map of "pack directory name" to pack.json contents. """ def list_packs(conn, _params) do - with {:ok, results} <- File.ls(@emoji_dir_path) do + # Create the directory first if it does not exist. This is probably the first request made + # with the API so it should be sufficient + with {:create_dir, :ok} <- {:create_dir, File.mkdir_p(@emoji_dir_path)}, + {:ls, {:ok, results}} <- {:ls, File.ls(@emoji_dir_path)} do pack_infos = results |> Enum.filter(&has_pack_json?/1) @@ -28,6 +31,19 @@ defmodule Pleroma.Web.PleromaAPI.EmojiAPIController do |> Enum.into(%{}) json(conn, pack_infos) + else + {:create_dir, {:error, e}} -> + conn + |> put_status(:internal_server_error) + |> json(%{error: "Failed to create the emoji pack directory at #{@emoji_dir_path}: #{e}"}) + + {:ls, {:error, e}} -> + conn + |> put_status(:internal_server_error) + |> json(%{ + error: + "Failed to get the contents of the emoji pack directory at #{@emoji_dir_path}: #{e}" + }) end end From f21dbbc021ff6d1e97695e08b93acc906fc861f6 Mon Sep 17 00:00:00 2001 From: vaartis Date: Tue, 24 Sep 2019 12:11:25 +0000 Subject: [PATCH 2/6] Move emoji_dir_path & cache_seconds_per_file --- .../controllers/emoji_api_controller.ex | 51 ++++++++++--------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex b/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex index be1f187ec..e8c4f57a7 100644 --- a/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex @@ -3,12 +3,12 @@ defmodule Pleroma.Web.PleromaAPI.EmojiAPIController do require Logger - @emoji_dir_path Path.join( - Pleroma.Config.get!([:instance, :static_dir]), - "emoji" - ) - - @cache_seconds_per_file Pleroma.Config.get!([:emoji, :shared_pack_cache_seconds_per_file]) + def emoji_dir_path() do + Path.join( + Pleroma.Config.get!([:instance, :static_dir]), + "emoji" + ) + end @doc """ Lists the packs available on the instance as JSON. @@ -19,8 +19,8 @@ defmodule Pleroma.Web.PleromaAPI.EmojiAPIController do def list_packs(conn, _params) do # Create the directory first if it does not exist. This is probably the first request made # with the API so it should be sufficient - with {:create_dir, :ok} <- {:create_dir, File.mkdir_p(@emoji_dir_path)}, - {:ls, {:ok, results}} <- {:ls, File.ls(@emoji_dir_path)} do + with {:create_dir, :ok} <- {:create_dir, File.mkdir_p(emoji_dir_path())}, + {:ls, {:ok, results}} <- {:ls, File.ls(emoji_dir_path())} do pack_infos = results |> Enum.filter(&has_pack_json?/1) @@ -35,33 +35,33 @@ defmodule Pleroma.Web.PleromaAPI.EmojiAPIController do {:create_dir, {:error, e}} -> conn |> put_status(:internal_server_error) - |> json(%{error: "Failed to create the emoji pack directory at #{@emoji_dir_path}: #{e}"}) + |> json(%{error: "Failed to create the emoji pack directory at #{emoji_dir_path()}: #{e}"}) {:ls, {:error, e}} -> conn |> put_status(:internal_server_error) |> json(%{ error: - "Failed to get the contents of the emoji pack directory at #{@emoji_dir_path}: #{e}" + "Failed to get the contents of the emoji pack directory at #{emoji_dir_path()}: #{e}" }) end end defp has_pack_json?(file) do - dir_path = Path.join(@emoji_dir_path, file) + dir_path = Path.join(emoji_dir_path(), file) # Filter to only use the pack.json packs File.dir?(dir_path) and File.exists?(Path.join(dir_path, "pack.json")) end defp load_pack(pack_name) do - pack_path = Path.join(@emoji_dir_path, pack_name) + pack_path = Path.join(emoji_dir_path(), pack_name) pack_file = Path.join(pack_path, "pack.json") {pack_name, Jason.decode!(File.read!(pack_file))} end defp validate_pack({name, pack}) do - pack_path = Path.join(@emoji_dir_path, name) + pack_path = Path.join(emoji_dir_path(), name) if can_download?(pack, pack_path) do archive_for_sha = make_archive(name, pack, pack_path) @@ -95,7 +95,8 @@ defmodule Pleroma.Web.PleromaAPI.EmojiAPIController do {:ok, {_, zip_result}} = :zip.zip('#{name}.zip', files, [:memory, cwd: to_charlist(pack_dir)]) - cache_ms = :timer.seconds(@cache_seconds_per_file * Enum.count(files)) + cache_seconds_per_file = Pleroma.Config.get!([:emoji, :shared_pack_cache_seconds_per_file]) + cache_ms = :timer.seconds(cache_seconds_per_file * Enum.count(files)) Cachex.put!( :emoji_packs_cache, @@ -131,7 +132,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") to download packs that the instance shares. """ def download_shared(conn, %{"name" => name}) do - pack_dir = Path.join(@emoji_dir_path, name) + pack_dir = Path.join(emoji_dir_path(), name) pack_file = Path.join(pack_dir, "pack.json") with {_, true} <- {:exists?, File.exists?(pack_file)}, @@ -211,7 +212,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") %{body: emoji_archive} <- Tesla.get!(uri), {_, true} <- {:checksum, Base.decode16!(sha) == :crypto.hash(:sha256, emoji_archive)} do local_name = data["as"] || name - pack_dir = Path.join(@emoji_dir_path, local_name) + pack_dir = Path.join(emoji_dir_path(), local_name) File.mkdir_p!(pack_dir) files = Enum.map(full_pack["files"], fn {_, path} -> to_charlist(path) end) @@ -249,7 +250,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") Creates an empty pack named `name` which then can be updated via the admin UI. """ def create(conn, %{"name" => name}) do - pack_dir = Path.join(@emoji_dir_path, name) + pack_dir = Path.join(emoji_dir_path(), name) if not File.exists?(pack_dir) do File.mkdir_p!(pack_dir) @@ -273,7 +274,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") Deletes the pack `name` and all it's files. """ def delete(conn, %{"name" => name}) do - pack_dir = Path.join(@emoji_dir_path, name) + pack_dir = Path.join(emoji_dir_path(), name) case File.rm_rf(pack_dir) do {:ok, _} -> @@ -292,7 +293,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") `new_data` is the new metadata for the pack, that will replace the old metadata. """ def update_metadata(conn, %{"pack_name" => name, "new_data" => new_data}) do - pack_file_p = Path.join([@emoji_dir_path, name, "pack.json"]) + pack_file_p = Path.join([emoji_dir_path(), name, "pack.json"]) full_pack = Jason.decode!(File.read!(pack_file_p)) @@ -376,7 +377,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") conn, %{"pack_name" => pack_name, "action" => "add", "shortcode" => shortcode} = params ) do - pack_dir = Path.join(@emoji_dir_path, pack_name) + pack_dir = Path.join(emoji_dir_path(), pack_name) pack_file_p = Path.join(pack_dir, "pack.json") full_pack = Jason.decode!(File.read!(pack_file_p)) @@ -424,7 +425,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") "action" => "remove", "shortcode" => shortcode }) do - pack_dir = Path.join(@emoji_dir_path, pack_name) + pack_dir = Path.join(emoji_dir_path(), pack_name) pack_file_p = Path.join(pack_dir, "pack.json") full_pack = Jason.decode!(File.read!(pack_file_p)) @@ -459,7 +460,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") conn, %{"pack_name" => pack_name, "action" => "update", "shortcode" => shortcode} = params ) do - pack_dir = Path.join(@emoji_dir_path, pack_name) + pack_dir = Path.join(emoji_dir_path(), pack_name) pack_file_p = Path.join(pack_dir, "pack.json") full_pack = Jason.decode!(File.read!(pack_file_p)) @@ -529,11 +530,11 @@ keeping it in cache for #{div(cache_ms, 1000)}s") assumed to be emojis and stored in the new `pack.json` file. """ def import_from_fs(conn, _params) do - with {:ok, results} <- File.ls(@emoji_dir_path) do + with {:ok, results} <- File.ls(emoji_dir_path()) do imported_pack_names = results |> Enum.filter(fn file -> - dir_path = Path.join(@emoji_dir_path, file) + dir_path = Path.join(emoji_dir_path(), file) # Find the directories that do NOT have pack.json File.dir?(dir_path) and not File.exists?(Path.join(dir_path, "pack.json")) end) @@ -549,7 +550,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") end defp write_pack_json_contents(dir) do - dir_path = Path.join(@emoji_dir_path, dir) + dir_path = Path.join(emoji_dir_path(), dir) emoji_txt_path = Path.join(dir_path, "emoji.txt") files_for_pack = files_for_pack(emoji_txt_path, dir_path) From a6e85215e1bd88e5cda71f75d0d748e58e227cca Mon Sep 17 00:00:00 2001 From: vaartis Date: Tue, 24 Sep 2019 12:15:52 +0000 Subject: [PATCH 3/6] Credo fix (remove parens on function definition) --- lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex b/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex index e8c4f57a7..b7eede6c9 100644 --- a/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex @@ -3,7 +3,7 @@ defmodule Pleroma.Web.PleromaAPI.EmojiAPIController do require Logger - def emoji_dir_path() do + def emoji_dir_path do Path.join( Pleroma.Config.get!([:instance, :static_dir]), "emoji" From ba9d35a9049e0d46900d2dd95afd27c09f327a2c Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 24 Sep 2019 19:18:07 +0300 Subject: [PATCH 4/6] Add an API endpoint for listing remote packs --- .../controllers/emoji_api_controller.ex | 52 ++++++++++++++----- lib/pleroma/web/router.ex | 1 + .../pleroma_api/emoji_api_controller_test.exs | 22 ++++++++ 3 files changed, 61 insertions(+), 14 deletions(-) diff --git a/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex b/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex index b7eede6c9..cf5a086fe 100644 --- a/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex @@ -10,6 +10,27 @@ defmodule Pleroma.Web.PleromaAPI.EmojiAPIController do ) end + @doc """ + Lists packs from the remote instance. + + Since JS cannot ask remote instances for their packs due to CPS, it has to + be done by the server + """ + def list_from(conn, %{"instance_address" => address}) do + address = String.trim(address) + + if shareable_packs_available(address) do + list_resp = + "#{address}/api/pleroma/emoji/packs" |> Tesla.get!() |> Map.get(:body) |> Jason.decode!() + + json(conn, list_resp) + else + conn + |> put_status(:internal_server_error) + |> json(%{error: "The requested instance does not support sharing emoji packs"}) + end + end + @doc """ Lists the packs available on the instance as JSON. @@ -156,6 +177,21 @@ keeping it in cache for #{div(cache_ms, 1000)}s") end end + defp shareable_packs_available(address) do + "#{address}/.well-known/nodeinfo" + |> Tesla.get!() + |> Map.get(:body) + |> Jason.decode!() + |> List.last() + |> Map.get("href") + # Get the actual nodeinfo address and fetch it + |> Tesla.get!() + |> Map.get(:body) + |> Jason.decode!() + |> get_in(["metadata", "features"]) + |> Enum.member?("shareable_emoji_packs") + end + @doc """ An admin endpoint to request downloading a pack named `pack_name` from the instance `instance_address`. @@ -164,21 +200,9 @@ keeping it in cache for #{div(cache_ms, 1000)}s") from that instance, otherwise it will be downloaded from the fallback source, if there is one. """ def download_from(conn, %{"instance_address" => address, "pack_name" => name} = data) do - shareable_packs_available = - "#{address}/.well-known/nodeinfo" - |> Tesla.get!() - |> Map.get(:body) - |> Jason.decode!() - |> List.last() - |> Map.get("href") - # Get the actual nodeinfo address and fetch it - |> Tesla.get!() - |> Map.get(:body) - |> Jason.decode!() - |> get_in(["metadata", "features"]) - |> Enum.member?("shareable_emoji_packs") + address = String.trim(address) - if shareable_packs_available do + if shareable_packs_available(address) do full_pack = "#{address}/api/pleroma/emoji/packs/list" |> Tesla.get!() diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index e583093d2..8bc051936 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -222,6 +222,7 @@ defmodule Pleroma.Web.Router do put("/:name", EmojiAPIController, :create) delete("/:name", EmojiAPIController, :delete) post("/download_from", EmojiAPIController, :download_from) + post("/list_from", EmojiAPIController, :list_from) end scope "/packs" do diff --git a/test/web/pleroma_api/emoji_api_controller_test.exs b/test/web/pleroma_api/emoji_api_controller_test.exs index c5a553692..166a0201d 100644 --- a/test/web/pleroma_api/emoji_api_controller_test.exs +++ b/test/web/pleroma_api/emoji_api_controller_test.exs @@ -33,6 +33,28 @@ defmodule Pleroma.Web.PleromaAPI.EmojiAPIControllerTest do refute pack["pack"]["can-download"] end + test "listing remote packs" do + admin = insert(:user, info: %{is_admin: true}) + conn = build_conn() |> assign(:user, admin) + + resp = conn |> get(emoji_api_path(conn, :list_packs)) |> json_response(200) + + mock(fn + %{method: :get, url: "https://example.com/.well-known/nodeinfo"} -> + json([%{href: "https://example.com/nodeinfo/2.1.json"}]) + + %{method: :get, url: "https://example.com/nodeinfo/2.1.json"} -> + json(%{metadata: %{features: ["shareable_emoji_packs"]}}) + + %{method: :get, url: "https://example.com/api/pleroma/emoji/packs"} -> + json(resp) + end) + + assert conn + |> post(emoji_api_path(conn, :list_from), %{instance_address: "https://example.com"}) + |> json_response(200) == resp + end + test "downloading a shared pack from download_shared" do conn = build_conn() From 118d6dcdf4b2c81b4cbe51fd43977722b3eee164 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 24 Sep 2019 19:38:05 +0300 Subject: [PATCH 5/6] Fix nodeinfo handling --- .../web/pleroma_api/controllers/emoji_api_controller.ex | 1 + test/web/pleroma_api/emoji_api_controller_test.exs | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex b/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex index cf5a086fe..545ad80c9 100644 --- a/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/emoji_api_controller.ex @@ -182,6 +182,7 @@ keeping it in cache for #{div(cache_ms, 1000)}s") |> Tesla.get!() |> Map.get(:body) |> Jason.decode!() + |> Map.get("links") |> List.last() |> Map.get("href") # Get the actual nodeinfo address and fetch it diff --git a/test/web/pleroma_api/emoji_api_controller_test.exs b/test/web/pleroma_api/emoji_api_controller_test.exs index 166a0201d..93a507a01 100644 --- a/test/web/pleroma_api/emoji_api_controller_test.exs +++ b/test/web/pleroma_api/emoji_api_controller_test.exs @@ -41,7 +41,7 @@ defmodule Pleroma.Web.PleromaAPI.EmojiAPIControllerTest do mock(fn %{method: :get, url: "https://example.com/.well-known/nodeinfo"} -> - json([%{href: "https://example.com/nodeinfo/2.1.json"}]) + json(%{links: [%{href: "https://example.com/nodeinfo/2.1.json"}]}) %{method: :get, url: "https://example.com/nodeinfo/2.1.json"} -> json(%{metadata: %{features: ["shareable_emoji_packs"]}}) @@ -77,13 +77,13 @@ defmodule Pleroma.Web.PleromaAPI.EmojiAPIControllerTest do mock(fn %{method: :get, url: "https://old-instance/.well-known/nodeinfo"} -> - json([%{href: "https://old-instance/nodeinfo/2.1.json"}]) + json(%{links: [%{href: "https://old-instance/nodeinfo/2.1.json"}]}) %{method: :get, url: "https://old-instance/nodeinfo/2.1.json"} -> json(%{metadata: %{features: []}}) %{method: :get, url: "https://example.com/.well-known/nodeinfo"} -> - json([%{href: "https://example.com/nodeinfo/2.1.json"}]) + json(%{links: [%{href: "https://example.com/nodeinfo/2.1.json"}]}) %{method: :get, url: "https://example.com/nodeinfo/2.1.json"} -> json(%{metadata: %{features: ["shareable_emoji_packs"]}}) From 1fd9c60f8706441d38eb4c17417df80e3cf220b1 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 24 Sep 2019 22:20:48 +0300 Subject: [PATCH 6/6] Fix emoji tags for shareable packs to be "pack:{name}" --- lib/pleroma/emoji/loader.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/emoji/loader.ex b/lib/pleroma/emoji/loader.ex index a29de0a33..4f4ee51d1 100644 --- a/lib/pleroma/emoji/loader.ex +++ b/lib/pleroma/emoji/loader.ex @@ -99,7 +99,7 @@ defmodule Pleroma.Emoji.Loader do contents["files"] |> Enum.map(fn {name, rel_file} -> filename = Path.join("/emoji/#{pack_name}", rel_file) - {name, filename, pack_name} + {name, filename, ["pack:#{pack_name}"]} end) else # Load from emoji.txt / all files