From 2ecc7d92308d624dc9edb50665d752a71f55f608 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 25 Aug 2020 00:38:10 +0200 Subject: [PATCH 1/4] transmogrifier: Remove mastodon emoji-format from emoji field --- lib/pleroma/web/activity_pub/transmogrifier.ex | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 76298c4a0..0831efadc 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -318,9 +318,6 @@ def fix_emoji(%{"tag" => tags} = object) when is_list(tags) do Map.put(mapping, name, data["icon"]["url"]) end) - # we merge mastodon and pleroma emoji into a single mapping, to allow for both wire formats - emoji = Map.merge(object["emoji"] || %{}, emoji) - Map.put(object, "emoji", emoji) end From a142da3e4f03f2bfee7af30cca59b0fdc82da73f Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 25 Aug 2020 01:16:12 +0200 Subject: [PATCH 2/4] Add new Emoji Ecto.Type and fix emoji in Question --- .../activity_pub/object_validators/emoji.ex | 34 +++++++++++++ .../object_validators/audio_validator.ex | 5 +- .../chat_message_validator.ex | 2 +- .../object_validators/event_validator.ex | 5 +- .../object_validators/note_validator.ex | 11 +++- .../object_validators/question_validator.ex | 5 +- .../chat_validation_test.exs | 1 + .../transmogrifier/question_handling_test.exs | 51 +++++++++++++++++++ 8 files changed, 105 insertions(+), 9 deletions(-) create mode 100644 lib/pleroma/ecto_type/activity_pub/object_validators/emoji.ex diff --git a/lib/pleroma/ecto_type/activity_pub/object_validators/emoji.ex b/lib/pleroma/ecto_type/activity_pub/object_validators/emoji.ex new file mode 100644 index 000000000..4aacc5c88 --- /dev/null +++ b/lib/pleroma/ecto_type/activity_pub/object_validators/emoji.ex @@ -0,0 +1,34 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.Emoji do + use Ecto.Type + + def type, do: :map + + def cast(data) when is_map(data) do + has_invalid_emoji? = + Enum.find(data, fn + {name, uri} when is_binary(name) and is_binary(uri) -> + # based on ObjectValidators.Uri.cast() + case URI.parse(uri) do + %URI{host: nil} -> true + %URI{host: ""} -> true + %URI{scheme: scheme} when scheme in ["https", "http"] -> false + _ -> true + end + + {_name, _uri} -> + true + end) + + if has_invalid_emoji?, do: :error, else: {:ok, data} + end + + def cast(_data), do: :error + + def dump(data), do: {:ok, data} + + def load(data), do: {:ok, data} +end diff --git a/lib/pleroma/web/activity_pub/object_validators/audio_validator.ex b/lib/pleroma/web/activity_pub/object_validators/audio_validator.ex index d1869f188..1a97c504a 100644 --- a/lib/pleroma/web/activity_pub/object_validators/audio_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/audio_validator.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioValidator do alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations + alias Pleroma.Web.ActivityPub.Transmogrifier import Ecto.Changeset @@ -33,8 +34,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioValidator do field(:attributedTo, ObjectValidators.ObjectID) field(:summary, :string) field(:published, ObjectValidators.DateTime) - # TODO: Write type - field(:emoji, :map, default: %{}) + field(:emoji, ObjectValidators.Emoji, default: %{}) field(:sensitive, :boolean, default: false) embeds_many(:attachment, AttachmentValidator) field(:replies_count, :integer, default: 0) @@ -83,6 +83,7 @@ defp fix(data) do data |> CommonFixes.fix_defaults() |> CommonFixes.fix_attribution() + |> Transmogrifier.fix_emoji() |> fix_url() end diff --git a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex index 91b475393..6acd4a771 100644 --- a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex @@ -22,7 +22,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator do field(:content, ObjectValidators.SafeText) field(:actor, ObjectValidators.ObjectID) field(:published, ObjectValidators.DateTime) - field(:emoji, :map, default: %{}) + field(:emoji, ObjectValidators.Emoji, default: %{}) embeds_one(:attachment, AttachmentValidator) end diff --git a/lib/pleroma/web/activity_pub/object_validators/event_validator.ex b/lib/pleroma/web/activity_pub/object_validators/event_validator.ex index 07e4821a4..0b4c99dc0 100644 --- a/lib/pleroma/web/activity_pub/object_validators/event_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/event_validator.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EventValidator do alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations + alias Pleroma.Web.ActivityPub.Transmogrifier import Ecto.Changeset @@ -39,8 +40,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EventValidator do field(:attributedTo, ObjectValidators.ObjectID) field(:published, ObjectValidators.DateTime) - # TODO: Write type - field(:emoji, :map, default: %{}) + field(:emoji, ObjectValidators.Emoji, default: %{}) field(:sensitive, :boolean, default: false) embeds_many(:attachment, AttachmentValidator) field(:replies_count, :integer, default: 0) @@ -74,6 +74,7 @@ defp fix(data) do data |> CommonFixes.fix_defaults() |> CommonFixes.fix_attribution() + |> Transmogrifier.fix_emoji() end def changeset(struct, data) do diff --git a/lib/pleroma/web/activity_pub/object_validators/note_validator.ex b/lib/pleroma/web/activity_pub/object_validators/note_validator.ex index 20e735619..ab4469a59 100644 --- a/lib/pleroma/web/activity_pub/object_validators/note_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/note_validator.ex @@ -6,6 +6,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do use Ecto.Schema alias Pleroma.EctoType.ActivityPub.ObjectValidators + alias Pleroma.Web.ActivityPub.Transmogrifier import Ecto.Changeset @@ -32,8 +33,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do field(:actor, ObjectValidators.ObjectID) field(:attributedTo, ObjectValidators.ObjectID) field(:published, ObjectValidators.DateTime) - # TODO: Write type - field(:emoji, :map, default: %{}) + field(:emoji, ObjectValidators.Emoji, default: %{}) field(:sensitive, :boolean, default: false) # TODO: Write type field(:attachment, {:array, :map}, default: []) @@ -53,7 +53,14 @@ def cast_and_validate(data) do |> validate_data() end + defp fix(data) do + data + |> Transmogrifier.fix_emoji() + end + def cast_data(data) do + data = fix(data) + %__MODULE__{} |> cast(data, __schema__(:fields)) end diff --git a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex index 712047424..934d3c1ea 100644 --- a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex @@ -10,6 +10,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations alias Pleroma.Web.ActivityPub.ObjectValidators.QuestionOptionsValidator + alias Pleroma.Web.ActivityPub.Transmogrifier import Ecto.Changeset @@ -35,8 +36,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do field(:attributedTo, ObjectValidators.ObjectID) field(:summary, :string) field(:published, ObjectValidators.DateTime) - # TODO: Write type - field(:emoji, :map, default: %{}) + field(:emoji, ObjectValidators.Emoji, default: %{}) field(:sensitive, :boolean, default: false) embeds_many(:attachment, AttachmentValidator) field(:replies_count, :integer, default: 0) @@ -85,6 +85,7 @@ defp fix(data) do data |> CommonFixes.fix_defaults() |> CommonFixes.fix_attribution() + |> Transmogrifier.fix_emoji() |> fix_closed() end diff --git a/test/web/activity_pub/object_validators/chat_validation_test.exs b/test/web/activity_pub/object_validators/chat_validation_test.exs index 50bf03515..16e4808e5 100644 --- a/test/web/activity_pub/object_validators/chat_validation_test.exs +++ b/test/web/activity_pub/object_validators/chat_validation_test.exs @@ -69,6 +69,7 @@ test "validates for a basic object we build", %{valid_chat_message: valid_chat_m assert {:ok, object, _meta} = ObjectValidator.validate(valid_chat_message, []) assert Map.put(valid_chat_message, "attachment", nil) == object + assert match?(%{"firefox" => _}, object["emoji"]) end test "validates for a basic object with an attachment", %{ diff --git a/test/web/activity_pub/transmogrifier/question_handling_test.exs b/test/web/activity_pub/transmogrifier/question_handling_test.exs index c82361828..74ee79543 100644 --- a/test/web/activity_pub/transmogrifier/question_handling_test.exs +++ b/test/web/activity_pub/transmogrifier/question_handling_test.exs @@ -106,6 +106,57 @@ test "Mastodon Question activity with HTML tags in plaintext" do assert Enum.sort(object.data["oneOf"]) == Enum.sort(options) end + test "Mastodon Question activity with custom emojis" do + options = [ + %{ + "type" => "Note", + "name" => ":blobcat:", + "replies" => %{"totalItems" => 0, "type" => "Collection"} + }, + %{ + "type" => "Note", + "name" => ":blobfox:", + "replies" => %{"totalItems" => 0, "type" => "Collection"} + } + ] + + tag = [ + %{ + "icon" => %{ + "type" => "Image", + "url" => "https://blob.cat/emoji/custom/blobcats/blobcat.png" + }, + "id" => "https://blob.cat/emoji/custom/blobcats/blobcat.png", + "name" => ":blobcat:", + "type" => "Emoji", + "updated" => "1970-01-01T00:00:00Z" + }, + %{ + "icon" => %{"type" => "Image", "url" => "https://blob.cat/emoji/blobfox/blobfox.png"}, + "id" => "https://blob.cat/emoji/blobfox/blobfox.png", + "name" => ":blobfox:", + "type" => "Emoji", + "updated" => "1970-01-01T00:00:00Z" + } + ] + + data = + File.read!("test/fixtures/mastodon-question-activity.json") + |> Poison.decode!() + |> Kernel.put_in(["object", "oneOf"], options) + |> Kernel.put_in(["object", "tag"], tag) + + {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data) + object = Object.normalize(activity, false) + + assert object.data["oneOf"] == options + + assert object.data["emoji"] == %{ + "blobcat" => "https://blob.cat/emoji/custom/blobcats/blobcat.png", + "blobfox" => "https://blob.cat/emoji/blobfox/blobfox.png" + } + end + test "returns an error if received a second time" do data = File.read!("test/fixtures/mastodon-question-activity.json") |> Poison.decode!() From b960cede9a3183098ac5eb330fbc4d1c14efc034 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 25 Aug 2020 02:18:33 +0200 Subject: [PATCH 3/4] common_fixes: Force inserting context and context_id --- .../web/activity_pub/object_validators/common_fixes.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex index 721749de0..720213d73 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex @@ -11,8 +11,8 @@ def fix_defaults(data) do Utils.create_context(data["context"] || data["conversation"]) data - |> Map.put_new("context", context) - |> Map.put_new("context_id", context_id) + |> Map.put("context", context) + |> Map.put("context_id", context_id) end def fix_attribution(data) do From d9a21e4784a83a0780b353c967520cd203f44f3f Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 25 Aug 2020 02:21:19 +0200 Subject: [PATCH 4/4] fetcher: Remove fix_object call for Question activities --- lib/pleroma/object/fetcher.ex | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 6fdbc8efd..d26c5adf5 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -36,8 +36,7 @@ defp maybe_reinject_internal_fields(_, new_data), do: new_data defp reinject_object(%Object{data: %{"type" => "Question"}} = object, new_data) do Logger.debug("Reinjecting object #{new_data["id"]}") - with new_data <- Transmogrifier.fix_object(new_data), - data <- maybe_reinject_internal_fields(object, new_data), + with data <- maybe_reinject_internal_fields(object, new_data), {:ok, data, _} <- ObjectValidator.validate(data, %{}), changeset <- Object.change(object, %{data: data}), changeset <- touch_changeset(changeset),