From b73e9ef68689a7094e80e2affa0af9b05e86effb Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 25 Aug 2020 09:19:53 +0200 Subject: [PATCH 01/28] transmogrifier: Call strip_internal_fields on pipeline ingestion --- lib/pleroma/web/activity_pub/transmogrifier.ex | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index af4384213..ec3b24206 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -550,6 +550,8 @@ def handle_incoming( _options ) when objtype in ~w{Question Answer ChatMessage Audio Event} do + data = Map.put(data, "object", strip_internal_fields(data["object"])) + with {:ok, %User{}} <- ObjectValidator.fetch_actor(data), {:ok, activity, _} <- Pipeline.common_pipeline(data, local: false) do {:ok, activity} From 846b59ccb09681bda0f54bed43f5b82883228e33 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 20 Aug 2020 02:00:04 +0200 Subject: [PATCH 02/28] Pipeline Ingestion: Video --- lib/pleroma/web/activity_pub/activity_pub.ex | 2 +- .../web/activity_pub/object_validator.ex | 16 +- ..._validator.ex => audio_video_validator.ex} | 43 ++++- .../object_validators/common_fixes.ex | 9 ++ .../create_generic_validator.ex | 8 +- lib/pleroma/web/activity_pub/side_effects.ex | 2 +- .../web/activity_pub/transmogrifier.ex | 37 +---- .../tesla_mock/framatube.org-video.json | 2 +- .../transmogrifier/video_handling_test.exs | 93 +++++++++++ test/web/activity_pub/transmogrifier_test.exs | 148 +----------------- 10 files changed, 158 insertions(+), 202 deletions(-) rename lib/pleroma/web/activity_pub/object_validators/{audio_validator.ex => audio_video_validator.ex} (75%) create mode 100644 test/web/activity_pub/transmogrifier/video_handling_test.exs diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 66a9f78a3..bceec8bd1 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -84,7 +84,7 @@ defp increase_replies_count_if_reply(%{ defp increase_replies_count_if_reply(_create_data), do: :noop - @object_types ~w[ChatMessage Question Answer Audio Event] + @object_types ~w[ChatMessage Question Answer Audio Video Event] @spec persist(map(), keyword()) :: {:ok, Activity.t() | Object.t()} def persist(%{"type" => type} = object, meta) when type in @object_types do with {:ok, object} <- Object.create(object) do diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index b77c06395..081f96389 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -12,11 +12,12 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do alias Pleroma.Activity alias Pleroma.EctoType.ActivityPub.ObjectValidators alias Pleroma.Object + alias Pleroma.Object.Containment alias Pleroma.User alias Pleroma.Web.ActivityPub.ObjectValidators.AcceptRejectValidator alias Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator alias Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator - alias Pleroma.Web.ActivityPub.ObjectValidators.AudioValidator + alias Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator alias Pleroma.Web.ActivityPub.ObjectValidators.BlockValidator alias Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator alias Pleroma.Web.ActivityPub.ObjectValidators.CreateChatMessageValidator @@ -149,10 +150,10 @@ def validate(%{"type" => "Question"} = object, meta) do end end - def validate(%{"type" => "Audio"} = object, meta) do + def validate(%{"type" => type} = object, meta) when type in ~w[Audio Video] do with {:ok, object} <- object - |> AudioValidator.cast_and_validate() + |> AudioVideoValidator.cast_and_validate() |> Ecto.Changeset.apply_action(:insert) do object = stringify_keys(object) {:ok, object, meta} @@ -198,7 +199,7 @@ def validate( %{"type" => "Create", "object" => %{"type" => objtype} = object} = create_activity, meta ) - when objtype in ~w[Question Answer Audio Event] do + when objtype in ~w[Question Answer Audio Video Event] do with {:ok, object_data} <- cast_and_apply(object), meta = Keyword.put(meta, :object_data, object_data |> stringify_keys), {:ok, create_activity} <- @@ -232,8 +233,8 @@ def cast_and_apply(%{"type" => "Answer"} = object) do AnswerValidator.cast_and_apply(object) end - def cast_and_apply(%{"type" => "Audio"} = object) do - AudioValidator.cast_and_apply(object) + def cast_and_apply(%{"type" => type} = object) when type in ~w[Audio Video] do + AudioVideoValidator.cast_and_apply(object) end def cast_and_apply(%{"type" => "Event"} = object) do @@ -262,7 +263,8 @@ def stringify_keys(object) when is_list(object) do def stringify_keys(object), do: object def fetch_actor(object) do - with {:ok, actor} <- ObjectValidators.ObjectID.cast(object["actor"]) do + with actor <- Containment.get_actor(object), + {:ok, actor} <- ObjectValidators.ObjectID.cast(actor) do User.get_or_fetch_by_ap_id(actor) end end diff --git a/lib/pleroma/web/activity_pub/object_validators/audio_validator.ex b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex similarity index 75% rename from lib/pleroma/web/activity_pub/object_validators/audio_validator.ex rename to lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex index 1a97c504a..a6119e627 100644 --- a/lib/pleroma/web/activity_pub/object_validators/audio_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex @@ -2,9 +2,10 @@ # Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioValidator do +defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator do use Ecto.Schema + alias Pleroma.EarmarkRenderer alias Pleroma.EctoType.ActivityPub.ObjectValidators alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes @@ -25,14 +26,19 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioValidator do # TODO: Write type field(:tag, {:array, :map}, default: []) field(:type, :string) + + field(:name, :string) + field(:summary, :string) field(:content, :string) + field(:context, :string) + # short identifier for PleromaFE to group statuses by context + field(:context_id, :integer) # TODO: Remove actor on objects field(:actor, ObjectValidators.ObjectID) field(:attributedTo, ObjectValidators.ObjectID) - field(:summary, :string) field(:published, ObjectValidators.DateTime) field(:emoji, ObjectValidators.Emoji, default: %{}) field(:sensitive, :boolean, default: false) @@ -40,10 +46,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioValidator do field(:replies_count, :integer, default: 0) field(:like_count, :integer, default: 0) field(:announcement_count, :integer, default: 0) - field(:inReplyTo, :string) + field(:inReplyTo, ObjectValidators.ObjectID) field(:url, ObjectValidators.Uri) - # short identifier for PleromaFE to group statuses by context - field(:context_id, :integer) field(:likes, {:array, :string}, default: []) field(:announcements, {:array, :string}, default: []) @@ -68,9 +72,18 @@ def cast_data(data) do defp fix_url(%{"url" => url} = data) when is_list(url) do attachment = - Enum.find(url, fn x -> is_map(x) and String.starts_with?(x["mimeType"], "audio/") end) + Enum.find(url, fn x -> + mime_type = x["mimeType"] || x["mediaType"] || "" - link_element = Enum.find(url, fn x -> is_map(x) and x["mimeType"] == "text/html" end) + is_map(x) and String.starts_with?(mime_type, ["video/", "audio/"]) + end) + + link_element = + Enum.find(url, fn x -> + mime_type = x["mimeType"] || x["mediaType"] || "" + + is_map(x) and mime_type == "text/html" + end) data |> Map.put("attachment", [attachment]) @@ -79,12 +92,26 @@ defp fix_url(%{"url" => url} = data) when is_list(url) do defp fix_url(data), do: data + defp fix_content(%{"mediaType" => "text/markdown", "content" => content} = data) + when is_binary(content) do + content = + content + |> Earmark.as_html!(%Earmark.Options{renderer: EarmarkRenderer}) + |> Pleroma.HTML.filter_tags() + + Map.put(data, "content", content) + end + + defp fix_content(data), do: data + defp fix(data) do data |> CommonFixes.fix_defaults() |> CommonFixes.fix_attribution() + |> CommonFixes.fix_actor() |> Transmogrifier.fix_emoji() |> fix_url() + |> fix_content() end def changeset(struct, data) do @@ -97,7 +124,7 @@ def changeset(struct, data) do def validate_data(data_cng) do data_cng - |> validate_inclusion(:type, ["Audio"]) + |> validate_inclusion(:type, ["Audio", "Video"]) |> validate_required([:id, :actor, :attributedTo, :type, :context, :attachment]) |> CommonValidations.validate_any_presence([:cc, :to]) |> CommonValidations.validate_fields_match([:actor, :attributedTo]) diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex index 720213d73..b3638cfc7 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex @@ -3,6 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do + alias Pleroma.Object.Containment alias Pleroma.Web.ActivityPub.Utils # based on Pleroma.Web.ActivityPub.Utils.lazy_put_objects_defaults @@ -19,4 +20,12 @@ def fix_attribution(data) do data |> Map.put_new("actor", data["attributedTo"]) end + + def fix_actor(data) do + actor = Containment.get_actor(data) + + data + |> Map.put("actor", actor) + |> Map.put("attributedTo", actor) + end end diff --git a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex index b3dbeea57..422ee07be 100644 --- a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex @@ -10,9 +10,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do alias Pleroma.EctoType.ActivityPub.ObjectValidators alias Pleroma.Object + alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes + alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations import Ecto.Changeset - import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations @primary_key false @@ -75,14 +76,15 @@ defp fix(data, meta) do data |> fix_context(meta) |> fix_addressing(meta) + |> CommonFixes.fix_actor() end def validate_data(cng, meta \\ []) do cng |> validate_required([:actor, :type, :object]) |> validate_inclusion(:type, ["Create"]) - |> validate_actor_presence() - |> validate_any_presence([:to, :cc]) + |> CommonValidations.validate_actor_presence() + |> CommonValidations.validate_any_presence([:to, :cc]) |> validate_actors_match(meta) |> validate_context_match(meta) |> validate_object_nonexistence() diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 46a8be767..b5c720c7a 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -336,7 +336,7 @@ def handle_object_creation(%{"type" => "Answer"} = object_map, meta) do end def handle_object_creation(%{"type" => objtype} = object, meta) - when objtype in ~w[Audio Question Event] do + when objtype in ~w[Audio Video Question Event] do with {:ok, object, meta} <- Pipeline.common_pipeline(object, meta) do {:ok, object, meta} end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index ec3b24206..e14936c10 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -7,7 +7,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do A module to handle coding from internal to wire ActivityPub and back. """ alias Pleroma.Activity - alias Pleroma.EarmarkRenderer alias Pleroma.EctoType.ActivityPub.ObjectValidators alias Pleroma.Maps alias Pleroma.Object @@ -45,7 +44,6 @@ def fix_object(object, options \\ []) do |> fix_addressing |> fix_summary |> fix_type(options) - |> fix_content end def fix_summary(%{"summary" => nil} = object) do @@ -274,24 +272,7 @@ def fix_url(%{"url" => url} = object) when is_map(url) do Map.put(object, "url", url["href"]) end - def fix_url(%{"type" => "Video", "url" => url} = object) when is_list(url) do - attachment = - Enum.find(url, fn x -> - media_type = x["mediaType"] || x["mimeType"] || "" - - is_map(x) and String.starts_with?(media_type, "video/") - end) - - link_element = - Enum.find(url, fn x -> is_map(x) and (x["mediaType"] || x["mimeType"]) == "text/html" end) - - object - |> Map.put("attachment", [attachment]) - |> Map.put("url", link_element["href"]) - end - - def fix_url(%{"type" => object_type, "url" => url} = object) - when object_type != "Video" and is_list(url) do + def fix_url(%{"url" => url} = object) when is_list(url) do first_element = Enum.at(url, 0) url_string = @@ -371,18 +352,6 @@ def fix_type(%{"inReplyTo" => reply_id, "name" => _} = object, options) def fix_type(object, _), do: object - defp fix_content(%{"mediaType" => "text/markdown", "content" => content} = object) - when is_binary(content) do - html_content = - content - |> Earmark.as_html!(%Earmark.Options{renderer: EarmarkRenderer}) - |> Pleroma.HTML.filter_tags() - - Map.merge(object, %{"content" => html_content, "mediaType" => "text/html"}) - end - - defp fix_content(object), do: object - # Reduce the object list to find the reported user. defp get_reported(objects) do Enum.reduce_while(objects, nil, fn ap_id, _ -> @@ -455,7 +424,7 @@ def handle_incoming( %{"type" => "Create", "object" => %{"type" => objtype} = object} = data, options ) - when objtype in ~w{Article Note Video Page} do + when objtype in ~w{Article Note Page} do actor = Containment.get_actor(data) with nil <- Activity.get_create_by_object_ap_id(object["id"]), @@ -549,7 +518,7 @@ def handle_incoming( %{"type" => "Create", "object" => %{"type" => objtype}} = data, _options ) - when objtype in ~w{Question Answer ChatMessage Audio Event} do + when objtype in ~w{Question Answer ChatMessage Audio Video Event} do data = Map.put(data, "object", strip_internal_fields(data["object"])) with {:ok, %User{}} <- ObjectValidator.fetch_actor(data), diff --git a/test/fixtures/tesla_mock/framatube.org-video.json b/test/fixtures/tesla_mock/framatube.org-video.json index 3d53f0c97..1fa529886 100644 --- a/test/fixtures/tesla_mock/framatube.org-video.json +++ b/test/fixtures/tesla_mock/framatube.org-video.json @@ -1 +1 @@ -{"type":"Video","id":"https://framatube.org/videos/watch/6050732a-8a7a-43d4-a6cd-809525a1d206","name":"Déframasoftisons Internet [Framasoft]","duration":"PT3622S","uuid":"6050732a-8a7a-43d4-a6cd-809525a1d206","tag":[{"type":"Hashtag","name":"déframasoftisons"},{"type":"Hashtag","name":"EPN23"},{"type":"Hashtag","name":"framaconf"},{"type":"Hashtag","name":"Framasoft"},{"type":"Hashtag","name":"pyg"}],"category":{"identifier":"15","name":"Science & Technology"},"views":122,"sensitive":false,"waitTranscoding":false,"state":1,"commentsEnabled":true,"downloadEnabled":true,"published":"2020-05-24T18:34:31.569Z","originallyPublishedAt":"2019-11-30T23:00:00.000Z","updated":"2020-07-05T09:01:01.720Z","mediaType":"text/markdown","content":"Après avoir mené avec un certain succès la campagne « Dégooglisons Internet » en 2014, l’association Framasoft annonce fin 2019 arrêter progressivement un certain nombre de ses services alternatifs aux GAFAM. Pourquoi ?\r\n\r\nTranscription par @april...","support":null,"subtitleLanguage":[],"icon":{"type":"Image","url":"https://framatube.org/static/thumbnails/6050732a-8a7a-43d4-a6cd-809525a1d206.jpg","mediaType":"image/jpeg","width":223,"height":122},"url":[{"type":"Link","mediaType":"text/html","href":"https://framatube.org/videos/watch/6050732a-8a7a-43d4-a6cd-809525a1d206"},{"type":"Link","mediaType":"video/mp4","href":"https://framatube.org/static/webseed/6050732a-8a7a-43d4-a6cd-809525a1d206-1080.mp4","height":1080,"size":1157359410,"fps":25},{"type":"Link","rel":["metadata","video/mp4"],"mediaType":"application/json","href":"https://framatube.org/api/v1/videos/6050732a-8a7a-43d4-a6cd-809525a1d206/metadata/1309939","height":1080,"fps":25},{"type":"Link","mediaType":"application/x-bittorrent","href":"https://framatube.org/static/torrents/6050732a-8a7a-43d4-a6cd-809525a1d206-1080.torrent","height":1080},{"type":"Link","mediaType":"application/x-bittorrent;x-scheme-handler/magnet","href":"magnet:?xs=https%3A%2F%2Fframatube.org%2Fstatic%2Ftorrents%2F6050732a-8a7a-43d4-a6cd-809525a1d206-1080.torrent&xt=urn:btih:381c9429900552e23a4eb506318f1fa01e4d63a8&dn=D%C3%A9framasoftisons+Internet+%5BFramasoft%5D&tr=wss%3A%2F%2Fframatube.org%3A443%2Ftracker%2Fsocket&tr=https%3A%2F%2Fframatube.org%2Ftracker%2Fannounce&ws=https%3A%2F%2Fframatube.org%2Fstatic%2Fwebseed%2F6050732a-8a7a-43d4-a6cd-809525a1d206-1080.mp4&ws=https%3A%2F%2Fpeertube.iselfhost.com%2Fstatic%2Fredundancy%2F6050732a-8a7a-43d4-a6cd-809525a1d206-1080.mp4&ws=https%3A%2F%2Ftube.privacytools.io%2Fstatic%2Fredundancy%2F6050732a-8a7a-43d4-a6cd-809525a1d206-1080.mp4&ws=https%3A%2F%2Fpeertube.live%2Fstatic%2Fredundancy%2F6050732a-8a7a-43d4-a6cd-809525a1d206-1080.mp4","height":1080},{"type":"Link","mediaType":"video/mp4","href":"https://framatube.org/static/webseed/6050732a-8a7a-43d4-a6cd-809525a1d206-480.mp4","height":480,"size":250095131,"fps":25},{"type":"Link","rel":["metadata","video/mp4"],"mediaType":"application/json","href":"https://framatube.org/api/v1/videos/6050732a-8a7a-43d4-a6cd-809525a1d206/metadata/1309941","height":480,"fps":25},{"type":"Link","mediaType":"application/x-bittorrent","href":"https://framatube.org/static/torrents/6050732a-8a7a-43d4-a6cd-809525a1d206-480.torrent","height":480},{"type":"Link","mediaType":"application/x-bittorrent;x-scheme-handler/magnet","href":"magnet:?xs=https%3A%2F%2Fframatube.org%2Fstatic%2Ftorrents%2F6050732a-8a7a-43d4-a6cd-809525a1d206-480.torrent&xt=urn:btih:a181dcbb5368ab5c31cc9ff07634becb72c344ee&dn=D%C3%A9framasoftisons+Internet+%5BFramasoft%5D&tr=wss%3A%2F%2Fframatube.org%3A443%2Ftracker%2Fsocket&tr=https%3A%2F%2Fframatube.org%2Ftracker%2Fannounce&ws=https%3A%2F%2Fframatube.org%2Fstatic%2Fwebseed%2F6050732a-8a7a-43d4-a6cd-809525a1d206-480.mp4&ws=https%3A%2F%2Fpeertube.iselfhost.com%2Fstatic%2Fredundancy%2F6050732a-8a7a-43d4-a6cd-809525a1d206-480.mp4&ws=https%3A%2F%2Ftube.privacytools.io%2Fstatic%2Fredundancy%2F6050732a-8a7a-43d4-a6cd-809525a1d206-480.mp4&ws=https%3A%2F%2Fpeertube.live%2Fstatic%2Fredundancy%2F6050732a-8a7a-43d4-a6cd-809525a1d206-480.mp4","height":480},{"type":"Link","mediaType":"video/mp4","href":"https://framatube.org/static/webseed/6050732a-8a7a-43d4-a6cd-809525a1d206-360.mp4","height":360,"size":171357733,"fps":25},{"type":"Link","rel":["metadata","video/mp4"],"mediaType":"application/json","href":"https://framatube.org/api/v1/videos/6050732a-8a7a-43d4-a6cd-809525a1d206/metadata/1309942","height":360,"fps":25},{"type":"Link","mediaType":"application/x-bittorrent","href":"https://framatube.org/static/torrents/6050732a-8a7a-43d4-a6cd-809525a1d206-360.torrent","height":360},{"type":"Link","mediaType":"application/x-bittorrent;x-scheme-handler/magnet","href":"magnet:?xs=https%3A%2F%2Fframatube.org%2Fstatic%2Ftorrents%2F6050732a-8a7a-43d4-a6cd-809525a1d206-360.torrent&xt=urn:btih:aedfa9479ea04a175eee0b0bd0bda64076308746&dn=D%C3%A9framasoftisons+Internet+%5BFramasoft%5D&tr=wss%3A%2F%2Fframatube.org%3A443%2Ftracker%2Fsocket&tr=https%3A%2F%2Fframatube.org%2Ftracker%2Fannounce&ws=https%3A%2F%2Fframatube.org%2Fstatic%2Fwebseed%2F6050732a-8a7a-43d4-a6cd-809525a1d206-360.mp4&ws=https%3A%2F%2Fpeertube.iselfhost.com%2Fstatic%2Fredundancy%2F6050732a-8a7a-43d4-a6cd-809525a1d206-360.mp4&ws=https%3A%2F%2Ftube.privacytools.io%2Fstatic%2Fredundancy%2F6050732a-8a7a-43d4-a6cd-809525a1d206-360.mp4&ws=https%3A%2F%2Fpeertube.live%2Fstatic%2Fredundancy%2F6050732a-8a7a-43d4-a6cd-809525a1d206-360.mp4","height":360},{"type":"Link","mediaType":"video/mp4","href":"https://framatube.org/static/webseed/6050732a-8a7a-43d4-a6cd-809525a1d206-720.mp4","height":720,"size":497100839,"fps":25},{"type":"Link","rel":["metadata","video/mp4"],"mediaType":"application/json","href":"https://framatube.org/api/v1/videos/6050732a-8a7a-43d4-a6cd-809525a1d206/metadata/1309943","height":720,"fps":25},{"type":"Link","mediaType":"application/x-bittorrent","href":"https://framatube.org/static/torrents/6050732a-8a7a-43d4-a6cd-809525a1d206-720.torrent","height":720},{"type":"Link","mediaType":"application/x-bittorrent;x-scheme-handler/magnet","href":"magnet:?xs=https%3A%2F%2Fframatube.org%2Fstatic%2Ftorrents%2F6050732a-8a7a-43d4-a6cd-809525a1d206-720.torrent&xt=urn:btih:71971668f82a3b24ac71bc3a982848dd8dc5a5f5&dn=D%C3%A9framasoftisons+Internet+%5BFramasoft%5D&tr=wss%3A%2F%2Fframatube.org%3A443%2Ftracker%2Fsocket&tr=https%3A%2F%2Fframatube.org%2Ftracker%2Fannounce&ws=https%3A%2F%2Fframatube.org%2Fstatic%2Fwebseed%2F6050732a-8a7a-43d4-a6cd-809525a1d206-720.mp4&ws=https%3A%2F%2Fpeertube.iselfhost.com%2Fstatic%2Fredundancy%2F6050732a-8a7a-43d4-a6cd-809525a1d206-720.mp4&ws=https%3A%2F%2Ftube.privacytools.io%2Fstatic%2Fredundancy%2F6050732a-8a7a-43d4-a6cd-809525a1d206-720.mp4&ws=https%3A%2F%2Fpeertube.live%2Fstatic%2Fredundancy%2F6050732a-8a7a-43d4-a6cd-809525a1d206-720.mp4","height":720},{"type":"Link","mediaType":"video/mp4","href":"https://framatube.org/static/webseed/6050732a-8a7a-43d4-a6cd-809525a1d206-240.mp4","height":240,"size":113038439,"fps":25},{"type":"Link","rel":["metadata","video/mp4"],"mediaType":"application/json","href":"https://framatube.org/api/v1/videos/6050732a-8a7a-43d4-a6cd-809525a1d206/metadata/1309944","height":240,"fps":25},{"type":"Link","mediaType":"application/x-bittorrent","href":"https://framatube.org/static/torrents/6050732a-8a7a-43d4-a6cd-809525a1d206-240.torrent","height":240},{"type":"Link","mediaType":"application/x-bittorrent;x-scheme-handler/magnet","href":"magnet:?xs=https%3A%2F%2Fframatube.org%2Fstatic%2Ftorrents%2F6050732a-8a7a-43d4-a6cd-809525a1d206-240.torrent&xt=urn:btih:c42aa6c95efb28d9f114ebd98537f7b00fa72246&dn=D%C3%A9framasoftisons+Internet+%5BFramasoft%5D&tr=wss%3A%2F%2Fframatube.org%3A443%2Ftracker%2Fsocket&tr=https%3A%2F%2Fframatube.org%2Ftracker%2Fannounce&ws=https%3A%2F%2Fframatube.org%2Fstatic%2Fwebseed%2F6050732a-8a7a-43d4-a6cd-809525a1d206-240.mp4&ws=https%3A%2F%2Fpeertube.iselfhost.com%2Fstatic%2Fredundancy%2F6050732a-8a7a-43d4-a6cd-809525a1d206-240.mp4&ws=https%3A%2F%2Ftube.privacytools.io%2Fstatic%2Fredundancy%2F6050732a-8a7a-43d4-a6cd-809525a1d206-240.mp4","height":240},{"type":"Link","mediaType":"application/x-mpegURL","href":"https://framatube.org/static/streaming-playlists/hls/6050732a-8a7a-43d4-a6cd-809525a1d206/master.m3u8","tag":[{"type":"Infohash","name":"f7428214539626e062f300f2ca4cf9154575144e"},{"type":"Infohash","name":"46e236dffb1ea6b9123a5396cbe88e97dd94cc6c"},{"type":"Infohash","name":"11f1045830b5d786c788f2594d19f128764e7d87"},{"type":"Infohash","name":"4327ad3e0d84de100130a27e9ab6fe40c4284f0e"},{"type":"Infohash","name":"41e2eee8e7b23a63c23a77c40a46de11492a4831"},{"type":"Link","name":"sha256","mediaType":"application/json","href":"https://framatube.org/static/streaming-playlists/hls/6050732a-8a7a-43d4-a6cd-809525a1d206/segments-sha256.json"},{"type":"Link","mediaType":"video/mp4","href":"https://framatube.org/static/streaming-playlists/hls/6050732a-8a7a-43d4-a6cd-809525a1d206/6050732a-8a7a-43d4-a6cd-809525a1d206-1080-fragmented.mp4","height":1080,"size":1156777472,"fps":25},{"type":"Link","rel":["metadata","video/mp4"],"mediaType":"application/json","href":"https://framatube.org/api/v1/videos/6050732a-8a7a-43d4-a6cd-809525a1d206/metadata/1309940","height":1080,"fps":25},{"type":"Link","mediaType":"application/x-bittorrent","href":"https://framatube.org/static/torrents/6050732a-8a7a-43d4-a6cd-809525a1d206-1080-hls.torrent","height":1080},{"type":"Link","mediaType":"application/x-bittorrent;x-scheme-handler/magnet","href":"magnet:?xs=https%3A%2F%2Fframatube.org%2Fstatic%2Ftorrents%2F6050732a-8a7a-43d4-a6cd-809525a1d206-1080-hls.torrent&xt=urn:btih:0204d780ebfab0d5d9d3476a038e812ad792deeb&dn=D%C3%A9framasoftisons+Internet+%5BFramasoft%5D&tr=wss%3A%2F%2Fframatube.org%3A443%2Ftracker%2Fsocket&tr=https%3A%2F%2Fframatube.org%2Ftracker%2Fannounce&ws=https%3A%2F%2Fframatube.org%2Fstatic%2Fstreaming-playlists%2Fhls%2F6050732a-8a7a-43d4-a6cd-809525a1d206%2F6050732a-8a7a-43d4-a6cd-809525a1d206-1080-fragmented.mp4","height":1080},{"type":"Link","mediaType":"video/mp4","href":"https://framatube.org/static/streaming-playlists/hls/6050732a-8a7a-43d4-a6cd-809525a1d206/6050732a-8a7a-43d4-a6cd-809525a1d206-480-fragmented.mp4","height":480,"size":249562889,"fps":25},{"type":"Link","rel":["metadata","video/mp4"],"mediaType":"application/json","href":"https://framatube.org/api/v1/videos/6050732a-8a7a-43d4-a6cd-809525a1d206/metadata/1309945","height":480,"fps":25},{"type":"Link","mediaType":"application/x-bittorrent","href":"https://framatube.org/static/torrents/6050732a-8a7a-43d4-a6cd-809525a1d206-480-hls.torrent","height":480},{"type":"Link","mediaType":"application/x-bittorrent;x-scheme-handler/magnet","href":"magnet:?xs=https%3A%2F%2Fframatube.org%2Fstatic%2Ftorrents%2F6050732a-8a7a-43d4-a6cd-809525a1d206-480-hls.torrent&xt=urn:btih:5d14f38ded29de629668fe1cfc61a75f4cce2628&dn=D%C3%A9framasoftisons+Internet+%5BFramasoft%5D&tr=wss%3A%2F%2Fframatube.org%3A443%2Ftracker%2Fsocket&tr=https%3A%2F%2Fframatube.org%2Ftracker%2Fannounce&ws=https%3A%2F%2Fframatube.org%2Fstatic%2Fstreaming-playlists%2Fhls%2F6050732a-8a7a-43d4-a6cd-809525a1d206%2F6050732a-8a7a-43d4-a6cd-809525a1d206-480-fragmented.mp4","height":480},{"type":"Link","mediaType":"video/mp4","href":"https://framatube.org/static/streaming-playlists/hls/6050732a-8a7a-43d4-a6cd-809525a1d206/6050732a-8a7a-43d4-a6cd-809525a1d206-360-fragmented.mp4","height":360,"size":170836415,"fps":25},{"type":"Link","rel":["metadata","video/mp4"],"mediaType":"application/json","href":"https://framatube.org/api/v1/videos/6050732a-8a7a-43d4-a6cd-809525a1d206/metadata/1309946","height":360,"fps":25},{"type":"Link","mediaType":"application/x-bittorrent","href":"https://framatube.org/static/torrents/6050732a-8a7a-43d4-a6cd-809525a1d206-360-hls.torrent","height":360},{"type":"Link","mediaType":"application/x-bittorrent;x-scheme-handler/magnet","href":"magnet:?xs=https%3A%2F%2Fframatube.org%2Fstatic%2Ftorrents%2F6050732a-8a7a-43d4-a6cd-809525a1d206-360-hls.torrent&xt=urn:btih:30125488789080ad405ebcee6c214945f31b8f30&dn=D%C3%A9framasoftisons+Internet+%5BFramasoft%5D&tr=wss%3A%2F%2Fframatube.org%3A443%2Ftracker%2Fsocket&tr=https%3A%2F%2Fframatube.org%2Ftracker%2Fannounce&ws=https%3A%2F%2Fframatube.org%2Fstatic%2Fstreaming-playlists%2Fhls%2F6050732a-8a7a-43d4-a6cd-809525a1d206%2F6050732a-8a7a-43d4-a6cd-809525a1d206-360-fragmented.mp4","height":360},{"type":"Link","mediaType":"video/mp4","href":"https://framatube.org/static/streaming-playlists/hls/6050732a-8a7a-43d4-a6cd-809525a1d206/6050732a-8a7a-43d4-a6cd-809525a1d206-720-fragmented.mp4","height":720,"size":496533741,"fps":25},{"type":"Link","rel":["metadata","video/mp4"],"mediaType":"application/json","href":"https://framatube.org/api/v1/videos/6050732a-8a7a-43d4-a6cd-809525a1d206/metadata/1309947","height":720,"fps":25},{"type":"Link","mediaType":"application/x-bittorrent","href":"https://framatube.org/static/torrents/6050732a-8a7a-43d4-a6cd-809525a1d206-720-hls.torrent","height":720},{"type":"Link","mediaType":"application/x-bittorrent;x-scheme-handler/magnet","href":"magnet:?xs=https%3A%2F%2Fframatube.org%2Fstatic%2Ftorrents%2F6050732a-8a7a-43d4-a6cd-809525a1d206-720-hls.torrent&xt=urn:btih:8ed1e8bccde709901c26e315fc8f53bfd26d1ba6&dn=D%C3%A9framasoftisons+Internet+%5BFramasoft%5D&tr=wss%3A%2F%2Fframatube.org%3A443%2Ftracker%2Fsocket&tr=https%3A%2F%2Fframatube.org%2Ftracker%2Fannounce&ws=https%3A%2F%2Fframatube.org%2Fstatic%2Fstreaming-playlists%2Fhls%2F6050732a-8a7a-43d4-a6cd-809525a1d206%2F6050732a-8a7a-43d4-a6cd-809525a1d206-720-fragmented.mp4","height":720},{"type":"Link","mediaType":"video/mp4","href":"https://framatube.org/static/streaming-playlists/hls/6050732a-8a7a-43d4-a6cd-809525a1d206/6050732a-8a7a-43d4-a6cd-809525a1d206-240-fragmented.mp4","height":240,"size":112529249,"fps":25},{"type":"Link","rel":["metadata","video/mp4"],"mediaType":"application/json","href":"https://framatube.org/api/v1/videos/6050732a-8a7a-43d4-a6cd-809525a1d206/metadata/1309948","height":240,"fps":25},{"type":"Link","mediaType":"application/x-bittorrent","href":"https://framatube.org/static/torrents/6050732a-8a7a-43d4-a6cd-809525a1d206-240-hls.torrent","height":240},{"type":"Link","mediaType":"application/x-bittorrent;x-scheme-handler/magnet","href":"magnet:?xs=https%3A%2F%2Fframatube.org%2Fstatic%2Ftorrents%2F6050732a-8a7a-43d4-a6cd-809525a1d206-240-hls.torrent&xt=urn:btih:8b452bf4e70b9078d4e74ca8b5523cc9dc70d10a&dn=D%C3%A9framasoftisons+Internet+%5BFramasoft%5D&tr=wss%3A%2F%2Fframatube.org%3A443%2Ftracker%2Fsocket&tr=https%3A%2F%2Fframatube.org%2Ftracker%2Fannounce&ws=https%3A%2F%2Fframatube.org%2Fstatic%2Fstreaming-playlists%2Fhls%2F6050732a-8a7a-43d4-a6cd-809525a1d206%2F6050732a-8a7a-43d4-a6cd-809525a1d206-240-fragmented.mp4","height":240}]}],"likes":"https://framatube.org/videos/watch/6050732a-8a7a-43d4-a6cd-809525a1d206/likes","dislikes":"https://framatube.org/videos/watch/6050732a-8a7a-43d4-a6cd-809525a1d206/dislikes","shares":"https://framatube.org/videos/watch/6050732a-8a7a-43d4-a6cd-809525a1d206/announces","comments":"https://framatube.org/videos/watch/6050732a-8a7a-43d4-a6cd-809525a1d206/comments","attributedTo":[{"type":"Person","id":"https://framatube.org/accounts/framasoft"},{"type":"Group","id":"https://framatube.org/video-channels/bf54d359-cfad-4935-9d45-9d6be93f63e8"}],"to":["https://www.w3.org/ns/activitystreams#Public"],"cc":["https://framatube.org/accounts/framasoft/followers"],"@context":["https://www.w3.org/ns/activitystreams","https://w3id.org/security/v1",{"RsaSignature2017":"https://w3id.org/security#RsaSignature2017"},{"pt":"https://joinpeertube.org/ns#","sc":"http://schema.org#","Hashtag":"as:Hashtag","uuid":"sc:identifier","category":"sc:category","licence":"sc:license","subtitleLanguage":"sc:subtitleLanguage","sensitive":"as:sensitive","language":"sc:inLanguage","Infohash":"pt:Infohash","Playlist":"pt:Playlist","PlaylistElement":"pt:PlaylistElement","originallyPublishedAt":"sc:datePublished","views":{"@type":"sc:Number","@id":"pt:views"},"state":{"@type":"sc:Number","@id":"pt:state"},"size":{"@type":"sc:Number","@id":"pt:size"},"fps":{"@type":"sc:Number","@id":"pt:fps"},"startTimestamp":{"@type":"sc:Number","@id":"pt:startTimestamp"},"stopTimestamp":{"@type":"sc:Number","@id":"pt:stopTimestamp"},"position":{"@type":"sc:Number","@id":"pt:position"},"commentsEnabled":{"@type":"sc:Boolean","@id":"pt:commentsEnabled"},"downloadEnabled":{"@type":"sc:Boolean","@id":"pt:downloadEnabled"},"waitTranscoding":{"@type":"sc:Boolean","@id":"pt:waitTranscoding"},"support":{"@type":"sc:Text","@id":"pt:support"},"likes":{"@id":"as:likes","@type":"@id"},"dislikes":{"@id":"as:dislikes","@type":"@id"},"playlists":{"@id":"pt:playlists","@type":"@id"},"shares":{"@id":"as:shares","@type":"@id"},"comments":{"@id":"as:comments","@type":"@id"}}]} \ No newline at end of file +{"type":"Create","id":"https://framatube.org/videos/watch/6050732a-8a7a-43d4-a6cd-809525a1d206/activity","actor":"https://framatube.org/accounts/framasoft","object":{"type":"Video","id":"https://framatube.org/videos/watch/6050732a-8a7a-43d4-a6cd-809525a1d206","name":"Déframasoftisons Internet [Framasoft]","duration":"PT3622S","uuid":"6050732a-8a7a-43d4-a6cd-809525a1d206","tag":[{"type":"Hashtag","name":"déframasoftisons"},{"type":"Hashtag","name":"EPN23"},{"type":"Hashtag","name":"framaconf"},{"type":"Hashtag","name":"Framasoft"},{"type":"Hashtag","name":"pyg"}],"category":{"identifier":"15","name":"Science & Technology"},"views":154,"sensitive":false,"waitTranscoding":false,"state":1,"commentsEnabled":true,"downloadEnabled":true,"published":"2020-05-24T18:34:31.569Z","originallyPublishedAt":"2019-11-30T23:00:00.000Z","updated":"2020-08-17T11:01:02.994Z","mediaType":"text/markdown","content":"Après avoir mené avec un certain succès la campagne « Dégooglisons Internet » en 2014, l’association Framasoft annonce fin 2019 arrêter progressivement un certain nombre de ses services alternatifs aux GAFAM. Pourquoi ?\r\n\r\nTranscription par @aprilorg ici : https://www.april.org/deframasoftisons-internet-pierre-yves-gosset-framasoft","support":null,"subtitleLanguage":[],"icon":[{"type":"Image","url":"https://framatube.org/static/thumbnails/6050732a-8a7a-43d4-a6cd-809525a1d206.jpg","mediaType":"image/jpeg","width":223,"height":122},{"type":"Image","url":"https://framatube.org/static/previews/6050732a-8a7a-43d4-a6cd-809525a1d206.jpg","mediaType":"image/jpeg","width":850,"height":480}],"url":[{"type":"Link","mediaType":"text/html","href":"https://framatube.org/videos/watch/6050732a-8a7a-43d4-a6cd-809525a1d206"},{"type":"Link","mediaType":"video/mp4","href":"https://framatube.org/static/webseed/6050732a-8a7a-43d4-a6cd-809525a1d206-1080.mp4","height":1080,"size":1157359410,"fps":25},{"type":"Link","rel":["metadata","video/mp4"],"mediaType":"application/json","href":"https://framatube.org/api/v1/videos/6050732a-8a7a-43d4-a6cd-809525a1d206/metadata/1309939","height":1080,"fps":25},{"type":"Link","mediaType":"application/x-bittorrent","href":"https://framatube.org/static/torrents/6050732a-8a7a-43d4-a6cd-809525a1d206-1080.torrent","height":1080},{"type":"Link","mediaType":"application/x-bittorrent;x-scheme-handler/magnet","href":"magnet:?xs=https%3A%2F%2Fframatube.org%2Fstatic%2Ftorrents%2F6050732a-8a7a-43d4-a6cd-809525a1d206-1080.torrent&xt=urn:btih:381c9429900552e23a4eb506318f1fa01e4d63a8&dn=D%C3%A9framasoftisons+Internet+%5BFramasoft%5D&tr=wss%3A%2F%2Fframatube.org%3A443%2Ftracker%2Fsocket&tr=https%3A%2F%2Fframatube.org%2Ftracker%2Fannounce&ws=https%3A%2F%2Fframatube.org%2Fstatic%2Fwebseed%2F6050732a-8a7a-43d4-a6cd-809525a1d206-1080.mp4&ws=https%3A%2F%2Fpeertube.iselfhost.com%2Fstatic%2Fredundancy%2F6050732a-8a7a-43d4-a6cd-809525a1d206-1080.mp4&ws=https%3A%2F%2Ftube.privacytools.io%2Fstatic%2Fredundancy%2F6050732a-8a7a-43d4-a6cd-809525a1d206-1080.mp4&ws=https%3A%2F%2Fpeertube.live%2Fstatic%2Fredundancy%2F6050732a-8a7a-43d4-a6cd-809525a1d206-1080.mp4","height":1080},{"type":"Link","mediaType":"video/mp4","href":"https://framatube.org/static/webseed/6050732a-8a7a-43d4-a6cd-809525a1d206-720.mp4","height":720,"size":497100839,"fps":25},{"type":"Link","rel":["metadata","video/mp4"],"mediaType":"application/json","href":"https://framatube.org/api/v1/videos/6050732a-8a7a-43d4-a6cd-809525a1d206/metadata/1309943","height":720,"fps":25},{"type":"Link","mediaType":"application/x-bittorrent","href":"https://framatube.org/static/torrents/6050732a-8a7a-43d4-a6cd-809525a1d206-720.torrent","height":720},{"type":"Link","mediaType":"application/x-bittorrent;x-scheme-handler/magnet","href":"magnet:?xs=https%3A%2F%2Fframatube.org%2Fstatic%2Ftorrents%2F6050732a-8a7a-43d4-a6cd-809525a1d206-720.torrent&xt=urn:btih:71971668f82a3b24ac71bc3a982848dd8dc5a5f5&dn=D%C3%A9framasoftisons+Internet+%5BFramasoft%5D&tr=wss%3A%2F%2Fframatube.org%3A443%2Ftracker%2Fsocket&tr=https%3A%2F%2Fframatube.org%2Ftracker%2Fannounce&ws=https%3A%2F%2Fframatube.org%2Fstatic%2Fwebseed%2F6050732a-8a7a-43d4-a6cd-809525a1d206-720.mp4&ws=https%3A%2F%2Ftube.privacytools.io%2Fstatic%2Fredundancy%2F6050732a-8a7a-43d4-a6cd-809525a1d206-720.mp4&ws=https%3A%2F%2Fpeertube.iselfhost.com%2Fstatic%2Fredundancy%2F6050732a-8a7a-43d4-a6cd-809525a1d206-720.mp4&ws=https%3A%2F%2Fpeertube.live%2Fstatic%2Fredundancy%2F6050732a-8a7a-43d4-a6cd-809525a1d206-720.mp4","height":720},{"type":"Link","mediaType":"video/mp4","href":"https://framatube.org/static/webseed/6050732a-8a7a-43d4-a6cd-809525a1d206-480.mp4","height":480,"size":250095131,"fps":25},{"type":"Link","rel":["metadata","video/mp4"],"mediaType":"application/json","href":"https://framatube.org/api/v1/videos/6050732a-8a7a-43d4-a6cd-809525a1d206/metadata/1309941","height":480,"fps":25},{"type":"Link","mediaType":"application/x-bittorrent","href":"https://framatube.org/static/torrents/6050732a-8a7a-43d4-a6cd-809525a1d206-480.torrent","height":480},{"type":"Link","mediaType":"application/x-bittorrent;x-scheme-handler/magnet","href":"magnet:?xs=https%3A%2F%2Fframatube.org%2Fstatic%2Ftorrents%2F6050732a-8a7a-43d4-a6cd-809525a1d206-480.torrent&xt=urn:btih:a181dcbb5368ab5c31cc9ff07634becb72c344ee&dn=D%C3%A9framasoftisons+Internet+%5BFramasoft%5D&tr=wss%3A%2F%2Fframatube.org%3A443%2Ftracker%2Fsocket&tr=https%3A%2F%2Fframatube.org%2Ftracker%2Fannounce&ws=https%3A%2F%2Fframatube.org%2Fstatic%2Fwebseed%2F6050732a-8a7a-43d4-a6cd-809525a1d206-480.mp4&ws=https%3A%2F%2Fpeertube.iselfhost.com%2Fstatic%2Fredundancy%2F6050732a-8a7a-43d4-a6cd-809525a1d206-480.mp4&ws=https%3A%2F%2Ftube.privacytools.io%2Fstatic%2Fredundancy%2F6050732a-8a7a-43d4-a6cd-809525a1d206-480.mp4&ws=https%3A%2F%2Fpeertube.live%2Fstatic%2Fredundancy%2F6050732a-8a7a-43d4-a6cd-809525a1d206-480.mp4","height":480},{"type":"Link","mediaType":"video/mp4","href":"https://framatube.org/static/webseed/6050732a-8a7a-43d4-a6cd-809525a1d206-360.mp4","height":360,"size":171357733,"fps":25},{"type":"Link","rel":["metadata","video/mp4"],"mediaType":"application/json","href":"https://framatube.org/api/v1/videos/6050732a-8a7a-43d4-a6cd-809525a1d206/metadata/1309942","height":360,"fps":25},{"type":"Link","mediaType":"application/x-bittorrent","href":"https://framatube.org/static/torrents/6050732a-8a7a-43d4-a6cd-809525a1d206-360.torrent","height":360},{"type":"Link","mediaType":"application/x-bittorrent;x-scheme-handler/magnet","href":"magnet:?xs=https%3A%2F%2Fframatube.org%2Fstatic%2Ftorrents%2F6050732a-8a7a-43d4-a6cd-809525a1d206-360.torrent&xt=urn:btih:aedfa9479ea04a175eee0b0bd0bda64076308746&dn=D%C3%A9framasoftisons+Internet+%5BFramasoft%5D&tr=wss%3A%2F%2Fframatube.org%3A443%2Ftracker%2Fsocket&tr=https%3A%2F%2Fframatube.org%2Ftracker%2Fannounce&ws=https%3A%2F%2Fframatube.org%2Fstatic%2Fwebseed%2F6050732a-8a7a-43d4-a6cd-809525a1d206-360.mp4&ws=https%3A%2F%2Fpeertube.iselfhost.com%2Fstatic%2Fredundancy%2F6050732a-8a7a-43d4-a6cd-809525a1d206-360.mp4&ws=https%3A%2F%2Ftube.privacytools.io%2Fstatic%2Fredundancy%2F6050732a-8a7a-43d4-a6cd-809525a1d206-360.mp4&ws=https%3A%2F%2Fpeertube.live%2Fstatic%2Fredundancy%2F6050732a-8a7a-43d4-a6cd-809525a1d206-360.mp4","height":360},{"type":"Link","mediaType":"video/mp4","href":"https://framatube.org/static/webseed/6050732a-8a7a-43d4-a6cd-809525a1d206-240.mp4","height":240,"size":113038439,"fps":25},{"type":"Link","rel":["metadata","video/mp4"],"mediaType":"application/json","href":"https://framatube.org/api/v1/videos/6050732a-8a7a-43d4-a6cd-809525a1d206/metadata/1309944","height":240,"fps":25},{"type":"Link","mediaType":"application/x-bittorrent","href":"https://framatube.org/static/torrents/6050732a-8a7a-43d4-a6cd-809525a1d206-240.torrent","height":240},{"type":"Link","mediaType":"application/x-bittorrent;x-scheme-handler/magnet","href":"magnet:?xs=https%3A%2F%2Fframatube.org%2Fstatic%2Ftorrents%2F6050732a-8a7a-43d4-a6cd-809525a1d206-240.torrent&xt=urn:btih:c42aa6c95efb28d9f114ebd98537f7b00fa72246&dn=D%C3%A9framasoftisons+Internet+%5BFramasoft%5D&tr=wss%3A%2F%2Fframatube.org%3A443%2Ftracker%2Fsocket&tr=https%3A%2F%2Fframatube.org%2Ftracker%2Fannounce&ws=https%3A%2F%2Fframatube.org%2Fstatic%2Fwebseed%2F6050732a-8a7a-43d4-a6cd-809525a1d206-240.mp4&ws=https%3A%2F%2Fpeertube.iselfhost.com%2Fstatic%2Fredundancy%2F6050732a-8a7a-43d4-a6cd-809525a1d206-240.mp4&ws=https%3A%2F%2Ftube.privacytools.io%2Fstatic%2Fredundancy%2F6050732a-8a7a-43d4-a6cd-809525a1d206-240.mp4","height":240},{"type":"Link","mediaType":"application/x-mpegURL","href":"https://framatube.org/static/streaming-playlists/hls/6050732a-8a7a-43d4-a6cd-809525a1d206/master.m3u8","tag":[{"type":"Infohash","name":"f7428214539626e062f300f2ca4cf9154575144e"},{"type":"Infohash","name":"46e236dffb1ea6b9123a5396cbe88e97dd94cc6c"},{"type":"Infohash","name":"11f1045830b5d786c788f2594d19f128764e7d87"},{"type":"Infohash","name":"4327ad3e0d84de100130a27e9ab6fe40c4284f0e"},{"type":"Infohash","name":"41e2eee8e7b23a63c23a77c40a46de11492a4831"},{"type":"Link","name":"sha256","mediaType":"application/json","href":"https://framatube.org/static/streaming-playlists/hls/6050732a-8a7a-43d4-a6cd-809525a1d206/segments-sha256.json"},{"type":"Link","mediaType":"video/mp4","href":"https://framatube.org/static/streaming-playlists/hls/6050732a-8a7a-43d4-a6cd-809525a1d206/6050732a-8a7a-43d4-a6cd-809525a1d206-1080-fragmented.mp4","height":1080,"size":1156777472,"fps":25},{"type":"Link","rel":["metadata","video/mp4"],"mediaType":"application/json","href":"https://framatube.org/api/v1/videos/6050732a-8a7a-43d4-a6cd-809525a1d206/metadata/1309940","height":1080,"fps":25},{"type":"Link","mediaType":"application/x-bittorrent","href":"https://framatube.org/static/torrents/6050732a-8a7a-43d4-a6cd-809525a1d206-1080-hls.torrent","height":1080},{"type":"Link","mediaType":"application/x-bittorrent;x-scheme-handler/magnet","href":"magnet:?xs=https%3A%2F%2Fframatube.org%2Fstatic%2Ftorrents%2F6050732a-8a7a-43d4-a6cd-809525a1d206-1080-hls.torrent&xt=urn:btih:0204d780ebfab0d5d9d3476a038e812ad792deeb&dn=D%C3%A9framasoftisons+Internet+%5BFramasoft%5D&tr=wss%3A%2F%2Fframatube.org%3A443%2Ftracker%2Fsocket&tr=https%3A%2F%2Fframatube.org%2Ftracker%2Fannounce&ws=https%3A%2F%2Fframatube.org%2Fstatic%2Fstreaming-playlists%2Fhls%2F6050732a-8a7a-43d4-a6cd-809525a1d206%2F6050732a-8a7a-43d4-a6cd-809525a1d206-1080-fragmented.mp4","height":1080},{"type":"Link","mediaType":"video/mp4","href":"https://framatube.org/static/streaming-playlists/hls/6050732a-8a7a-43d4-a6cd-809525a1d206/6050732a-8a7a-43d4-a6cd-809525a1d206-720-fragmented.mp4","height":720,"size":496533741,"fps":25},{"type":"Link","rel":["metadata","video/mp4"],"mediaType":"application/json","href":"https://framatube.org/api/v1/videos/6050732a-8a7a-43d4-a6cd-809525a1d206/metadata/1309947","height":720,"fps":25},{"type":"Link","mediaType":"application/x-bittorrent","href":"https://framatube.org/static/torrents/6050732a-8a7a-43d4-a6cd-809525a1d206-720-hls.torrent","height":720},{"type":"Link","mediaType":"application/x-bittorrent;x-scheme-handler/magnet","href":"magnet:?xs=https%3A%2F%2Fframatube.org%2Fstatic%2Ftorrents%2F6050732a-8a7a-43d4-a6cd-809525a1d206-720-hls.torrent&xt=urn:btih:8ed1e8bccde709901c26e315fc8f53bfd26d1ba6&dn=D%C3%A9framasoftisons+Internet+%5BFramasoft%5D&tr=wss%3A%2F%2Fframatube.org%3A443%2Ftracker%2Fsocket&tr=https%3A%2F%2Fframatube.org%2Ftracker%2Fannounce&ws=https%3A%2F%2Fframatube.org%2Fstatic%2Fstreaming-playlists%2Fhls%2F6050732a-8a7a-43d4-a6cd-809525a1d206%2F6050732a-8a7a-43d4-a6cd-809525a1d206-720-fragmented.mp4","height":720},{"type":"Link","mediaType":"video/mp4","href":"https://framatube.org/static/streaming-playlists/hls/6050732a-8a7a-43d4-a6cd-809525a1d206/6050732a-8a7a-43d4-a6cd-809525a1d206-480-fragmented.mp4","height":480,"size":249562889,"fps":25},{"type":"Link","rel":["metadata","video/mp4"],"mediaType":"application/json","href":"https://framatube.org/api/v1/videos/6050732a-8a7a-43d4-a6cd-809525a1d206/metadata/1309945","height":480,"fps":25},{"type":"Link","mediaType":"application/x-bittorrent","href":"https://framatube.org/static/torrents/6050732a-8a7a-43d4-a6cd-809525a1d206-480-hls.torrent","height":480},{"type":"Link","mediaType":"application/x-bittorrent;x-scheme-handler/magnet","href":"magnet:?xs=https%3A%2F%2Fframatube.org%2Fstatic%2Ftorrents%2F6050732a-8a7a-43d4-a6cd-809525a1d206-480-hls.torrent&xt=urn:btih:5d14f38ded29de629668fe1cfc61a75f4cce2628&dn=D%C3%A9framasoftisons+Internet+%5BFramasoft%5D&tr=wss%3A%2F%2Fframatube.org%3A443%2Ftracker%2Fsocket&tr=https%3A%2F%2Fframatube.org%2Ftracker%2Fannounce&ws=https%3A%2F%2Fframatube.org%2Fstatic%2Fstreaming-playlists%2Fhls%2F6050732a-8a7a-43d4-a6cd-809525a1d206%2F6050732a-8a7a-43d4-a6cd-809525a1d206-480-fragmented.mp4","height":480},{"type":"Link","mediaType":"video/mp4","href":"https://framatube.org/static/streaming-playlists/hls/6050732a-8a7a-43d4-a6cd-809525a1d206/6050732a-8a7a-43d4-a6cd-809525a1d206-360-fragmented.mp4","height":360,"size":170836415,"fps":25},{"type":"Link","rel":["metadata","video/mp4"],"mediaType":"application/json","href":"https://framatube.org/api/v1/videos/6050732a-8a7a-43d4-a6cd-809525a1d206/metadata/1309946","height":360,"fps":25},{"type":"Link","mediaType":"application/x-bittorrent","href":"https://framatube.org/static/torrents/6050732a-8a7a-43d4-a6cd-809525a1d206-360-hls.torrent","height":360},{"type":"Link","mediaType":"application/x-bittorrent;x-scheme-handler/magnet","href":"magnet:?xs=https%3A%2F%2Fframatube.org%2Fstatic%2Ftorrents%2F6050732a-8a7a-43d4-a6cd-809525a1d206-360-hls.torrent&xt=urn:btih:30125488789080ad405ebcee6c214945f31b8f30&dn=D%C3%A9framasoftisons+Internet+%5BFramasoft%5D&tr=wss%3A%2F%2Fframatube.org%3A443%2Ftracker%2Fsocket&tr=https%3A%2F%2Fframatube.org%2Ftracker%2Fannounce&ws=https%3A%2F%2Fframatube.org%2Fstatic%2Fstreaming-playlists%2Fhls%2F6050732a-8a7a-43d4-a6cd-809525a1d206%2F6050732a-8a7a-43d4-a6cd-809525a1d206-360-fragmented.mp4","height":360},{"type":"Link","mediaType":"video/mp4","href":"https://framatube.org/static/streaming-playlists/hls/6050732a-8a7a-43d4-a6cd-809525a1d206/6050732a-8a7a-43d4-a6cd-809525a1d206-240-fragmented.mp4","height":240,"size":112529249,"fps":25},{"type":"Link","rel":["metadata","video/mp4"],"mediaType":"application/json","href":"https://framatube.org/api/v1/videos/6050732a-8a7a-43d4-a6cd-809525a1d206/metadata/1309948","height":240,"fps":25},{"type":"Link","mediaType":"application/x-bittorrent","href":"https://framatube.org/static/torrents/6050732a-8a7a-43d4-a6cd-809525a1d206-240-hls.torrent","height":240},{"type":"Link","mediaType":"application/x-bittorrent;x-scheme-handler/magnet","href":"magnet:?xs=https%3A%2F%2Fframatube.org%2Fstatic%2Ftorrents%2F6050732a-8a7a-43d4-a6cd-809525a1d206-240-hls.torrent&xt=urn:btih:8b452bf4e70b9078d4e74ca8b5523cc9dc70d10a&dn=D%C3%A9framasoftisons+Internet+%5BFramasoft%5D&tr=wss%3A%2F%2Fframatube.org%3A443%2Ftracker%2Fsocket&tr=https%3A%2F%2Fframatube.org%2Ftracker%2Fannounce&ws=https%3A%2F%2Fframatube.org%2Fstatic%2Fstreaming-playlists%2Fhls%2F6050732a-8a7a-43d4-a6cd-809525a1d206%2F6050732a-8a7a-43d4-a6cd-809525a1d206-240-fragmented.mp4","height":240}]}],"likes":"https://framatube.org/videos/watch/6050732a-8a7a-43d4-a6cd-809525a1d206/likes","dislikes":"https://framatube.org/videos/watch/6050732a-8a7a-43d4-a6cd-809525a1d206/dislikes","shares":"https://framatube.org/videos/watch/6050732a-8a7a-43d4-a6cd-809525a1d206/announces","comments":"https://framatube.org/videos/watch/6050732a-8a7a-43d4-a6cd-809525a1d206/comments","attributedTo":[{"type":"Person","id":"https://framatube.org/accounts/framasoft"},{"type":"Group","id":"https://framatube.org/video-channels/bf54d359-cfad-4935-9d45-9d6be93f63e8"}],"to":["https://www.w3.org/ns/activitystreams#Public"],"cc":["https://framatube.org/accounts/framasoft/followers"]},"to":["https://www.w3.org/ns/activitystreams#Public"],"cc":["https://framatube.org/accounts/framasoft/followers"],"@context":["https://www.w3.org/ns/activitystreams","https://w3id.org/security/v1",{"RsaSignature2017":"https://w3id.org/security#RsaSignature2017"},{"pt":"https://joinpeertube.org/ns#","sc":"http://schema.org#","Hashtag":"as:Hashtag","uuid":"sc:identifier","category":"sc:category","licence":"sc:license","subtitleLanguage":"sc:subtitleLanguage","sensitive":"as:sensitive","language":"sc:inLanguage","Infohash":"pt:Infohash","Playlist":"pt:Playlist","PlaylistElement":"pt:PlaylistElement","originallyPublishedAt":"sc:datePublished","views":{"@type":"sc:Number","@id":"pt:views"},"state":{"@type":"sc:Number","@id":"pt:state"},"size":{"@type":"sc:Number","@id":"pt:size"},"fps":{"@type":"sc:Number","@id":"pt:fps"},"startTimestamp":{"@type":"sc:Number","@id":"pt:startTimestamp"},"stopTimestamp":{"@type":"sc:Number","@id":"pt:stopTimestamp"},"position":{"@type":"sc:Number","@id":"pt:position"},"commentsEnabled":{"@type":"sc:Boolean","@id":"pt:commentsEnabled"},"downloadEnabled":{"@type":"sc:Boolean","@id":"pt:downloadEnabled"},"waitTranscoding":{"@type":"sc:Boolean","@id":"pt:waitTranscoding"},"support":{"@type":"sc:Text","@id":"pt:support"},"likes":{"@id":"as:likes","@type":"@id"},"dislikes":{"@id":"as:dislikes","@type":"@id"},"playlists":{"@id":"pt:playlists","@type":"@id"},"shares":{"@id":"as:shares","@type":"@id"},"comments":{"@id":"as:comments","@type":"@id"}}]} \ No newline at end of file diff --git a/test/web/activity_pub/transmogrifier/video_handling_test.exs b/test/web/activity_pub/transmogrifier/video_handling_test.exs new file mode 100644 index 000000000..69c953a2e --- /dev/null +++ b/test/web/activity_pub/transmogrifier/video_handling_test.exs @@ -0,0 +1,93 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.Transmogrifier.VideoHandlingTest do + use Oban.Testing, repo: Pleroma.Repo + use Pleroma.DataCase + + alias Pleroma.Activity + alias Pleroma.Object + alias Pleroma.Object.Fetcher + alias Pleroma.Web.ActivityPub.Transmogrifier + + setup_all do + Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) + :ok + end + + test "skip converting the content when it is nil" do + data = + File.read!("test/fixtures/tesla_mock/framatube.org-video.json") + |> Jason.decode!() + |> Kernel.put_in(["object", "content"], nil) + + {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data) + + assert object = Object.normalize(activity, false) + + assert object.data["content"] == nil + end + + test "it converts content of object to html" do + data = File.read!("test/fixtures/tesla_mock/framatube.org-video.json") |> Jason.decode!() + + {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data) + + assert object = Object.normalize(activity, false) + + assert object.data["content"] == + "

Après avoir mené avec un certain succès la campagne « Dégooglisons Internet » en 2014, l’association Framasoft annonce fin 2019 arrêter progressivement un certain nombre de ses services alternatifs aux GAFAM. Pourquoi ?

Transcription par @aprilorg ici : https://www.april.org/deframasoftisons-internet-pierre-yves-gosset-framasoft

" + end + + test "it remaps video URLs as attachments if necessary" do + {:ok, object} = + Fetcher.fetch_object_from_id( + "https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3" + ) + + assert object.data["url"] == + "https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3" + + assert object.data["attachment"] == [ + %{ + "type" => "Link", + "mediaType" => "video/mp4", + "name" => nil, + "url" => [ + %{ + "href" => + "https://peertube.moe/static/webseed/df5f464b-be8d-46fb-ad81-2d4c2d1630e3-480.mp4", + "mediaType" => "video/mp4", + "type" => "Link" + } + ] + } + ] + + data = File.read!("test/fixtures/tesla_mock/framatube.org-video.json") |> Jason.decode!() + + {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data) + + assert object = Object.normalize(activity, false) + + assert object.data["attachment"] == [ + %{ + "type" => "Link", + "mediaType" => "video/mp4", + "name" => nil, + "url" => [ + %{ + "href" => + "https://framatube.org/static/webseed/6050732a-8a7a-43d4-a6cd-809525a1d206-1080.mp4", + "mediaType" => "video/mp4", + "type" => "Link" + } + ] + } + ] + + assert object.data["url"] == + "https://framatube.org/videos/watch/6050732a-8a7a-43d4-a6cd-809525a1d206" + end +end diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index cc55a7be7..0a3291d49 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -8,7 +8,6 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do alias Pleroma.Activity alias Pleroma.Object - alias Pleroma.Object.Fetcher alias Pleroma.Tests.ObanHelpers alias Pleroma.User alias Pleroma.Web.ActivityPub.Transmogrifier @@ -355,83 +354,6 @@ test "it works for incoming unfollows with an existing follow" do refute User.following?(User.get_cached_by_ap_id(data["actor"]), user) end - test "skip converting the content when it is nil" do - object_id = "https://peertube.social/videos/watch/278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe" - - {:ok, object} = Fetcher.fetch_and_contain_remote_object_from_id(object_id) - - result = - Pleroma.Web.ActivityPub.Transmogrifier.fix_object(Map.merge(object, %{"content" => nil})) - - assert result["content"] == nil - end - - test "it converts content of object to html" do - object_id = "https://peertube.social/videos/watch/278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe" - - {:ok, %{"content" => content_markdown}} = - Fetcher.fetch_and_contain_remote_object_from_id(object_id) - - {:ok, %Pleroma.Object{data: %{"content" => content}} = object} = - Fetcher.fetch_object_from_id(object_id) - - assert content_markdown == - "Support this and our other Michigan!/usr/group videos and meetings. Learn more at http://mug.org/membership\n\nTwenty Years in Jail: FreeBSD's Jails, Then and Now\n\nJails started as a limited virtualization system, but over the last two years they've..." - - assert content == - "

Support this and our other Michigan!/usr/group videos and meetings. Learn more at http://mug.org/membership

Twenty Years in Jail: FreeBSD’s Jails, Then and Now

Jails started as a limited virtualization system, but over the last two years they’ve…

" - - assert object.data["mediaType"] == "text/html" - end - - test "it remaps video URLs as attachments if necessary" do - {:ok, object} = - Fetcher.fetch_object_from_id( - "https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3" - ) - - assert object.data["url"] == - "https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3" - - assert object.data["attachment"] == [ - %{ - "type" => "Link", - "mediaType" => "video/mp4", - "url" => [ - %{ - "href" => - "https://peertube.moe/static/webseed/df5f464b-be8d-46fb-ad81-2d4c2d1630e3-480.mp4", - "mediaType" => "video/mp4", - "type" => "Link" - } - ] - } - ] - - {:ok, object} = - Fetcher.fetch_object_from_id( - "https://framatube.org/videos/watch/6050732a-8a7a-43d4-a6cd-809525a1d206" - ) - - assert object.data["attachment"] == [ - %{ - "type" => "Link", - "mediaType" => "video/mp4", - "url" => [ - %{ - "href" => - "https://framatube.org/static/webseed/6050732a-8a7a-43d4-a6cd-809525a1d206-1080.mp4", - "mediaType" => "video/mp4", - "type" => "Link" - } - ] - } - ] - - assert object.data["url"] == - "https://framatube.org/videos/watch/6050732a-8a7a-43d4-a6cd-809525a1d206" - end - test "it accepts Flag activities" do user = insert(:user) other_user = insert(:user) @@ -1133,75 +1055,7 @@ test "fixes data for object when url is map" do } end - test "fixes data for video object" do - object = %{ - "type" => "Video", - "url" => [ - %{ - "type" => "Link", - "mimeType" => "video/mp4", - "href" => "https://peede8d-46fb-ad81-2d4c2d1630e3-480.mp4" - }, - %{ - "type" => "Link", - "mimeType" => "video/mp4", - "href" => "https://peertube46fb-ad81-2d4c2d1630e3-240.mp4" - }, - %{ - "type" => "Link", - "mimeType" => "text/html", - "href" => "https://peertube.-2d4c2d1630e3" - }, - %{ - "type" => "Link", - "mimeType" => "text/html", - "href" => "https://peertube.-2d4c2d16377-42" - } - ] - } - - assert Transmogrifier.fix_url(object) == %{ - "attachment" => [ - %{ - "href" => "https://peede8d-46fb-ad81-2d4c2d1630e3-480.mp4", - "mimeType" => "video/mp4", - "type" => "Link" - } - ], - "type" => "Video", - "url" => "https://peertube.-2d4c2d1630e3" - } - end - - test "fixes url for not Video object" do - object = %{ - "type" => "Text", - "url" => [ - %{ - "type" => "Link", - "mimeType" => "text/html", - "href" => "https://peertube.-2d4c2d1630e3" - }, - %{ - "type" => "Link", - "mimeType" => "text/html", - "href" => "https://peertube.-2d4c2d16377-42" - } - ] - } - - assert Transmogrifier.fix_url(object) == %{ - "type" => "Text", - "url" => "https://peertube.-2d4c2d1630e3" - } - - assert Transmogrifier.fix_url(%{"type" => "Text", "url" => []}) == %{ - "type" => "Text", - "url" => "" - } - end - - test "retunrs not modified object" do + test "returns non-modified object" do assert Transmogrifier.fix_url(%{"type" => "Text"}) == %{"type" => "Text"} end end From 2132b24a9df8116e12abc8c458cff4c3850aeda0 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 20 Aug 2020 04:27:59 +0200 Subject: [PATCH 03/28] object_validators: likes & announcements as [ObjectID] --- .../activity_pub/object_validators/audio_video_validator.ex | 4 ++-- .../web/activity_pub/object_validators/note_validator.ex | 4 ++-- .../web/activity_pub/object_validators/question_validator.ex | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex index a6119e627..16973e5db 100644 --- a/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex @@ -49,8 +49,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator do field(:inReplyTo, ObjectValidators.ObjectID) field(:url, ObjectValidators.Uri) - field(:likes, {:array, :string}, default: []) - field(:announcements, {:array, :string}, default: []) + field(:likes, {:array, ObjectValidators.ObjectID}, default: []) + field(:announcements, {:array, ObjectValidators.ObjectID}, default: []) end def cast_and_apply(data) do diff --git a/lib/pleroma/web/activity_pub/object_validators/note_validator.ex b/lib/pleroma/web/activity_pub/object_validators/note_validator.ex index ab4469a59..e47cbaaea 100644 --- a/lib/pleroma/web/activity_pub/object_validators/note_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/note_validator.ex @@ -43,8 +43,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do field(:inReplyTo, ObjectValidators.ObjectID) field(:url, ObjectValidators.Uri) - field(:likes, {:array, :string}, default: []) - field(:announcements, {:array, :string}, default: []) + field(:likes, {:array, ObjectValidators.ObjectID}, default: []) + field(:announcements, {:array, ObjectValidators.ObjectID}, default: []) end def cast_and_validate(data) do diff --git a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex index 934d3c1ea..9310485dc 100644 --- a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex @@ -47,8 +47,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do # short identifier for PleromaFE to group statuses by context field(:context_id, :integer) - field(:likes, {:array, :string}, default: []) - field(:announcements, {:array, :string}, default: []) + field(:likes, {:array, ObjectValidators.ObjectID}, default: []) + field(:announcements, {:array, ObjectValidators.ObjectID}, default: []) field(:closed, ObjectValidators.DateTime) field(:voters, {:array, ObjectValidators.ObjectID}, default: []) From 1b3d5956b1be7faac4e1230d788307650acce991 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 20 Aug 2020 20:03:07 +0200 Subject: [PATCH 04/28] Pipeline Ingestion: Article --- lib/pleroma/web/activity_pub/activity_pub.ex | 2 +- .../web/activity_pub/object_validator.ex | 17 ++++- ...validator.ex => article_note_validator.ex} | 55 +++++++++++--- lib/pleroma/web/activity_pub/side_effects.ex | 2 +- .../web/activity_pub/transmogrifier.ex | 4 +- .../wedistribute-create-article.json | 1 + ...st.exs => article_note_validator_test.exs} | 6 +- .../transmogrifier/article_handling_test.exs | 75 +++++++++++++++++++ test/web/activity_pub/transmogrifier_test.exs | 9 --- 9 files changed, 143 insertions(+), 28 deletions(-) rename lib/pleroma/web/activity_pub/object_validators/{note_validator.ex => article_note_validator.ex} (59%) create mode 100644 test/fixtures/tesla_mock/wedistribute-create-article.json rename test/web/activity_pub/object_validators/{note_validator_test.exs => article_note_validator_test.exs} (76%) create mode 100644 test/web/activity_pub/transmogrifier/article_handling_test.exs diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index bceec8bd1..3ab045737 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -84,7 +84,7 @@ defp increase_replies_count_if_reply(%{ defp increase_replies_count_if_reply(_create_data), do: :noop - @object_types ~w[ChatMessage Question Answer Audio Video Event] + @object_types ~w[ChatMessage Question Answer Audio Video Event Article] @spec persist(map(), keyword()) :: {:ok, Activity.t() | Object.t()} def persist(%{"type" => type} = object, meta) when type in @object_types do with {:ok, object} <- Object.create(object) do diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index 081f96389..bd0a2a8dc 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -17,6 +17,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do alias Pleroma.Web.ActivityPub.ObjectValidators.AcceptRejectValidator alias Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator alias Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator + alias Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator alias Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator alias Pleroma.Web.ActivityPub.ObjectValidators.BlockValidator alias Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator @@ -160,6 +161,16 @@ def validate(%{"type" => type} = object, meta) when type in ~w[Audio Video] do end end + def validate(%{"type" => "Article"} = object, meta) do + with {:ok, object} <- + object + |> ArticleNoteValidator.cast_and_validate() + |> Ecto.Changeset.apply_action(:insert) do + object = stringify_keys(object) + {:ok, object, meta} + end + end + def validate(%{"type" => "Answer"} = object, meta) do with {:ok, object} <- object @@ -199,7 +210,7 @@ def validate( %{"type" => "Create", "object" => %{"type" => objtype} = object} = create_activity, meta ) - when objtype in ~w[Question Answer Audio Video Event] do + when objtype in ~w[Question Answer Audio Video Event Article] do with {:ok, object_data} <- cast_and_apply(object), meta = Keyword.put(meta, :object_data, object_data |> stringify_keys), {:ok, create_activity} <- @@ -241,6 +252,10 @@ def cast_and_apply(%{"type" => "Event"} = object) do EventValidator.cast_and_apply(object) end + def cast_and_apply(%{"type" => "Article"} = object) do + ArticleNoteValidator.cast_and_apply(object) + end + def cast_and_apply(o), do: {:error, {:validator_not_set, o}} # is_struct/1 isn't present in Elixir 1.8.x diff --git a/lib/pleroma/web/activity_pub/object_validators/note_validator.ex b/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex similarity index 59% rename from lib/pleroma/web/activity_pub/object_validators/note_validator.ex rename to lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex index e47cbaaea..5b7dad517 100644 --- a/lib/pleroma/web/activity_pub/object_validators/note_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex @@ -2,15 +2,19 @@ # Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do +defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator do use Ecto.Schema alias Pleroma.EctoType.ActivityPub.ObjectValidators + alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator + alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes + alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations alias Pleroma.Web.ActivityPub.Transmogrifier import Ecto.Changeset @primary_key false + @derive Jason.Encoder embedded_schema do field(:id, ObjectValidators.ObjectID, primary_key: true) @@ -30,13 +34,14 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do # short identifier for PleromaFE to group statuses by context field(:context_id, :integer) + # TODO: Remove actor on objects field(:actor, ObjectValidators.ObjectID) + field(:attributedTo, ObjectValidators.ObjectID) field(:published, ObjectValidators.DateTime) field(:emoji, ObjectValidators.Emoji, default: %{}) field(:sensitive, :boolean, default: false) - # TODO: Write type - field(:attachment, {:array, :map}, default: []) + embeds_many(:attachment, AttachmentValidator) field(:replies_count, :integer, default: 0) field(:like_count, :integer, default: 0) field(:announcement_count, :integer, default: 0) @@ -47,27 +52,55 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do field(:announcements, {:array, ObjectValidators.ObjectID}, default: []) end + def cast_and_apply(data) do + data + |> cast_data + |> apply_action(:insert) + end + def cast_and_validate(data) do data |> cast_data() |> validate_data() end - defp fix(data) do - data - |> Transmogrifier.fix_emoji() - end - def cast_data(data) do data = fix(data) %__MODULE__{} - |> cast(data, __schema__(:fields)) + |> changeset(data) + end + + defp fix_url(%{"url" => url} = data) when is_map(url) do + Map.put(data, "url", url["href"]) + end + + defp fix_url(data), do: data + + defp fix(data) do + data + |> CommonFixes.fix_defaults() + |> CommonFixes.fix_attribution() + |> CommonFixes.fix_actor() + |> fix_url() + |> Transmogrifier.fix_emoji() + end + + def changeset(struct, data) do + data = fix(data) + + struct + |> cast(data, __schema__(:fields) -- [:attachment]) + |> cast_embed(:attachment) end def validate_data(data_cng) do data_cng - |> validate_inclusion(:type, ["Note"]) - |> validate_required([:id, :actor, :to, :cc, :type, :content, :context]) + |> validate_inclusion(:type, ["Article", "Note"]) + |> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id]) + |> CommonValidations.validate_any_presence([:cc, :to]) + |> CommonValidations.validate_fields_match([:actor, :attributedTo]) + |> CommonValidations.validate_actor_presence() + |> CommonValidations.validate_host_match() end end diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index b5c720c7a..b9a83a544 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -336,7 +336,7 @@ def handle_object_creation(%{"type" => "Answer"} = object_map, meta) do end def handle_object_creation(%{"type" => objtype} = object, meta) - when objtype in ~w[Audio Video Question Event] do + when objtype in ~w[Audio Video Question Event Article] do with {:ok, object, meta} <- Pipeline.common_pipeline(object, meta) do {:ok, object, meta} end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index e14936c10..80f529704 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -424,7 +424,7 @@ def handle_incoming( %{"type" => "Create", "object" => %{"type" => objtype} = object} = data, options ) - when objtype in ~w{Article Note Page} do + when objtype in ~w{Note Page} do actor = Containment.get_actor(data) with nil <- Activity.get_create_by_object_ap_id(object["id"]), @@ -518,7 +518,7 @@ def handle_incoming( %{"type" => "Create", "object" => %{"type" => objtype}} = data, _options ) - when objtype in ~w{Question Answer ChatMessage Audio Video Event} do + when objtype in ~w{Question Answer ChatMessage Audio Video Event Article} do data = Map.put(data, "object", strip_internal_fields(data["object"])) with {:ok, %User{}} <- ObjectValidator.fetch_actor(data), diff --git a/test/fixtures/tesla_mock/wedistribute-create-article.json b/test/fixtures/tesla_mock/wedistribute-create-article.json new file mode 100644 index 000000000..3cfef8b99 --- /dev/null +++ b/test/fixtures/tesla_mock/wedistribute-create-article.json @@ -0,0 +1 @@ +{"@context":["https:\/\/www.w3.org\/ns\/activitystreams"],"type":"Create","actor":"https:\/\/wedistribute.org\/wp-json\/pterotype\/v1\/actor\/-blog","object":{"@context":["https:\/\/www.w3.org\/ns\/activitystreams"],"type":"Article","name":"The end is near: Mastodon plans to drop OStatus support","content":"\n

The days of OStatus are numbered. The venerable protocol has served as a glue between many different types of servers since the early days of the Fediverse, connecting StatusNet (now GNU Social) to Friendica, Hubzilla, Mastodon, and Pleroma.<\/p>\n\n\n\n

Now that many fediverse platforms support ActivityPub as a successor protocol, Mastodon appears to be drawing a line in the sand. In a Patreon update<\/a>, Eugen Rochko writes:<\/p>\n\n\n\n

...OStatus...has overstayed its welcome in the code...and now that most of the network uses ActivityPub, it's time for it to go. <\/p>Eugen Rochko, Mastodon creator<\/cite><\/blockquote>\n\n\n\n

The pull request<\/a> to remove Pubsubhubbub and Salmon, two of the main components of OStatus, has already been merged into Mastodon's master branch.<\/p>\n\n\n\n

Some projects will be left in the dark as a side effect of this. GNU Social and PostActiv, for example, both only communicate using OStatus. While some discussion<\/a> exists regarding adopting ActivityPub for GNU Social, and a plugin is in development<\/a>, it hasn't been formally adopted yet. We just hope that the Free Software Foundation's instance<\/a> gets updated in time!<\/p>\n","summary":"One of the largest platforms in the federated social web is dropping the protocol that it started with.","attributedTo":"https:\/\/wedistribute.org\/wp-json\/pterotype\/v1\/actor\/-blog","url":"https:\/\/wedistribute.org\/2019\/07\/mastodon-drops-ostatus\/","to":["https:\/\/www.w3.org\/ns\/activitystreams#Public","https:\/\/wedistribute.org\/wp-json\/pterotype\/v1\/actor\/-blog\/followers"],"id":"https:\/\/wedistribute.org\/wp-json\/pterotype\/v1\/object\/85810","likes":"https:\/\/wedistribute.org\/wp-json\/pterotype\/v1\/object\/85810\/likes","shares":"https:\/\/wedistribute.org\/wp-json\/pterotype\/v1\/object\/85810\/shares"},"to":["https:\/\/www.w3.org\/ns\/activitystreams#Public","https:\/\/wedistribute.org\/wp-json\/pterotype\/v1\/actor\/-blog\/followers"],"id":"https:\/\/wedistribute.org\/wp-json\/pterotype\/v1\/object\/85809"} \ No newline at end of file diff --git a/test/web/activity_pub/object_validators/note_validator_test.exs b/test/web/activity_pub/object_validators/article_note_validator_test.exs similarity index 76% rename from test/web/activity_pub/object_validators/note_validator_test.exs rename to test/web/activity_pub/object_validators/article_note_validator_test.exs index 30c481ffb..cc6dab872 100644 --- a/test/web/activity_pub/object_validators/note_validator_test.exs +++ b/test/web/activity_pub/object_validators/article_note_validator_test.exs @@ -2,10 +2,10 @@ # Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidatorTest do +defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidatorTest do use Pleroma.DataCase - alias Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator + alias Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator alias Pleroma.Web.ActivityPub.Utils import Pleroma.Factory @@ -29,7 +29,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidatorTest do end test "a basic note validates", %{note: note} do - %{valid?: true} = NoteValidator.cast_and_validate(note) + %{valid?: true} = ArticleNoteValidator.cast_and_validate(note) end end end diff --git a/test/web/activity_pub/transmogrifier/article_handling_test.exs b/test/web/activity_pub/transmogrifier/article_handling_test.exs new file mode 100644 index 000000000..9b12a470a --- /dev/null +++ b/test/web/activity_pub/transmogrifier/article_handling_test.exs @@ -0,0 +1,75 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.Transmogrifier.ArticleHandlingTest do + use Oban.Testing, repo: Pleroma.Repo + use Pleroma.DataCase + + alias Pleroma.Activity + alias Pleroma.Object + alias Pleroma.Object.Fetcher + alias Pleroma.Web.ActivityPub.Transmogrifier + + test "Pterotype (Wordpress Plugin) Article" do + Tesla.Mock.mock(fn %{url: "https://wedistribute.org/wp-json/pterotype/v1/actor/-blog"} -> + %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/wedistribute-user.json")} + end) + + data = + File.read!("test/fixtures/tesla_mock/wedistribute-create-article.json") |> Jason.decode!() + + {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) + + object = Object.normalize(data["object"]) + + assert object.data["name"] == "The end is near: Mastodon plans to drop OStatus support" + + assert object.data["summary"] == + "One of the largest platforms in the federated social web is dropping the protocol that it started with." + + assert object.data["url"] == "https://wedistribute.org/2019/07/mastodon-drops-ostatus/" + end + + test "Plume Article" do + Tesla.Mock.mock(fn + %{url: "https://baptiste.gelez.xyz/~/PlumeDevelopment/this-month-in-plume-june-2018/"} -> + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/baptiste.gelex.xyz-article.json") + } + + %{url: "https://baptiste.gelez.xyz/@/BaptisteGelez"} -> + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/baptiste.gelex.xyz-user.json") + } + end) + + {:ok, object} = + Fetcher.fetch_object_from_id( + "https://baptiste.gelez.xyz/~/PlumeDevelopment/this-month-in-plume-june-2018/" + ) + + assert object.data["name"] == "This Month in Plume: June 2018" + + assert object.data["url"] == + "https://baptiste.gelez.xyz/~/PlumeDevelopment/this-month-in-plume-june-2018/" + end + + test "Prismo Article" do + Tesla.Mock.mock(fn %{url: "https://prismo.news/@mxb"} -> + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/https___prismo.news__mxb.json") + } + end) + + data = File.read!("test/fixtures/prismo-url-map.json") |> Jason.decode!() + + {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) + object = Object.normalize(data["object"]) + + assert object.data["url"] == "https://prismo.news/posts/83" + end +end diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 0a3291d49..561674f01 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -44,15 +44,6 @@ test "it works for incoming notices with tag not being an array (kroeg)" do assert "test" in object.data["tag"] end - test "it works for incoming notices with url not being a string (prismo)" do - data = File.read!("test/fixtures/prismo-url-map.json") |> Poison.decode!() - - {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) - object = Object.normalize(data["object"]) - - assert object.data["url"] == "https://prismo.news/posts/83" - end - test "it cleans up incoming notices which are not really DMs" do user = insert(:user) other_user = insert(:user) From f18178cb096b9a00ed12ff0fe36893f118ec6649 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Sat, 22 Aug 2020 02:01:33 +0200 Subject: [PATCH 05/28] AttachmentValidator: directly embed url schema and pass it fix_media_type --- .../object_validators/attachment_validator.ex | 21 ++++++++++++++-- .../object_validators/url_object_validator.ex | 24 ------------------- 2 files changed, 19 insertions(+), 26 deletions(-) delete mode 100644 lib/pleroma/web/activity_pub/object_validators/url_object_validator.ex diff --git a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex index c8b148280..df102a134 100644 --- a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do use Ecto.Schema + alias Pleroma.EctoType.ActivityPub.ObjectValidators alias Pleroma.Web.ActivityPub.ObjectValidators.UrlObjectValidator import Ecto.Changeset @@ -15,7 +16,11 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do field(:mediaType, :string, default: "application/octet-stream") field(:name, :string) - embeds_many(:url, UrlObjectValidator) + embeds_many :url, UrlObjectValidator, primary_key: false do + field(:type, :string) + field(:href, ObjectValidators.Uri) + field(:mediaType, :string, default: "application/octet-stream") + end end def cast_and_validate(data) do @@ -37,7 +42,18 @@ def changeset(struct, data) do struct |> cast(data, [:type, :mediaType, :name]) - |> cast_embed(:url, required: true) + |> cast_embed(:url, with: &url_changeset/2) + |> validate_inclusion(:type, ~w[Link Document Audio Image Video]) + |> validate_required([:type, :mediaType, :url]) + end + + def url_changeset(struct, data) do + data = fix_media_type(data) + + struct + |> cast(data, [:type, :href, :mediaType]) + |> validate_inclusion(:type, ["Link"]) + |> validate_required([:type, :href, :mediaType]) end def fix_media_type(data) do @@ -75,6 +91,7 @@ defp fix_url(data) do def validate_data(cng) do cng + |> validate_inclusion(:type, ~w[Document Audio Image Video]) |> validate_required([:mediaType, :url, :type]) end end diff --git a/lib/pleroma/web/activity_pub/object_validators/url_object_validator.ex b/lib/pleroma/web/activity_pub/object_validators/url_object_validator.ex deleted file mode 100644 index 881030f38..000000000 --- a/lib/pleroma/web/activity_pub/object_validators/url_object_validator.ex +++ /dev/null @@ -1,24 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.ActivityPub.ObjectValidators.UrlObjectValidator do - use Ecto.Schema - - alias Pleroma.EctoType.ActivityPub.ObjectValidators - - import Ecto.Changeset - @primary_key false - - embedded_schema do - field(:type, :string) - field(:href, ObjectValidators.Uri) - field(:mediaType, :string, default: "application/octet-stream") - end - - def changeset(struct, data) do - struct - |> cast(data, __schema__(:fields)) - |> validate_required([:type, :href, :mediaType]) - end -end From f1f44069ae525fd21127e5ceccc61016c12f4427 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 11 Sep 2020 19:58:58 +0200 Subject: [PATCH 06/28] Fetcher: Correctly return MRF reject reason --- lib/pleroma/object/fetcher.ex | 4 ++-- lib/pleroma/web/activity_pub/activity_pub.ex | 4 ++-- test/object/fetcher_test.exs | 25 +++++++++++++++----- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 1de2ce6c3..24dc7cb95 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -98,8 +98,8 @@ def fetch_object_from_id(id, options \\ []) do {:containment, _} -> {:error, "Object containment failed."} - {:transmogrifier, {:error, {:reject, nil}}} -> - {:reject, nil} + {:transmogrifier, {:error, {:reject, e}}} -> + {:reject, e} {:transmogrifier, _} = e -> {:error, e} diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 66a9f78a3..b2205bff7 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -154,8 +154,8 @@ def insert(map, local \\ true, fake \\ false, bypass_actor_check \\ false) when {:remote_limit_pass, _} -> {:error, :remote_limit} - {:reject, reason} -> - {:error, reason} + {:reject, _} = e -> + {:error, e} end end diff --git a/test/object/fetcher_test.exs b/test/object/fetcher_test.exs index 16cfa7f5c..3173ee31c 100644 --- a/test/object/fetcher_test.exs +++ b/test/object/fetcher_test.exs @@ -6,10 +6,13 @@ defmodule Pleroma.Object.FetcherTest do use Pleroma.DataCase alias Pleroma.Activity + alias Pleroma.Config alias Pleroma.Object alias Pleroma.Object.Fetcher - import Tesla.Mock + + import ExUnit.CaptureLog import Mock + import Tesla.Mock setup do mock(fn @@ -71,20 +74,20 @@ test "it works when fetching the OP actor errors out" do setup do: clear_config([:instance, :federation_incoming_replies_max_depth]) test "it returns thread depth exceeded error if thread depth is exceeded" do - Pleroma.Config.put([:instance, :federation_incoming_replies_max_depth], 0) + Config.put([:instance, :federation_incoming_replies_max_depth], 0) assert {:error, "Max thread distance exceeded."} = Fetcher.fetch_object_from_id(@ap_id, depth: 1) end test "it fetches object if max thread depth is restricted to 0 and depth is not specified" do - Pleroma.Config.put([:instance, :federation_incoming_replies_max_depth], 0) + Config.put([:instance, :federation_incoming_replies_max_depth], 0) assert {:ok, _} = Fetcher.fetch_object_from_id(@ap_id) end test "it fetches object if requested depth does not exceed max thread depth" do - Pleroma.Config.put([:instance, :federation_incoming_replies_max_depth], 10) + Config.put([:instance, :federation_incoming_replies_max_depth], 10) assert {:ok, _} = Fetcher.fetch_object_from_id(@ap_id, depth: 10) end @@ -120,6 +123,16 @@ test "it fetches an object" do assert object == object_again end + + test "Return MRF reason when fetched status is rejected by one" do + clear_config([:mrf_keyword, :reject], ["yeah"]) + clear_config([:mrf, :policies], [Pleroma.Web.ActivityPub.MRF.KeywordPolicy]) + + assert {:reject, "[KeywordPolicy] Matches with rejected keyword"} == + Fetcher.fetch_object_from_id( + "http://mastodon.example.org/@admin/99541947525187367" + ) + end end describe "implementation quirks" do @@ -212,7 +225,7 @@ test "it can refetch pruned objects" do Pleroma.Signature, [:passthrough], [] do - Pleroma.Config.put([:activitypub, :sign_object_fetches], true) + Config.put([:activitypub, :sign_object_fetches], true) Fetcher.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367") @@ -223,7 +236,7 @@ test "it can refetch pruned objects" do Pleroma.Signature, [:passthrough], [] do - Pleroma.Config.put([:activitypub, :sign_object_fetches], false) + Config.put([:activitypub, :sign_object_fetches], false) Fetcher.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367") From f88dc1937e5aa4208143fa68400a5c38a1b9eddf Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 31 Aug 2020 16:48:24 -0500 Subject: [PATCH 07/28] MastodonAPI.StatusView.get_user/1 --> CommonAPI.get_user/1 --- .../web/admin_api/views/status_view.ex | 3 ++- lib/pleroma/web/common_api/common_api.ex | 17 +++++++++++++ .../web/mastodon_api/views/status_view.ex | 25 +++---------------- .../web/pleroma_api/views/scrobble_view.ex | 4 +-- test/web/common_api/common_api_test.exs | 20 +++++++++++++++ 5 files changed, 45 insertions(+), 24 deletions(-) diff --git a/lib/pleroma/web/admin_api/views/status_view.ex b/lib/pleroma/web/admin_api/views/status_view.ex index 500800be2..6042a22b6 100644 --- a/lib/pleroma/web/admin_api/views/status_view.ex +++ b/lib/pleroma/web/admin_api/views/status_view.ex @@ -8,6 +8,7 @@ defmodule Pleroma.Web.AdminAPI.StatusView do require Pleroma.Constants alias Pleroma.Web.AdminAPI + alias Pleroma.Web.CommonAPI alias Pleroma.Web.MastodonAPI defdelegate merge_account_views(user), to: AdminAPI.AccountView @@ -17,7 +18,7 @@ def render("index.json", opts) do end def render("show.json", %{activity: %{data: %{"object" => _object}} = activity} = opts) do - user = MastodonAPI.StatusView.get_user(activity.data["actor"]) + user = CommonAPI.get_user(activity.data["actor"]) MastodonAPI.StatusView.render("show.json", opts) |> Map.merge(%{account: merge_account_views(user)}) diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 5ad2b91c2..d6e9d3d67 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -550,4 +550,21 @@ def hide_reblogs(%User{} = user, %User{} = target) do def show_reblogs(%User{} = user, %User{} = target) do UserRelationship.delete_reblog_mute(user, target) end + + def get_user(ap_id, fake_record_fallback \\ true) do + cond do + user = User.get_cached_by_ap_id(ap_id) -> + user + + user = User.get_by_guessed_nickname(ap_id) -> + user + + fake_record_fallback -> + # TODO: refactor (fake records is never a good idea) + User.error_user(ap_id) + + true -> + nil + end + end end diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 3fe1967be..66732d09e 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -56,23 +56,6 @@ defp get_replied_to_activities(activities) do end) end - def get_user(ap_id, fake_record_fallback \\ true) do - cond do - user = User.get_cached_by_ap_id(ap_id) -> - user - - user = User.get_by_guessed_nickname(ap_id) -> - user - - fake_record_fallback -> - # TODO: refactor (fake records is never a good idea) - User.error_user(ap_id) - - true -> - nil - end - end - defp get_context_id(%{data: %{"context_id" => context_id}}) when not is_nil(context_id), do: context_id @@ -120,7 +103,7 @@ def render("index.json", opts) do # Note: unresolved users are filtered out actors = (activities ++ parent_activities) - |> Enum.map(&get_user(&1.data["actor"], false)) + |> Enum.map(&CommonAPI.get_user(&1.data["actor"], false)) |> Enum.filter(& &1) UserRelationship.view_relationships_option(reading_user, actors, subset: :source_mutes) @@ -139,7 +122,7 @@ def render( "show.json", %{activity: %{data: %{"type" => "Announce", "object" => _object}} = activity} = opts ) do - user = get_user(activity.data["actor"]) + user = CommonAPI.get_user(activity.data["actor"]) created_at = Utils.to_masto_date(activity.data["published"]) activity_object = Object.normalize(activity) @@ -212,7 +195,7 @@ def render( def render("show.json", %{activity: %{data: %{"object" => _object}} = activity} = opts) do object = Object.normalize(activity) - user = get_user(activity.data["actor"]) + user = CommonAPI.get_user(activity.data["actor"]) user_follower_address = user.follower_address like_count = object.data["like_count"] || 0 @@ -266,7 +249,7 @@ def render("show.json", %{activity: %{data: %{"object" => _object}} = activity} reply_to = get_reply_to(activity, opts) - reply_to_user = reply_to && get_user(reply_to.data["actor"]) + reply_to_user = reply_to && CommonAPI.get_user(reply_to.data["actor"]) content = object diff --git a/lib/pleroma/web/pleroma_api/views/scrobble_view.ex b/lib/pleroma/web/pleroma_api/views/scrobble_view.ex index bbff93abe..95bd4c368 100644 --- a/lib/pleroma/web/pleroma_api/views/scrobble_view.ex +++ b/lib/pleroma/web/pleroma_api/views/scrobble_view.ex @@ -10,14 +10,14 @@ defmodule Pleroma.Web.PleromaAPI.ScrobbleView do alias Pleroma.Activity alias Pleroma.HTML alias Pleroma.Object + alias Pleroma.Web.CommonAPI alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.MastodonAPI.AccountView - alias Pleroma.Web.MastodonAPI.StatusView def render("show.json", %{activity: %Activity{data: %{"type" => "Listen"}} = activity} = opts) do object = Object.normalize(activity) - user = StatusView.get_user(activity.data["actor"]) + user = CommonAPI.get_user(activity.data["actor"]) created_at = Utils.to_masto_date(activity.data["published"]) %{ diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs index 4ba6232dc..d171b344a 100644 --- a/test/web/common_api/common_api_test.exs +++ b/test/web/common_api/common_api_test.exs @@ -1126,4 +1126,24 @@ test "respects visibility=private" do assert Visibility.get_visibility(activity) == "private" end end + + describe "get_user/1" do + test "gets user by ap_id" do + user = insert(:user) + assert CommonAPI.get_user(user.ap_id) == user + end + + test "gets user by guessed nickname" do + user = insert(:user, ap_id: "", nickname: "mario@mushroom.kingdom") + assert CommonAPI.get_user("https://mushroom.kingdom/users/mario") == user + end + + test "fallback" do + assert %User{ + name: "", + ap_id: "", + nickname: "erroruser@example.com" + } = CommonAPI.get_user("") + end + end end From b40a627ab02f9f63eac42ce6fc65282fc6cb6b92 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 31 Aug 2020 19:56:05 -0500 Subject: [PATCH 08/28] AdminAPI: delete a chat message --- lib/pleroma/moderation_log.ex | 24 +++++++++ .../admin_api/controllers/chat_controller.ex | 37 +++++++++++++ .../operations/admin/chat_operation.ex | 44 +++++++++++++++ lib/pleroma/web/router.ex | 2 + test/support/factory.ex | 54 +++++++++++++++++++ .../controllers/chat_controller_test.exs | 53 ++++++++++++++++++ 6 files changed, 214 insertions(+) create mode 100644 lib/pleroma/web/admin_api/controllers/chat_controller.ex create mode 100644 lib/pleroma/web/api_spec/operations/admin/chat_operation.ex create mode 100644 test/web/admin_api/controllers/chat_controller_test.exs diff --git a/lib/pleroma/moderation_log.ex b/lib/pleroma/moderation_log.ex index 31c9afe2a..47036a6f6 100644 --- a/lib/pleroma/moderation_log.ex +++ b/lib/pleroma/moderation_log.ex @@ -320,6 +320,19 @@ def insert_log(%{ |> insert_log_entry_with_message() end + @spec insert_log(%{actor: User, action: String.t(), subject_id: String.t()}) :: + {:ok, ModerationLog} | {:error, any} + def insert_log(%{actor: %User{} = actor, action: "chat_message_delete", subject_id: subject_id}) do + %ModerationLog{ + data: %{ + "actor" => %{"nickname" => actor.nickname}, + "action" => "chat_message_delete", + "subject_id" => subject_id + } + } + |> insert_log_entry_with_message() + end + @spec insert_log_entry_with_message(ModerationLog) :: {:ok, ModerationLog} | {:error, any} defp insert_log_entry_with_message(entry) do entry.data["message"] @@ -627,6 +640,17 @@ def get_log_entry_message(%ModerationLog{ "@#{actor_nickname} updated users: #{users_to_nicknames_string(subjects)}" end + @spec get_log_entry_message(ModerationLog) :: String.t() + def get_log_entry_message(%ModerationLog{ + data: %{ + "actor" => %{"nickname" => actor_nickname}, + "action" => "chat_message_delete", + "subject_id" => subject_id + } + }) do + "@#{actor_nickname} deleted chat message ##{subject_id}" + end + defp nicknames_to_string(nicknames) do nicknames |> Enum.map(&"@#{&1}") diff --git a/lib/pleroma/web/admin_api/controllers/chat_controller.ex b/lib/pleroma/web/admin_api/controllers/chat_controller.ex new file mode 100644 index 000000000..bcce824d2 --- /dev/null +++ b/lib/pleroma/web/admin_api/controllers/chat_controller.ex @@ -0,0 +1,37 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.AdminAPI.ChatController do + use Pleroma.Web, :controller + + alias Pleroma.Activity + alias Pleroma.ModerationLog + alias Pleroma.Plugs.OAuthScopesPlug + alias Pleroma.Web.CommonAPI + + require Logger + + plug(Pleroma.Web.ApiSpec.CastAndValidate) + + plug( + OAuthScopesPlug, + %{scopes: ["write:chats"], admin: true} when action in [:delete_message] + ) + + action_fallback(Pleroma.Web.AdminAPI.FallbackController) + + defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.Admin.ChatOperation + + def delete_message(%{assigns: %{user: user}} = conn, %{message_id: id}) do + with {:ok, %Activity{}} <- CommonAPI.delete(id, user) do + ModerationLog.insert_log(%{ + action: "chat_message_delete", + actor: user, + subject_id: id + }) + + json(conn, %{}) + end + end +end diff --git a/lib/pleroma/web/api_spec/operations/admin/chat_operation.ex b/lib/pleroma/web/api_spec/operations/admin/chat_operation.ex new file mode 100644 index 000000000..7045fd7ce --- /dev/null +++ b/lib/pleroma/web/api_spec/operations/admin/chat_operation.ex @@ -0,0 +1,44 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.Admin.ChatOperation do + alias OpenApiSpex.Operation + alias Pleroma.Web.ApiSpec.Schemas.ApiError + alias Pleroma.Web.ApiSpec.Schemas.FlakeID + + import Pleroma.Web.ApiSpec.Helpers + + def open_api_operation(action) do + operation = String.to_existing_atom("#{action}_operation") + apply(__MODULE__, operation, []) + end + + def delete_message_operation do + %Operation{ + tags: ["Admin", "Chats"], + summary: "Delete an individual chat message", + operationId: "AdminAPI.ChatController.delete", + parameters: [id_param(), message_id_param()] ++ admin_api_params(), + security: [%{"oAuth" => ["write:chats"]}], + responses: %{ + 200 => empty_object_response(), + 404 => Operation.response("Not Found", "application/json", ApiError) + } + } + end + + def id_param do + Operation.parameter(:id, :path, FlakeID, "Chat ID", + example: "9umDrYheeY451cQnEe", + required: true + ) + end + + def message_id_param do + Operation.parameter(:message_id, :path, FlakeID, "Chat message ID", + example: "9umDrYheeY451cQnEe", + required: true + ) + end +end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index c6433cc53..e438768ed 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -214,6 +214,8 @@ defmodule Pleroma.Web.Router do get("/media_proxy_caches", MediaProxyCacheController, :index) post("/media_proxy_caches/delete", MediaProxyCacheController, :delete) post("/media_proxy_caches/purge", MediaProxyCacheController, :purge) + + delete("/chats/:id/messages/:message_id", ChatController, :delete_message) end scope "/api/pleroma/emoji", Pleroma.Web.PleromaAPI do diff --git a/test/support/factory.ex b/test/support/factory.ex index 486eda8da..61ca4587c 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -460,4 +460,58 @@ def filter_factory do phrase: "cofe" } end + + def chat_factory(attrs \\ %{}) do + user = attrs[:user] || insert(:user) + recipient = attrs[:recipient] || insert(:user) + + %Pleroma.Chat{ + user_id: user.id, + recipient: recipient.ap_id + } + end + + def chat_message_factory(attrs \\ %{}) do + text = sequence(:text, &"This is :moominmamma: chat message #{&1}") + chat = attrs[:chat] || insert(:chat) + + data = %{ + "type" => "ChatMessage", + "content" => text, + "id" => Pleroma.Web.ActivityPub.Utils.generate_object_id(), + "actor" => User.get_by_id(chat.user_id).ap_id, + "to" => [chat.recipient], + "published" => DateTime.utc_now() |> DateTime.to_iso8601() + } + + %Pleroma.Object{ + data: merge_attributes(data, Map.get(attrs, :data, %{})) + } + end + + def chat_message_activity_factory(attrs \\ %{}) do + chat = attrs[:chat] || insert(:chat) + chat_message = attrs[:chat_message] || insert(:chat_message, chat: chat) + + data_attrs = attrs[:data_attrs] || %{} + attrs = Map.drop(attrs, [:chat, :chat_message, :data_attrs]) + + data = + %{ + "id" => Pleroma.Web.ActivityPub.Utils.generate_activity_id(), + "type" => "Create", + "actor" => chat_message.data["actor"], + "to" => chat_message.data["to"], + "object" => chat_message.data["id"], + "published" => DateTime.utc_now() |> DateTime.to_iso8601() + } + |> Map.merge(data_attrs) + + %Pleroma.Activity{ + data: data, + actor: data["actor"], + recipients: data["to"] + } + |> Map.merge(attrs) + end end diff --git a/test/web/admin_api/controllers/chat_controller_test.exs b/test/web/admin_api/controllers/chat_controller_test.exs new file mode 100644 index 000000000..4527437af --- /dev/null +++ b/test/web/admin_api/controllers/chat_controller_test.exs @@ -0,0 +1,53 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.AdminAPI.ChatControllerTest do + use Pleroma.Web.ConnCase + + import Pleroma.Factory + + alias Pleroma.Activity + alias Pleroma.Config + alias Pleroma.ModerationLog + alias Pleroma.Repo + + setup do + admin = insert(:user, is_admin: true) + token = insert(:oauth_admin_token, user: admin) + + conn = + build_conn() + |> assign(:user, admin) + |> assign(:token, token) + + {:ok, %{admin: admin, token: token, conn: conn}} + end + + describe "DELETE /api/pleroma/admin/chats/:id/messages/:message_id" do + setup do + chat = insert(:chat) + message = insert(:chat_message_activity, chat: chat) + %{chat: chat, message: message} + end + + test "deletes chat message", %{conn: conn, chat: chat, message: message, admin: admin} do + conn + |> delete("/api/pleroma/admin/chats/#{chat.id}/messages/#{message.id}") + |> json_response_and_validate_schema(:ok) + + refute Activity.get_by_id(message.id) + + log_entry = Repo.one(ModerationLog) + + assert ModerationLog.get_log_entry_message(log_entry) == + "@#{admin.nickname} deleted chat message ##{message.id}" + end + + test "returns 404 when the chat message does not exist", %{conn: conn} do + conn = delete(conn, "/api/pleroma/admin/chats/test/messages/test") + + assert json_response_and_validate_schema(conn, :not_found) == %{"error" => "Not found"} + end + end +end From fb0de073439b5e3be823e736b44608e80f1027f1 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 31 Aug 2020 20:23:33 -0500 Subject: [PATCH 09/28] AdminAPI: list chats for a user --- .../controllers/admin_api_controller.ex | 27 +++++++++++++++++++ lib/pleroma/web/router.ex | 3 +++ .../controllers/admin_api_controller_test.exs | 18 +++++++++++++ 3 files changed, 48 insertions(+) diff --git a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex index f5e4d49f9..9b66c2f10 100644 --- a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do use Pleroma.Web, :controller + import Ecto.Query import Pleroma.Web.ControllerHelper, only: [json_response: 3] alias Pleroma.Config @@ -21,6 +22,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do alias Pleroma.Web.AdminAPI.ModerationLogView alias Pleroma.Web.AdminAPI.Search alias Pleroma.Web.Endpoint + alias Pleroma.Web.PleromaAPI alias Pleroma.Web.Router require Logger @@ -68,6 +70,12 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do when action in [:list_user_statuses, :list_instance_statuses] ) + plug( + OAuthScopesPlug, + %{scopes: ["read:chats"], admin: true} + when action in [:list_user_chats] + ) + plug( OAuthScopesPlug, %{scopes: ["read"], admin: true} @@ -256,6 +264,25 @@ def list_user_statuses(%{assigns: %{user: admin}} = conn, %{"nickname" => nickna end end + def list_user_chats(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname} = _params) do + with %User{id: user_id} <- User.get_cached_by_nickname_or_id(nickname, for: admin) do + chats = + from(c in Pleroma.Chat, + where: c.user_id == ^user_id, + order_by: [desc: c.updated_at], + inner_join: u in User, + on: u.ap_id == c.recipient + ) + |> Pleroma.Repo.all() + + conn + |> put_view(PleromaAPI.ChatView) + |> render("index.json", chats: chats) + else + _ -> {:error, :not_found} + end + end + def user_toggle_activation(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do user = User.get_cached_by_nickname(nickname) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index e438768ed..ad3282df4 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -178,6 +178,7 @@ defmodule Pleroma.Web.Router do get("/users", AdminAPIController, :list_users) get("/users/:nickname", AdminAPIController, :user_show) get("/users/:nickname/statuses", AdminAPIController, :list_user_statuses) + get("/users/:nickname/chats", AdminAPIController, :list_user_chats) get("/instances/:instance/statuses", AdminAPIController, :list_instance_statuses) @@ -215,6 +216,8 @@ defmodule Pleroma.Web.Router do post("/media_proxy_caches/delete", MediaProxyCacheController, :delete) post("/media_proxy_caches/purge", MediaProxyCacheController, :purge) + # get("/chats/:id", ChatController, :show) + # get("/chats/:id/messages", ChatController, :messages) delete("/chats/:id/messages/:message_id", ChatController, :delete_message) end diff --git a/test/web/admin_api/controllers/admin_api_controller_test.exs b/test/web/admin_api/controllers/admin_api_controller_test.exs index dbf478edf..cf5637246 100644 --- a/test/web/admin_api/controllers/admin_api_controller_test.exs +++ b/test/web/admin_api/controllers/admin_api_controller_test.exs @@ -1510,6 +1510,24 @@ test "excludes reblogs by default", %{conn: conn, user: user} do end end + describe "GET /api/pleroma/admin/users/:nickname/chats" do + setup do + user = insert(:user) + + insert(:chat, user: user) + insert(:chat, user: user) + insert(:chat, user: user) + + %{user: user} + end + + test "renders user's statuses", %{conn: conn, user: user} do + conn = get(conn, "/api/pleroma/admin/users/#{user.nickname}/chats") + + assert json_response(conn, 200) |> length() == 3 + end + end + describe "GET /api/pleroma/admin/moderation_log" do setup do moderator = insert(:user, is_moderator: true) From c41430b23eaf3fd15b227e66215aa2a4ff31dfdb Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 1 Sep 2020 19:05:24 -0500 Subject: [PATCH 10/28] Refactor with Chat.for_user_query/1 --- lib/pleroma/chat.ex | 12 ++++++++++++ .../admin_api/controllers/admin_api_controller.ex | 8 +------- .../web/pleroma_api/controllers/chat_controller.ex | 9 ++------- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/lib/pleroma/chat.ex b/lib/pleroma/chat.ex index 24a86371e..b38c5c3dd 100644 --- a/lib/pleroma/chat.ex +++ b/lib/pleroma/chat.ex @@ -6,7 +6,9 @@ defmodule Pleroma.Chat do use Ecto.Schema import Ecto.Changeset + import Ecto.Query + alias Pleroma.Chat alias Pleroma.Repo alias Pleroma.User @@ -69,4 +71,14 @@ def bump_or_create(user_id, recipient) do conflict_target: [:user_id, :recipient] ) end + + @spec for_user_query(FlakeId.Ecto.CompatType.t()) :: Ecto.Query.t() + def for_user_query(user_id) do + from(c in Chat, + where: c.user_id == ^user_id, + order_by: [desc: c.updated_at], + inner_join: u in User, + on: u.ap_id == c.recipient + ) + end end diff --git a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex index 9b66c2f10..fccdbabb4 100644 --- a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex @@ -5,7 +5,6 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do use Pleroma.Web, :controller - import Ecto.Query import Pleroma.Web.ControllerHelper, only: [json_response: 3] alias Pleroma.Config @@ -267,12 +266,7 @@ def list_user_statuses(%{assigns: %{user: admin}} = conn, %{"nickname" => nickna def list_user_chats(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname} = _params) do with %User{id: user_id} <- User.get_cached_by_nickname_or_id(nickname, for: admin) do chats = - from(c in Pleroma.Chat, - where: c.user_id == ^user_id, - order_by: [desc: c.updated_at], - inner_join: u in User, - on: u.ap_id == c.recipient - ) + Pleroma.Chat.for_user_query(user_id) |> Pleroma.Repo.all() conn diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex index 1f2e953f7..27c9a2e0f 100644 --- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex @@ -146,13 +146,8 @@ def index(%{assigns: %{user: %{id: user_id} = user}} = conn, _params) do blocked_ap_ids = User.blocked_users_ap_ids(user) chats = - from(c in Chat, - where: c.user_id == ^user_id, - where: c.recipient not in ^blocked_ap_ids, - order_by: [desc: c.updated_at], - inner_join: u in User, - on: u.ap_id == c.recipient - ) + Chat.for_user_query(user_id) + |> where([c], c.recipient not in ^blocked_ap_ids) |> Repo.all() conn From f13b52a703d5c60cf12b2fff69f458e5c467c783 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 1 Sep 2020 19:39:34 -0500 Subject: [PATCH 11/28] AdminAPI: list messages in a chat --- .../admin_api/controllers/chat_controller.ex | 27 ++++++++++ .../operations/admin/chat_operation.ex | 26 ++++++++- lib/pleroma/web/router.ex | 2 +- .../controllers/chat_controller_test.exs | 54 +++++++++++++++++++ 4 files changed, 107 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/admin_api/controllers/chat_controller.ex b/lib/pleroma/web/admin_api/controllers/chat_controller.ex index bcce824d2..b423188d7 100644 --- a/lib/pleroma/web/admin_api/controllers/chat_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/chat_controller.ex @@ -6,14 +6,23 @@ defmodule Pleroma.Web.AdminAPI.ChatController do use Pleroma.Web, :controller alias Pleroma.Activity + alias Pleroma.Chat + alias Pleroma.Chat.MessageReference alias Pleroma.ModerationLog + alias Pleroma.Pagination alias Pleroma.Plugs.OAuthScopesPlug alias Pleroma.Web.CommonAPI + alias Pleroma.Web.PleromaAPI.Chat.MessageReferenceView require Logger plug(Pleroma.Web.ApiSpec.CastAndValidate) + plug( + OAuthScopesPlug, + %{scopes: ["read:chats"], admin: true} when action in [:messages] + ) + plug( OAuthScopesPlug, %{scopes: ["write:chats"], admin: true} when action in [:delete_message] @@ -34,4 +43,22 @@ def delete_message(%{assigns: %{user: user}} = conn, %{message_id: id}) do json(conn, %{}) end end + + def messages(conn, %{id: id} = params) do + with %Chat{} = chat <- Chat.get_by_id(id) do + cm_refs = + chat + |> MessageReference.for_chat_query() + |> Pagination.fetch_paginated(params) + + conn + |> put_view(MessageReferenceView) + |> render("index.json", chat_message_references: cm_refs) + else + _ -> + conn + |> put_status(:not_found) + |> json(%{error: "not found"}) + end + end end diff --git a/lib/pleroma/web/api_spec/operations/admin/chat_operation.ex b/lib/pleroma/web/api_spec/operations/admin/chat_operation.ex index 7045fd7ce..a382bd35a 100644 --- a/lib/pleroma/web/api_spec/operations/admin/chat_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/chat_operation.ex @@ -16,7 +16,7 @@ def open_api_operation(action) do def delete_message_operation do %Operation{ - tags: ["Admin", "Chats"], + tags: ["admin", "chat"], summary: "Delete an individual chat message", operationId: "AdminAPI.ChatController.delete", parameters: [id_param(), message_id_param()] ++ admin_api_params(), @@ -28,6 +28,30 @@ def delete_message_operation do } end + def messages_operation do + %Operation{ + tags: ["admin", "chat"], + summary: "Get the most recent messages of the chat", + operationId: "AdminAPI.ChatController.messages", + parameters: + [Operation.parameter(:id, :path, :string, "The ID of the Chat")] ++ + pagination_params(), + responses: %{ + 200 => + Operation.response( + "The messages in the chat", + "application/json", + Pleroma.Web.ApiSpec.ChatOperation.chat_messages_response() + ) + }, + security: [ + %{ + "oAuth" => ["read:chats"] + } + ] + } + end + def id_param do Operation.parameter(:id, :path, FlakeID, "Chat ID", example: "9umDrYheeY451cQnEe", diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index ad3282df4..02836114a 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -217,7 +217,7 @@ defmodule Pleroma.Web.Router do post("/media_proxy_caches/purge", MediaProxyCacheController, :purge) # get("/chats/:id", ChatController, :show) - # get("/chats/:id/messages", ChatController, :messages) + get("/chats/:id/messages", ChatController, :messages) delete("/chats/:id/messages/:message_id", ChatController, :delete_message) end diff --git a/test/web/admin_api/controllers/chat_controller_test.exs b/test/web/admin_api/controllers/chat_controller_test.exs index 4527437af..f61e2a1fa 100644 --- a/test/web/admin_api/controllers/chat_controller_test.exs +++ b/test/web/admin_api/controllers/chat_controller_test.exs @@ -8,9 +8,11 @@ defmodule Pleroma.Web.AdminAPI.ChatControllerTest do import Pleroma.Factory alias Pleroma.Activity + alias Pleroma.Chat alias Pleroma.Config alias Pleroma.ModerationLog alias Pleroma.Repo + alias Pleroma.Web.CommonAPI setup do admin = insert(:user, is_admin: true) @@ -50,4 +52,56 @@ test "returns 404 when the chat message does not exist", %{conn: conn} do assert json_response_and_validate_schema(conn, :not_found) == %{"error" => "Not found"} end end + + describe "GET /api/pleroma/admin/chats/:id/messages" do + test "it paginates", %{conn: conn} do + user = insert(:user) + recipient = insert(:user) + + Enum.each(1..30, fn _ -> + {:ok, _} = CommonAPI.post_chat_message(user, recipient, "hey") + end) + + chat = Chat.get(user.id, recipient.ap_id) + + result = + conn + |> get("/api/pleroma/admin/chats/#{chat.id}/messages") + |> json_response_and_validate_schema(200) + + assert length(result) == 20 + + result = + conn + |> get("/api/pleroma/admin/chats/#{chat.id}/messages?max_id=#{List.last(result)["id"]}") + |> json_response_and_validate_schema(200) + + assert length(result) == 10 + end + + test "it returns the messages for a given chat", %{conn: conn} do + user = insert(:user) + other_user = insert(:user) + third_user = insert(:user) + + {:ok, _} = CommonAPI.post_chat_message(user, other_user, "hey") + {:ok, _} = CommonAPI.post_chat_message(user, third_user, "hey") + {:ok, _} = CommonAPI.post_chat_message(user, other_user, "how are you?") + {:ok, _} = CommonAPI.post_chat_message(other_user, user, "fine, how about you?") + + chat = Chat.get(user.id, other_user.ap_id) + + result = + conn + |> get("/api/pleroma/admin/chats/#{chat.id}/messages") + |> json_response_and_validate_schema(200) + + result + |> Enum.each(fn message -> + assert message["chat_id"] == chat.id |> to_string() + end) + + assert length(result) == 3 + end + end end From 9dd0b23da424c380a37897d8bf69ab241efa6f91 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 1 Sep 2020 19:49:46 -0500 Subject: [PATCH 12/28] AdminAPI: show chat --- .../admin_api/controllers/chat_controller.ex | 11 ++++++- .../operations/admin/chat_operation.ex | 32 +++++++++++++++++++ lib/pleroma/web/router.ex | 2 +- .../controllers/chat_controller_test.exs | 16 ++++++++++ 4 files changed, 59 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/admin_api/controllers/chat_controller.ex b/lib/pleroma/web/admin_api/controllers/chat_controller.ex index b423188d7..ac362c430 100644 --- a/lib/pleroma/web/admin_api/controllers/chat_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/chat_controller.ex @@ -13,6 +13,7 @@ defmodule Pleroma.Web.AdminAPI.ChatController do alias Pleroma.Plugs.OAuthScopesPlug alias Pleroma.Web.CommonAPI alias Pleroma.Web.PleromaAPI.Chat.MessageReferenceView + alias Pleroma.Web.PleromaAPI.ChatView require Logger @@ -20,7 +21,7 @@ defmodule Pleroma.Web.AdminAPI.ChatController do plug( OAuthScopesPlug, - %{scopes: ["read:chats"], admin: true} when action in [:messages] + %{scopes: ["read:chats"], admin: true} when action in [:show, :messages] ) plug( @@ -61,4 +62,12 @@ def messages(conn, %{id: id} = params) do |> json(%{error: "not found"}) end end + + def show(conn, %{id: id}) do + with %Chat{} = chat <- Chat.get_by_id(id) do + conn + |> put_view(ChatView) + |> render("show.json", chat: chat) + end + end end diff --git a/lib/pleroma/web/api_spec/operations/admin/chat_operation.ex b/lib/pleroma/web/api_spec/operations/admin/chat_operation.ex index a382bd35a..3550d531e 100644 --- a/lib/pleroma/web/api_spec/operations/admin/chat_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/chat_operation.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.ChatOperation do alias OpenApiSpex.Operation alias Pleroma.Web.ApiSpec.Schemas.ApiError + alias Pleroma.Web.ApiSpec.Schemas.Chat alias Pleroma.Web.ApiSpec.Schemas.FlakeID import Pleroma.Web.ApiSpec.Helpers @@ -52,6 +53,37 @@ def messages_operation do } end + def show_operation do + %Operation{ + tags: ["chat"], + summary: "Create a chat", + operationId: "AdminAPI.ChatController.show", + parameters: [ + Operation.parameter( + :id, + :path, + :string, + "The id of the chat", + required: true, + example: "1234" + ) + ], + responses: %{ + 200 => + Operation.response( + "The existing chat", + "application/json", + Chat + ) + }, + security: [ + %{ + "oAuth" => ["read"] + } + ] + } + end + def id_param do Operation.parameter(:id, :path, FlakeID, "Chat ID", example: "9umDrYheeY451cQnEe", diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 02836114a..e4440d442 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -216,7 +216,7 @@ defmodule Pleroma.Web.Router do post("/media_proxy_caches/delete", MediaProxyCacheController, :delete) post("/media_proxy_caches/purge", MediaProxyCacheController, :purge) - # get("/chats/:id", ChatController, :show) + get("/chats/:id", ChatController, :show) get("/chats/:id/messages", ChatController, :messages) delete("/chats/:id/messages/:message_id", ChatController, :delete_message) end diff --git a/test/web/admin_api/controllers/chat_controller_test.exs b/test/web/admin_api/controllers/chat_controller_test.exs index f61e2a1fa..63c195b99 100644 --- a/test/web/admin_api/controllers/chat_controller_test.exs +++ b/test/web/admin_api/controllers/chat_controller_test.exs @@ -104,4 +104,20 @@ test "it returns the messages for a given chat", %{conn: conn} do assert length(result) == 3 end end + + describe "GET /api/pleroma/admin/chats/:id" do + test "it returns a chat", %{conn: conn} do + user = insert(:user) + other_user = insert(:user) + + {:ok, chat} = Chat.get_or_create(user.id, other_user.ap_id) + + result = + conn + |> get("/api/pleroma/admin/chats/#{chat.id}") + |> json_response_and_validate_schema(200) + + assert result["id"] == to_string(chat.id) + end + end end From 02d70228b566d5de2cbdd6d1f9958caf2db173f1 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 1 Sep 2020 20:40:36 -0500 Subject: [PATCH 13/28] AdminAPI: fix delete chat message --- .../admin_api/controllers/chat_controller.ex | 20 ++++++++-- .../operations/admin/chat_operation.ex | 40 +++++++++---------- .../controllers/chat_controller_test.exs | 39 ++++++++++-------- 3 files changed, 56 insertions(+), 43 deletions(-) diff --git a/lib/pleroma/web/admin_api/controllers/chat_controller.ex b/lib/pleroma/web/admin_api/controllers/chat_controller.ex index ac362c430..61d45b970 100644 --- a/lib/pleroma/web/admin_api/controllers/chat_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/chat_controller.ex @@ -33,15 +33,27 @@ defmodule Pleroma.Web.AdminAPI.ChatController do defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.Admin.ChatOperation - def delete_message(%{assigns: %{user: user}} = conn, %{message_id: id}) do - with {:ok, %Activity{}} <- CommonAPI.delete(id, user) do + def delete_message(%{assigns: %{user: user}} = conn, %{ + message_id: message_id, + id: chat_id + }) do + with %MessageReference{object: %{data: %{"id" => object_ap_id}}} = cm_ref <- + MessageReference.get_by_id(message_id), + ^chat_id <- to_string(cm_ref.chat_id), + %Activity{id: activity_id} <- Activity.get_create_by_object_ap_id(object_ap_id), + {:ok, _} <- CommonAPI.delete(activity_id, user) do ModerationLog.insert_log(%{ action: "chat_message_delete", actor: user, - subject_id: id + subject_id: message_id }) - json(conn, %{}) + conn + |> put_view(MessageReferenceView) + |> render("show.json", chat_message_reference: cm_ref) + else + _e -> + {:error, :could_not_delete} end end diff --git a/lib/pleroma/web/api_spec/operations/admin/chat_operation.ex b/lib/pleroma/web/api_spec/operations/admin/chat_operation.ex index 3550d531e..d3e5dfc1c 100644 --- a/lib/pleroma/web/api_spec/operations/admin/chat_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/chat_operation.ex @@ -4,9 +4,8 @@ defmodule Pleroma.Web.ApiSpec.Admin.ChatOperation do alias OpenApiSpex.Operation - alias Pleroma.Web.ApiSpec.Schemas.ApiError alias Pleroma.Web.ApiSpec.Schemas.Chat - alias Pleroma.Web.ApiSpec.Schemas.FlakeID + alias Pleroma.Web.ApiSpec.Schemas.ChatMessage import Pleroma.Web.ApiSpec.Helpers @@ -19,13 +18,24 @@ def delete_message_operation do %Operation{ tags: ["admin", "chat"], summary: "Delete an individual chat message", - operationId: "AdminAPI.ChatController.delete", - parameters: [id_param(), message_id_param()] ++ admin_api_params(), - security: [%{"oAuth" => ["write:chats"]}], + operationId: "AdminAPI.ChatController.delete_message", + parameters: [ + Operation.parameter(:id, :path, :string, "The ID of the Chat"), + Operation.parameter(:message_id, :path, :string, "The ID of the message") + ], responses: %{ - 200 => empty_object_response(), - 404 => Operation.response("Not Found", "application/json", ApiError) - } + 200 => + Operation.response( + "The deleted ChatMessage", + "application/json", + ChatMessage + ) + }, + security: [ + %{ + "oAuth" => ["write:chats"] + } + ] } end @@ -83,18 +93,4 @@ def show_operation do ] } end - - def id_param do - Operation.parameter(:id, :path, FlakeID, "Chat ID", - example: "9umDrYheeY451cQnEe", - required: true - ) - end - - def message_id_param do - Operation.parameter(:message_id, :path, FlakeID, "Chat message ID", - example: "9umDrYheeY451cQnEe", - required: true - ) - end end diff --git a/test/web/admin_api/controllers/chat_controller_test.exs b/test/web/admin_api/controllers/chat_controller_test.exs index 63c195b99..9393dd49b 100644 --- a/test/web/admin_api/controllers/chat_controller_test.exs +++ b/test/web/admin_api/controllers/chat_controller_test.exs @@ -7,9 +7,10 @@ defmodule Pleroma.Web.AdminAPI.ChatControllerTest do import Pleroma.Factory - alias Pleroma.Activity alias Pleroma.Chat + alias Pleroma.Chat.MessageReference alias Pleroma.Config + alias Pleroma.Object alias Pleroma.ModerationLog alias Pleroma.Repo alias Pleroma.Web.CommonAPI @@ -27,29 +28,33 @@ defmodule Pleroma.Web.AdminAPI.ChatControllerTest do end describe "DELETE /api/pleroma/admin/chats/:id/messages/:message_id" do - setup do - chat = insert(:chat) - message = insert(:chat_message_activity, chat: chat) - %{chat: chat, message: message} - end + test "it deletes a message from the chat", %{conn: conn, admin: admin} do + user = insert(:user) + recipient = insert(:user) - test "deletes chat message", %{conn: conn, chat: chat, message: message, admin: admin} do - conn - |> delete("/api/pleroma/admin/chats/#{chat.id}/messages/#{message.id}") - |> json_response_and_validate_schema(:ok) + {:ok, message} = + CommonAPI.post_chat_message(user, recipient, "Hello darkness my old friend") - refute Activity.get_by_id(message.id) + object = Object.normalize(message, false) + + chat = Chat.get(user.id, recipient.ap_id) + + cm_ref = MessageReference.for_chat_and_object(chat, object) + + result = + conn + |> put_req_header("content-type", "application/json") + |> delete("/api/pleroma/admin/chats/#{chat.id}/messages/#{cm_ref.id}") + |> json_response_and_validate_schema(200) log_entry = Repo.one(ModerationLog) assert ModerationLog.get_log_entry_message(log_entry) == - "@#{admin.nickname} deleted chat message ##{message.id}" - end + "@#{admin.nickname} deleted chat message ##{cm_ref.id}" - test "returns 404 when the chat message does not exist", %{conn: conn} do - conn = delete(conn, "/api/pleroma/admin/chats/test/messages/test") - - assert json_response_and_validate_schema(conn, :not_found) == %{"error" => "Not found"} + assert result["id"] == cm_ref.id + refute MessageReference.get_by_id(cm_ref.id) + assert %{data: %{"type" => "Tombstone"}} = Object.get_by_id(object.id) end end From c361df11b4e31bc6b369a4feebdbaa82987c2eec Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 1 Sep 2020 20:56:42 -0500 Subject: [PATCH 14/28] Docs: AdminAPI chat moderation --- docs/API/admin_api.md | 111 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) diff --git a/docs/API/admin_api.md b/docs/API/admin_api.md index c0ea074f0..7bdbd17aa 100644 --- a/docs/API/admin_api.md +++ b/docs/API/admin_api.md @@ -1334,3 +1334,114 @@ Loads json generated from `config/descriptions.exs`. { } ``` + +## GET /api/pleroma/admin/users/:nickname/chats + +### List a user's chats + +- Params: None + +- Response: + +```json +[ + { + "account": { + "id": "someflakeid", + "username": "somenick", + ... + }, + "id" : "1", + "unread" : 2, + "last_message" : {...}, // The last message in that chat + "updated_at": "2020-04-21T15:11:46.000Z" + } +] +``` + +## GET /api/pleroma/admin/chats/:chat_id + +### View a single chat + +- Params: None + +- Response: + +```json +{ + "account": { + "id": "someflakeid", + "username": "somenick", + ... + }, + "id" : "1", + "unread" : 2, + "last_message" : {...}, // The last message in that chat + "updated_at": "2020-04-21T15:11:46.000Z" +} +``` + +## GET /api/pleroma/admin/chats/:chat_id/messages + +### List the messages in a chat + +- Params: None + +- Response: + +```json +[ + { + "account_id": "someflakeid", + "chat_id": "1", + "content": "Check this out :firefox:", + "created_at": "2020-04-21T15:11:46.000Z", + "emojis": [ + { + "shortcode": "firefox", + "static_url": "https://dontbulling.me/emoji/Firefox.gif", + "url": "https://dontbulling.me/emoji/Firefox.gif", + "visible_in_picker": false + } + ], + "id": "13", + "unread": true + }, + { + "account_id": "someflakeid", + "chat_id": "1", + "content": "Whats' up?", + "created_at": "2020-04-21T15:06:45.000Z", + "emojis": [], + "id": "12", + "unread": false + } +] +``` + +## DELETE /api/pleroma/admin/chats/:chat_id/messages/:message_id + +### Delete a single message + +- Params: None + +- Response: + +```json +{ + "account_id": "someflakeid", + "chat_id": "1", + "content": "Check this out :firefox:", + "created_at": "2020-04-21T15:11:46.000Z", + "emojis": [ + { + "shortcode": "firefox", + "static_url": "https://dontbulling.me/emoji/Firefox.gif", + "url": "https://dontbulling.me/emoji/Firefox.gif", + "visible_in_picker": false + } + ], + "id": "13", + "unread": false +} +``` From 67726453f85eb5bb51bf82e7decf23a4f1d184af Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 1 Sep 2020 21:12:21 -0500 Subject: [PATCH 15/28] Credo fix --- test/web/admin_api/controllers/chat_controller_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/web/admin_api/controllers/chat_controller_test.exs b/test/web/admin_api/controllers/chat_controller_test.exs index 9393dd49b..bca9d440d 100644 --- a/test/web/admin_api/controllers/chat_controller_test.exs +++ b/test/web/admin_api/controllers/chat_controller_test.exs @@ -10,8 +10,8 @@ defmodule Pleroma.Web.AdminAPI.ChatControllerTest do alias Pleroma.Chat alias Pleroma.Chat.MessageReference alias Pleroma.Config - alias Pleroma.Object alias Pleroma.ModerationLog + alias Pleroma.Object alias Pleroma.Repo alias Pleroma.Web.CommonAPI From e229536e5cca65d811f85d25c86bf3c92b3d8c45 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 10 Sep 2020 01:44:32 -0500 Subject: [PATCH 16/28] Chat Moderation: use explicit `sender` and `recipient` fields --- docs/API/admin_api.md | 14 +++++++-- .../controllers/admin_api_controller.ex | 5 +--- .../admin_api/controllers/chat_controller.ex | 4 +-- lib/pleroma/web/admin_api/views/chat_view.ex | 30 +++++++++++++++++++ .../controllers/chat_controller_test.exs | 3 ++ 5 files changed, 48 insertions(+), 8 deletions(-) create mode 100644 lib/pleroma/web/admin_api/views/chat_view.ex diff --git a/docs/API/admin_api.md b/docs/API/admin_api.md index 7bdbd17aa..eadb455ee 100644 --- a/docs/API/admin_api.md +++ b/docs/API/admin_api.md @@ -1346,7 +1346,12 @@ Loads json generated from `config/descriptions.exs`. ```json [ { - "account": { + "sender": { + "id": "someflakeid", + "username": "somenick", + ... + }, + "receiver": { "id": "someflakeid", "username": "somenick", ... @@ -1369,7 +1374,12 @@ Loads json generated from `config/descriptions.exs`. ```json { - "account": { + "sender": { + "id": "someflakeid", + "username": "somenick", + ... + }, + "receiver": { "id": "someflakeid", "username": "somenick", ... diff --git a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex index fccdbabb4..d5713c3dd 100644 --- a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex @@ -21,11 +21,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do alias Pleroma.Web.AdminAPI.ModerationLogView alias Pleroma.Web.AdminAPI.Search alias Pleroma.Web.Endpoint - alias Pleroma.Web.PleromaAPI alias Pleroma.Web.Router - require Logger - @users_page_size 50 plug( @@ -270,7 +267,7 @@ def list_user_chats(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname} |> Pleroma.Repo.all() conn - |> put_view(PleromaAPI.ChatView) + |> put_view(AdminAPI.ChatView) |> render("index.json", chats: chats) else _ -> {:error, :not_found} diff --git a/lib/pleroma/web/admin_api/controllers/chat_controller.ex b/lib/pleroma/web/admin_api/controllers/chat_controller.ex index 61d45b970..967600d69 100644 --- a/lib/pleroma/web/admin_api/controllers/chat_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/chat_controller.ex @@ -11,9 +11,9 @@ defmodule Pleroma.Web.AdminAPI.ChatController do alias Pleroma.ModerationLog alias Pleroma.Pagination alias Pleroma.Plugs.OAuthScopesPlug + alias Pleroma.Web.AdminAPI alias Pleroma.Web.CommonAPI alias Pleroma.Web.PleromaAPI.Chat.MessageReferenceView - alias Pleroma.Web.PleromaAPI.ChatView require Logger @@ -78,7 +78,7 @@ def messages(conn, %{id: id} = params) do def show(conn, %{id: id}) do with %Chat{} = chat <- Chat.get_by_id(id) do conn - |> put_view(ChatView) + |> put_view(AdminAPI.ChatView) |> render("show.json", chat: chat) end end diff --git a/lib/pleroma/web/admin_api/views/chat_view.ex b/lib/pleroma/web/admin_api/views/chat_view.ex new file mode 100644 index 000000000..847df1423 --- /dev/null +++ b/lib/pleroma/web/admin_api/views/chat_view.ex @@ -0,0 +1,30 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.AdminAPI.ChatView do + use Pleroma.Web, :view + + alias Pleroma.Chat + alias Pleroma.User + alias Pleroma.Web.MastodonAPI + alias Pleroma.Web.PleromaAPI + + def render("index.json", %{chats: chats} = opts) do + render_many(chats, __MODULE__, "show.json", Map.delete(opts, :chats)) + end + + def render("show.json", %{chat: %Chat{user_id: user_id}} = opts) do + user = User.get_by_id(user_id) + sender = MastodonAPI.AccountView.render("show.json", user: user, skip_visibility_check: true) + + serialized_chat = PleromaAPI.ChatView.render("show.json", opts) + + serialized_chat + |> Map.put(:sender, sender) + |> Map.put(:receiver, serialized_chat[:account]) + |> Map.delete(:account) + end + + def render(view, opts), do: PleromaAPI.ChatView.render(view, opts) +end diff --git a/test/web/admin_api/controllers/chat_controller_test.exs b/test/web/admin_api/controllers/chat_controller_test.exs index bca9d440d..840f18aa2 100644 --- a/test/web/admin_api/controllers/chat_controller_test.exs +++ b/test/web/admin_api/controllers/chat_controller_test.exs @@ -123,6 +123,9 @@ test "it returns a chat", %{conn: conn} do |> json_response_and_validate_schema(200) assert result["id"] == to_string(chat.id) + assert %{} = result["sender"] + assert %{} = result["receiver"] + refute result["account"] end end end From dfb831ca39db3098d6d585448a6ff8e938e51e8c Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 11 Sep 2020 14:00:34 -0500 Subject: [PATCH 17/28] Chat moderation: add tests for unauthorized access --- docs/API/admin_api.md | 2 +- .../controllers/admin_api_controller_test.exs | 29 +++++++ .../controllers/chat_controller_test.exs | 80 ++++++++++++++++++- 3 files changed, 109 insertions(+), 2 deletions(-) diff --git a/docs/API/admin_api.md b/docs/API/admin_api.md index eadb455ee..bc96abbf0 100644 --- a/docs/API/admin_api.md +++ b/docs/API/admin_api.md @@ -1395,7 +1395,7 @@ Loads json generated from `config/descriptions.exs`. ### List the messages in a chat -- Params: None +- Params: `max_id`, `min_id` - Response: diff --git a/test/web/admin_api/controllers/admin_api_controller_test.exs b/test/web/admin_api/controllers/admin_api_controller_test.exs index cf5637246..dbeeb7f3d 100644 --- a/test/web/admin_api/controllers/admin_api_controller_test.exs +++ b/test/web/admin_api/controllers/admin_api_controller_test.exs @@ -1528,6 +1528,35 @@ test "renders user's statuses", %{conn: conn, user: user} do end end + describe "GET /api/pleroma/admin/users/:nickname/chats unauthorized" do + setup do + user = insert(:user) + insert(:chat, user: user) + %{conn: conn} = oauth_access(["read:chats"]) + %{conn: conn, user: user} + end + + test "returns 403", %{conn: conn, user: user} do + conn + |> get("/api/pleroma/admin/users/#{user.nickname}/chats") + |> json_response(403) + end + end + + describe "GET /api/pleroma/admin/users/:nickname/chats unauthenticated" do + setup do + user = insert(:user) + insert(:chat, user: user) + %{conn: build_conn(), user: user} + end + + test "returns 403", %{conn: conn, user: user} do + conn + |> get("/api/pleroma/admin/users/#{user.nickname}/chats") + |> json_response(403) + end + end + describe "GET /api/pleroma/admin/moderation_log" do setup do moderator = insert(:user, is_moderator: true) diff --git a/test/web/admin_api/controllers/chat_controller_test.exs b/test/web/admin_api/controllers/chat_controller_test.exs index 840f18aa2..ccca3521a 100644 --- a/test/web/admin_api/controllers/chat_controller_test.exs +++ b/test/web/admin_api/controllers/chat_controller_test.exs @@ -15,7 +15,7 @@ defmodule Pleroma.Web.AdminAPI.ChatControllerTest do alias Pleroma.Repo alias Pleroma.Web.CommonAPI - setup do + defp admin_setup do admin = insert(:user, is_admin: true) token = insert(:oauth_admin_token, user: admin) @@ -28,6 +28,8 @@ defmodule Pleroma.Web.AdminAPI.ChatControllerTest do end describe "DELETE /api/pleroma/admin/chats/:id/messages/:message_id" do + setup do: admin_setup() + test "it deletes a message from the chat", %{conn: conn, admin: admin} do user = insert(:user) recipient = insert(:user) @@ -59,6 +61,8 @@ test "it deletes a message from the chat", %{conn: conn, admin: admin} do end describe "GET /api/pleroma/admin/chats/:id/messages" do + setup do: admin_setup() + test "it paginates", %{conn: conn} do user = insert(:user) recipient = insert(:user) @@ -111,6 +115,8 @@ test "it returns the messages for a given chat", %{conn: conn} do end describe "GET /api/pleroma/admin/chats/:id" do + setup do: admin_setup() + test "it returns a chat", %{conn: conn} do user = insert(:user) other_user = insert(:user) @@ -128,4 +134,76 @@ test "it returns a chat", %{conn: conn} do refute result["account"] end end + + describe "unauthorized chat moderation" do + setup do + user = insert(:user) + recipient = insert(:user) + + {:ok, message} = CommonAPI.post_chat_message(user, recipient, "Yo") + object = Object.normalize(message, false) + chat = Chat.get(user.id, recipient.ap_id) + cm_ref = MessageReference.for_chat_and_object(chat, object) + + %{conn: conn} = oauth_access(["read:chats", "write:chats"]) + %{conn: conn, chat: chat, cm_ref: cm_ref} + end + + test "DELETE /api/pleroma/admin/chats/:id/messages/:message_id", %{conn: conn, chat: chat, cm_ref: cm_ref} do + conn + |> put_req_header("content-type", "application/json") + |> delete("/api/pleroma/admin/chats/#{chat.id}/messages/#{cm_ref.id}") + |> json_response(403) + + assert MessageReference.get_by_id(cm_ref.id) == cm_ref + end + + test "GET /api/pleroma/admin/chats/:id/messages", %{conn: conn, chat: chat} do + conn + |> get("/api/pleroma/admin/chats/#{chat.id}/messages") + |> json_response(403) + end + + test "GET /api/pleroma/admin/chats/:id", %{conn: conn, chat: chat} do + conn + |> get("/api/pleroma/admin/chats/#{chat.id}") + |> json_response(403) + end + end + + describe "unauthenticated chat moderation" do + setup do + user = insert(:user) + recipient = insert(:user) + + {:ok, message} = CommonAPI.post_chat_message(user, recipient, "Yo") + object = Object.normalize(message, false) + chat = Chat.get(user.id, recipient.ap_id) + cm_ref = MessageReference.for_chat_and_object(chat, object) + + %{conn: build_conn(), chat: chat, cm_ref: cm_ref} + end + + test "DELETE /api/pleroma/admin/chats/:id/messages/:message_id", %{conn: conn, chat: chat, cm_ref: cm_ref} do + conn + |> put_req_header("content-type", "application/json") + |> delete("/api/pleroma/admin/chats/#{chat.id}/messages/#{cm_ref.id}") + |> json_response(403) + + assert MessageReference.get_by_id(cm_ref.id) == cm_ref + end + + test "GET /api/pleroma/admin/chats/:id/messages", %{conn: conn, chat: chat} do + conn + |> get("/api/pleroma/admin/chats/#{chat.id}/messages") + |> json_response(403) + end + + test "GET /api/pleroma/admin/chats/:id", %{conn: conn, chat: chat} do + conn + |> get("/api/pleroma/admin/chats/#{chat.id}") + |> json_response(403) + end + end + end From bc86d0a906e58becb94c5a73552f90abbe494c28 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 11 Sep 2020 14:29:56 -0500 Subject: [PATCH 18/28] Chat moderation: fix formatting --- .../admin_api/controllers/chat_controller_test.exs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/test/web/admin_api/controllers/chat_controller_test.exs b/test/web/admin_api/controllers/chat_controller_test.exs index ccca3521a..e81484ce6 100644 --- a/test/web/admin_api/controllers/chat_controller_test.exs +++ b/test/web/admin_api/controllers/chat_controller_test.exs @@ -149,7 +149,11 @@ test "it returns a chat", %{conn: conn} do %{conn: conn, chat: chat, cm_ref: cm_ref} end - test "DELETE /api/pleroma/admin/chats/:id/messages/:message_id", %{conn: conn, chat: chat, cm_ref: cm_ref} do + test "DELETE /api/pleroma/admin/chats/:id/messages/:message_id", %{ + conn: conn, + chat: chat, + cm_ref: cm_ref + } do conn |> put_req_header("content-type", "application/json") |> delete("/api/pleroma/admin/chats/#{chat.id}/messages/#{cm_ref.id}") @@ -184,7 +188,11 @@ test "GET /api/pleroma/admin/chats/:id", %{conn: conn, chat: chat} do %{conn: build_conn(), chat: chat, cm_ref: cm_ref} end - test "DELETE /api/pleroma/admin/chats/:id/messages/:message_id", %{conn: conn, chat: chat, cm_ref: cm_ref} do + test "DELETE /api/pleroma/admin/chats/:id/messages/:message_id", %{ + conn: conn, + chat: chat, + cm_ref: cm_ref + } do conn |> put_req_header("content-type", "application/json") |> delete("/api/pleroma/admin/chats/#{chat.id}/messages/#{cm_ref.id}") @@ -205,5 +213,4 @@ test "GET /api/pleroma/admin/chats/:id", %{conn: conn, chat: chat} do |> json_response(403) end end - end From 3e53ab4e98e6294f593f2185998f555ccd6fee73 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Mon, 14 Sep 2020 14:08:12 +0300 Subject: [PATCH 19/28] added notification constraints --- .../migration_helper/notification_backfill.ex | 15 +++++----- lib/pleroma/repo.ex | 9 ++++-- ...8_delete_notification_without_activity.exs | 30 +++++++++++++++++++ ...914105800_add_notification_constraints.exs | 23 ++++++++++++++ 4 files changed, 67 insertions(+), 10 deletions(-) create mode 100644 priv/repo/migrations/20200914105638_delete_notification_without_activity.exs create mode 100644 priv/repo/migrations/20200914105800_add_notification_constraints.exs diff --git a/lib/pleroma/migration_helper/notification_backfill.ex b/lib/pleroma/migration_helper/notification_backfill.ex index d260e62ca..24f4733fe 100644 --- a/lib/pleroma/migration_helper/notification_backfill.ex +++ b/lib/pleroma/migration_helper/notification_backfill.ex @@ -19,13 +19,13 @@ def fill_in_notification_types do query |> Repo.chunk_stream(100) |> Enum.each(fn notification -> - type = - notification.activity - |> type_from_activity() + if notification.activity do + type = type_from_activity(notification.activity) - notification - |> Ecto.Changeset.change(%{type: type}) - |> Repo.update() + notification + |> Ecto.Changeset.change(%{type: type}) + |> Repo.update() + end end) end @@ -72,8 +72,7 @@ defp type_from_activity(%{data: %{"type" => type}} = activity) do "pleroma:emoji_reaction" "Create" -> - activity - |> type_from_activity_object() + type_from_activity_object(activity) t -> raise "No notification type for activity type #{t}" diff --git a/lib/pleroma/repo.ex b/lib/pleroma/repo.ex index f317e4d58..a75610879 100644 --- a/lib/pleroma/repo.ex +++ b/lib/pleroma/repo.ex @@ -49,7 +49,7 @@ def get_assoc(resource, association) do end end - def chunk_stream(query, chunk_size) do + def chunk_stream(query, chunk_size, returns_as \\ :one) do # We don't actually need start and end funcitons of resource streaming, # but it seems to be the only way to not fetch records one-by-one and # have individual records be the elements of the stream, instead of @@ -69,7 +69,12 @@ def chunk_stream(query, chunk_size) do records -> last_id = List.last(records).id - {records, last_id} + + if returns_as == :one do + {records, last_id} + else + {[records], last_id} + end end end, fn _ -> :ok end diff --git a/priv/repo/migrations/20200914105638_delete_notification_without_activity.exs b/priv/repo/migrations/20200914105638_delete_notification_without_activity.exs new file mode 100644 index 000000000..f5b339101 --- /dev/null +++ b/priv/repo/migrations/20200914105638_delete_notification_without_activity.exs @@ -0,0 +1,30 @@ +defmodule Pleroma.Repo.Migrations.DeleteNotificationWithoutActivity do + use Ecto.Migration + + import Ecto.Query + alias Pleroma.Repo + + def up do + from( + q in Pleroma.Notification, + left_join: c in assoc(q, :activity), + select: %{id: type(q.id, :integer)}, + where: is_nil(c.id) + ) + |> Repo.chunk_stream(1_000, :bacthes) + |> Stream.each(fn records -> + notification_ids = Enum.map(records, fn %{id: id} -> id end) + + Repo.delete_all( + from(n in "notifications", + where: n.id in ^notification_ids + ) + ) + end) + |> Stream.run() + end + + def down do + :ok + end +end diff --git a/priv/repo/migrations/20200914105800_add_notification_constraints.exs b/priv/repo/migrations/20200914105800_add_notification_constraints.exs new file mode 100644 index 000000000..a65c35fd0 --- /dev/null +++ b/priv/repo/migrations/20200914105800_add_notification_constraints.exs @@ -0,0 +1,23 @@ +defmodule Pleroma.Repo.Migrations.AddNotificationConstraints do + use Ecto.Migration + + def up do + drop(constraint(:notifications, "notifications_activity_id_fkey")) + + alter table(:notifications) do + modify(:activity_id, references(:activities, type: :uuid, on_delete: :delete_all), + null: false + ) + end + end + + def down do + drop(constraint(:notifications, "notifications_activity_id_fkey")) + + alter table(:notifications) do + modify(:activity_id, references(:activities, type: :uuid, on_delete: :delete_all), + null: true + ) + end + end +end From f879d07fa1a046b5aa33de8e445b1ca803fa1d04 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Tue, 15 Sep 2020 15:32:49 +0300 Subject: [PATCH 20/28] fixed tests --- test/marker_test.exs | 4 ++-- test/repo_test.exs | 4 +++- .../web/mastodon_api/controllers/account_controller_test.exs | 5 ++++- test/web/mastodon_api/controllers/marker_controller_test.exs | 2 +- test/web/mastodon_api/views/account_view_test.exs | 2 +- 5 files changed, 11 insertions(+), 6 deletions(-) diff --git a/test/marker_test.exs b/test/marker_test.exs index 5b6d0b4a4..7b3943c7b 100644 --- a/test/marker_test.exs +++ b/test/marker_test.exs @@ -33,8 +33,8 @@ test "return empty multi" do test "returns user markers" do user = insert(:user) marker = insert(:marker, user: user) - insert(:notification, user: user) - insert(:notification, user: user) + insert(:notification, user: user, activity: insert(:note_activity)) + insert(:notification, user: user, activity: insert(:note_activity)) insert(:marker, timeline: "home", user: user) assert Marker.get_markers( diff --git a/test/repo_test.exs b/test/repo_test.exs index 92e827c95..72c0b5071 100644 --- a/test/repo_test.exs +++ b/test/repo_test.exs @@ -37,7 +37,9 @@ test "get one-to-one assoc from repo" do test "get one-to-many assoc from repo" do user = insert(:user) - notification = refresh_record(insert(:notification, user: user)) + + notification = + refresh_record(insert(:notification, user: user, activity: insert(:note_activity))) assert Repo.get_assoc(user, :notifications) == {:ok, [notification]} end diff --git a/test/web/mastodon_api/controllers/account_controller_test.exs b/test/web/mastodon_api/controllers/account_controller_test.exs index 17a1e7d66..f7f1369e4 100644 --- a/test/web/mastodon_api/controllers/account_controller_test.exs +++ b/test/web/mastodon_api/controllers/account_controller_test.exs @@ -1442,7 +1442,10 @@ test "returns lists to which the account belongs" do describe "verify_credentials" do test "verify_credentials" do %{user: user, conn: conn} = oauth_access(["read:accounts"]) - [notification | _] = insert_list(7, :notification, user: user) + + [notification | _] = + insert_list(7, :notification, user: user, activity: insert(:note_activity)) + Pleroma.Notification.set_read_up_to(user, notification.id) conn = get(conn, "/api/v1/accounts/verify_credentials") diff --git a/test/web/mastodon_api/controllers/marker_controller_test.exs b/test/web/mastodon_api/controllers/marker_controller_test.exs index 6dd40fb4a..9f0481120 100644 --- a/test/web/mastodon_api/controllers/marker_controller_test.exs +++ b/test/web/mastodon_api/controllers/marker_controller_test.exs @@ -11,7 +11,7 @@ defmodule Pleroma.Web.MastodonAPI.MarkerControllerTest do test "gets markers with correct scopes", %{conn: conn} do user = insert(:user) token = insert(:oauth_token, user: user, scopes: ["read:statuses"]) - insert_list(7, :notification, user: user) + insert_list(7, :notification, user: user, activity: insert(:note_activity)) {:ok, %{"notifications" => marker}} = Pleroma.Marker.upsert( diff --git a/test/web/mastodon_api/views/account_view_test.exs b/test/web/mastodon_api/views/account_view_test.exs index 9f22f9dcf..c5f491d6b 100644 --- a/test/web/mastodon_api/views/account_view_test.exs +++ b/test/web/mastodon_api/views/account_view_test.exs @@ -448,7 +448,7 @@ test "shows unread_conversation_count only to the account owner" do test "shows unread_count only to the account owner" do user = insert(:user) - insert_list(7, :notification, user: user) + insert_list(7, :notification, user: user, activity: insert(:note_activity)) other_user = insert(:user) user = User.get_cached_by_ap_id(user.ap_id) From c74fad9e06cdb272a1378082908448f7f0b592ac Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 16 Sep 2020 03:18:50 +0000 Subject: [PATCH 21/28] Apply 1 suggestion(s) to 1 file(s) --- .../20200914105638_delete_notification_without_activity.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/priv/repo/migrations/20200914105638_delete_notification_without_activity.exs b/priv/repo/migrations/20200914105638_delete_notification_without_activity.exs index f5b339101..9333fc5a1 100644 --- a/priv/repo/migrations/20200914105638_delete_notification_without_activity.exs +++ b/priv/repo/migrations/20200914105638_delete_notification_without_activity.exs @@ -11,7 +11,7 @@ def up do select: %{id: type(q.id, :integer)}, where: is_nil(c.id) ) - |> Repo.chunk_stream(1_000, :bacthes) + |> Repo.chunk_stream(1_000, :batches) |> Stream.each(fn records -> notification_ids = Enum.map(records, fn %{id: id} -> id end) From 599f8bb152ca0669d17baa5f313f00f0791209b6 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Wed, 16 Sep 2020 09:47:18 +0300 Subject: [PATCH 22/28] RepoStreamer.chunk_stream -> Repo.chunk_stream --- lib/mix/tasks/pleroma/database.ex | 4 ++-- lib/mix/tasks/pleroma/user.ex | 4 ++-- lib/pleroma/repo.ex | 14 +++++++++++++ lib/pleroma/repo_streamer.ex | 34 ------------------------------- lib/pleroma/user.ex | 3 +-- test/repo_test.exs | 28 +++++++++++++++++++++++++ 6 files changed, 47 insertions(+), 40 deletions(-) delete mode 100644 lib/pleroma/repo_streamer.ex diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index 7f1108dcf..a01c36ece 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -99,7 +99,7 @@ def run(["fix_likes_collections"]) do where: fragment("(?)->>'likes' is not null", object.data), select: %{id: object.id, likes: fragment("(?)->>'likes'", object.data)} ) - |> Pleroma.RepoStreamer.chunk_stream(100) + |> Pleroma.Repo.chunk_stream(100, :batches) |> Stream.each(fn objects -> ids = objects @@ -145,7 +145,7 @@ def run(["ensure_expiration"]) do |> where(local: true) |> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data)) |> where([_a, o], fragment("?->>'type' = 'Note'", o.data)) - |> Pleroma.RepoStreamer.chunk_stream(100) + |> Pleroma.Repo.chunk_stream(100, :batches) |> Stream.each(fn activities -> Enum.each(activities, fn activity -> expires_at = diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex index 01824aa18..b20c49d89 100644 --- a/lib/mix/tasks/pleroma/user.ex +++ b/lib/mix/tasks/pleroma/user.ex @@ -179,7 +179,7 @@ def run(["deactivate_all_from_instance", instance]) do start_pleroma() Pleroma.User.Query.build(%{nickname: "@#{instance}"}) - |> Pleroma.RepoStreamer.chunk_stream(500) + |> Pleroma.Repo.chunk_stream(500, :batches) |> Stream.each(fn users -> users |> Enum.each(fn user -> @@ -370,7 +370,7 @@ def run(["list"]) do start_pleroma() Pleroma.User.Query.build(%{local: true}) - |> Pleroma.RepoStreamer.chunk_stream(500) + |> Pleroma.Repo.chunk_stream(500, :batches) |> Stream.each(fn users -> users |> Enum.each(fn user -> diff --git a/lib/pleroma/repo.ex b/lib/pleroma/repo.ex index a75610879..4524bd5e2 100644 --- a/lib/pleroma/repo.ex +++ b/lib/pleroma/repo.ex @@ -49,6 +49,20 @@ def get_assoc(resource, association) do end end + @doc """ + Returns a lazy enumerable that emits all entries from the data store matching the given query. + + `returns_as` use to group records. use the `batches` option to fetch records in bulk. + + ## Examples + + # fetch records one-by-one + iex> Pleroma.Repo.chunk_stream(Pleroma.Activity.Queries.by_actor(ap_id), 500) + + # fetch records in bulk + iex> Pleroma.Repo.chunk_stream(Pleroma.Activity.Queries.by_actor(ap_id), 500, :batches) + """ + @spec chunk_stream(Ecto.Query.t(), integer(), atom()) :: Enumerable.t() def chunk_stream(query, chunk_size, returns_as \\ :one) do # We don't actually need start and end funcitons of resource streaming, # but it seems to be the only way to not fetch records one-by-one and diff --git a/lib/pleroma/repo_streamer.ex b/lib/pleroma/repo_streamer.ex deleted file mode 100644 index cb4d7bb7a..000000000 --- a/lib/pleroma/repo_streamer.ex +++ /dev/null @@ -1,34 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.RepoStreamer do - alias Pleroma.Repo - import Ecto.Query - - def chunk_stream(query, chunk_size) do - Stream.unfold(0, fn - :halt -> - {[], :halt} - - last_id -> - query - |> order_by(asc: :id) - |> where([r], r.id > ^last_id) - |> limit(^chunk_size) - |> Repo.all() - |> case do - [] -> - {[], :halt} - - records -> - last_id = List.last(records).id - {records, last_id} - end - end) - |> Stream.take_while(fn - [] -> false - _ -> true - end) - end -end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index e73d19964..57497eb83 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -25,7 +25,6 @@ defmodule Pleroma.User do alias Pleroma.Object alias Pleroma.Registration alias Pleroma.Repo - alias Pleroma.RepoStreamer alias Pleroma.User alias Pleroma.UserRelationship alias Pleroma.Web @@ -1775,7 +1774,7 @@ def delete_notifications_from_user_activities(%User{ap_id: ap_id}) do def delete_user_activities(%User{ap_id: ap_id} = user) do ap_id |> Activity.Queries.by_actor() - |> RepoStreamer.chunk_stream(50) + |> Repo.chunk_stream(50, :batches) |> Stream.each(fn activities -> Enum.each(activities, fn activity -> delete_activity(activity, user) end) end) diff --git a/test/repo_test.exs b/test/repo_test.exs index 72c0b5071..155791be2 100644 --- a/test/repo_test.exs +++ b/test/repo_test.exs @@ -49,4 +49,32 @@ test "return error if has not assoc " do assert Repo.get_assoc(token, :user) == {:error, :not_found} end end + + describe "chunk_stream/3" do + test "fetch records one-by-one" do + users = insert_list(50, :user) + + {fetch_users, 50} = + from(t in User) + |> Repo.chunk_stream(5) + |> Enum.reduce({[], 0}, fn %User{} = user, {acc, count} -> + {acc ++ [user], count + 1} + end) + + assert users == fetch_users + end + + test "fetch records in bulk" do + users = insert_list(50, :user) + + {fetch_users, 10} = + from(t in User) + |> Repo.chunk_stream(5, :batches) + |> Enum.reduce({[], 0}, fn users, {acc, count} -> + {acc ++ users, count + 1} + end) + + assert users == fetch_users + end + end end From adb1b0282dfbced2b2986c90cff765be37dd5151 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 16 Sep 2020 17:23:05 +0300 Subject: [PATCH 23/28] ConnectionPool Worker: use monitor flush instead of checking ref `:flush` removes the DOWN message if one had arrived, so this check should no longer be necessary. --- lib/pleroma/gun/connection_pool/worker.ex | 25 ++++++++--------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/lib/pleroma/gun/connection_pool/worker.ex b/lib/pleroma/gun/connection_pool/worker.ex index 49d41e4c7..bf57e9e5f 100644 --- a/lib/pleroma/gun/connection_pool/worker.ex +++ b/lib/pleroma/gun/connection_pool/worker.ex @@ -93,25 +93,18 @@ def handle_call(:remove_client, {client_pid, _}, %{key: key} = state) do end) {ref, state} = pop_in(state.client_monitors[client_pid]) - # DOWN message can receive right after `remove_client` call and cause worker to terminate - state = - if is_nil(ref) do - state + + Process.demonitor(ref, [:flush]) + + timer = + if used_by == [] do + max_idle = Pleroma.Config.get([:connections_pool, :max_idle_time], 30_000) + Process.send_after(self(), :idle_close, max_idle) else - Process.demonitor(ref) - - timer = - if used_by == [] do - max_idle = Pleroma.Config.get([:connections_pool, :max_idle_time], 30_000) - Process.send_after(self(), :idle_close, max_idle) - else - nil - end - - %{state | timer: timer} + nil end - {:reply, :ok, state, :hibernate} + {:reply, :ok, %{state | timer: timer}, :hibernate} end @impl true From 7a88b726bf81e1610ade2b07ffd6af672b701600 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 16 Sep 2020 17:29:16 +0200 Subject: [PATCH 24/28] User: Remote users don't need to be confirmed or approved --- lib/pleroma/user.ex | 4 ++-- test/user_test.exs | 15 ++++++++++++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 57497eb83..1ffe60dfc 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -275,9 +275,9 @@ def binary_id(%User{} = user), do: binary_id(user.id) @spec account_status(User.t()) :: account_status() def account_status(%User{deactivated: true}), do: :deactivated def account_status(%User{password_reset_pending: true}), do: :password_reset_pending - def account_status(%User{approval_pending: true}), do: :approval_pending + def account_status(%User{local: true, approval_pending: true}), do: :approval_pending - def account_status(%User{confirmation_pending: true}) do + def account_status(%User{local: true, confirmation_pending: true}) do if Config.get([:instance, :account_activation_required]) do :confirmation_pending else diff --git a/test/user_test.exs b/test/user_test.exs index a910226b2..060918d71 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -1676,7 +1676,7 @@ test "returns true when the account is itself" do assert User.visible_for(user, user) == :visible end - test "returns false when the account is unauthenticated and auth is required" do + test "returns false when the account is unconfirmed and confirmation is required" do Pleroma.Config.put([:instance, :account_activation_required], true) user = insert(:user, local: true, confirmation_pending: true) @@ -1685,14 +1685,23 @@ test "returns false when the account is unauthenticated and auth is required" do refute User.visible_for(user, other_user) == :visible end - test "returns true when the account is unauthenticated and auth is not required" do + test "returns true when the account is unconfirmed and confirmation is required but the account is remote" do + Pleroma.Config.put([:instance, :account_activation_required], true) + + user = insert(:user, local: false, confirmation_pending: true) + other_user = insert(:user, local: true) + + assert User.visible_for(user, other_user) == :visible + end + + test "returns true when the account is unconfirmed and confirmation is not required" do user = insert(:user, local: true, confirmation_pending: true) other_user = insert(:user, local: true) assert User.visible_for(user, other_user) == :visible end - test "returns true when the account is unauthenticated and being viewed by a privileged account (auth required)" do + test "returns true when the account is unconfirmed and being viewed by a privileged account (confirmation required)" do Pleroma.Config.put([:instance, :account_activation_required], true) user = insert(:user, local: true, confirmation_pending: true) From 73e0e6a8a2efaac513077317229f72b128f3a3ea Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 16 Sep 2020 10:56:42 -0500 Subject: [PATCH 25/28] Remove unused import --- test/object/fetcher_test.exs | 1 - 1 file changed, 1 deletion(-) diff --git a/test/object/fetcher_test.exs b/test/object/fetcher_test.exs index 3173ee31c..14d2c645f 100644 --- a/test/object/fetcher_test.exs +++ b/test/object/fetcher_test.exs @@ -10,7 +10,6 @@ defmodule Pleroma.Object.FetcherTest do alias Pleroma.Object alias Pleroma.Object.Fetcher - import ExUnit.CaptureLog import Mock import Tesla.Mock From e39ff2616b6694f97ab793bc60b5caa7b509f0b1 Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 17 Sep 2020 13:29:26 +0200 Subject: [PATCH 26/28] Admin chat api tests: Small additions. --- test/web/admin_api/controllers/admin_api_controller_test.exs | 2 +- test/web/admin_api/controllers/chat_controller_test.exs | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/test/web/admin_api/controllers/admin_api_controller_test.exs b/test/web/admin_api/controllers/admin_api_controller_test.exs index 3476fd0b4..e6ad210a2 100644 --- a/test/web/admin_api/controllers/admin_api_controller_test.exs +++ b/test/web/admin_api/controllers/admin_api_controller_test.exs @@ -1521,7 +1521,7 @@ test "excludes reblogs by default", %{conn: conn, user: user} do %{user: user} end - test "renders user's statuses", %{conn: conn, user: user} do + test "renders user's chats", %{conn: conn, user: user} do conn = get(conn, "/api/pleroma/admin/users/#{user.nickname}/chats") assert json_response(conn, 200) |> length() == 3 diff --git a/test/web/admin_api/controllers/chat_controller_test.exs b/test/web/admin_api/controllers/chat_controller_test.exs index e81484ce6..bd4c9c9d1 100644 --- a/test/web/admin_api/controllers/chat_controller_test.exs +++ b/test/web/admin_api/controllers/chat_controller_test.exs @@ -40,8 +40,10 @@ test "it deletes a message from the chat", %{conn: conn, admin: admin} do object = Object.normalize(message, false) chat = Chat.get(user.id, recipient.ap_id) + recipient_chat = Chat.get(recipient.id, user.ap_id) cm_ref = MessageReference.for_chat_and_object(chat, object) + recipient_cm_ref = MessageReference.for_chat_and_object(recipient_chat, object) result = conn @@ -56,6 +58,7 @@ test "it deletes a message from the chat", %{conn: conn, admin: admin} do assert result["id"] == cm_ref.id refute MessageReference.get_by_id(cm_ref.id) + refute MessageReference.get_by_id(recipient_cm_ref.id) assert %{data: %{"type" => "Tombstone"}} = Object.get_by_id(object.id) end end From 5e3c70afa5c02926a5578628431487e92b2175e9 Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 17 Sep 2020 13:37:25 +0200 Subject: [PATCH 27/28] AdminAPI Chat tests: Remove factory. The factory system doesn't work too well with how the chats are done. Instead of tempting people to use it, let's rather use the CommonAPI system for now. --- test/support/factory.ex | 54 ------------------- .../controllers/admin_api_controller_test.exs | 13 +++-- 2 files changed, 8 insertions(+), 59 deletions(-) diff --git a/test/support/factory.ex b/test/support/factory.ex index e59d83242..2fdfabbc5 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -441,58 +441,4 @@ def filter_factory do phrase: "cofe" } end - - def chat_factory(attrs \\ %{}) do - user = attrs[:user] || insert(:user) - recipient = attrs[:recipient] || insert(:user) - - %Pleroma.Chat{ - user_id: user.id, - recipient: recipient.ap_id - } - end - - def chat_message_factory(attrs \\ %{}) do - text = sequence(:text, &"This is :moominmamma: chat message #{&1}") - chat = attrs[:chat] || insert(:chat) - - data = %{ - "type" => "ChatMessage", - "content" => text, - "id" => Pleroma.Web.ActivityPub.Utils.generate_object_id(), - "actor" => User.get_by_id(chat.user_id).ap_id, - "to" => [chat.recipient], - "published" => DateTime.utc_now() |> DateTime.to_iso8601() - } - - %Pleroma.Object{ - data: merge_attributes(data, Map.get(attrs, :data, %{})) - } - end - - def chat_message_activity_factory(attrs \\ %{}) do - chat = attrs[:chat] || insert(:chat) - chat_message = attrs[:chat_message] || insert(:chat_message, chat: chat) - - data_attrs = attrs[:data_attrs] || %{} - attrs = Map.drop(attrs, [:chat, :chat_message, :data_attrs]) - - data = - %{ - "id" => Pleroma.Web.ActivityPub.Utils.generate_activity_id(), - "type" => "Create", - "actor" => chat_message.data["actor"], - "to" => chat_message.data["to"], - "object" => chat_message.data["id"], - "published" => DateTime.utc_now() |> DateTime.to_iso8601() - } - |> Map.merge(data_attrs) - - %Pleroma.Activity{ - data: data, - actor: data["actor"], - recipients: data["to"] - } - |> Map.merge(attrs) - end end diff --git a/test/web/admin_api/controllers/admin_api_controller_test.exs b/test/web/admin_api/controllers/admin_api_controller_test.exs index e6ad210a2..e4d3512de 100644 --- a/test/web/admin_api/controllers/admin_api_controller_test.exs +++ b/test/web/admin_api/controllers/admin_api_controller_test.exs @@ -1513,10 +1513,11 @@ test "excludes reblogs by default", %{conn: conn, user: user} do describe "GET /api/pleroma/admin/users/:nickname/chats" do setup do user = insert(:user) + recipients = insert_list(3, :user) - insert(:chat, user: user) - insert(:chat, user: user) - insert(:chat, user: user) + Enum.each(recipients, fn recipient -> + CommonAPI.post_chat_message(user, recipient, "yo") + end) %{user: user} end @@ -1531,7 +1532,8 @@ test "renders user's chats", %{conn: conn, user: user} do describe "GET /api/pleroma/admin/users/:nickname/chats unauthorized" do setup do user = insert(:user) - insert(:chat, user: user) + recipient = insert(:user) + CommonAPI.post_chat_message(user, recipient, "yo") %{conn: conn} = oauth_access(["read:chats"]) %{conn: conn, user: user} end @@ -1546,7 +1548,8 @@ test "returns 403", %{conn: conn, user: user} do describe "GET /api/pleroma/admin/users/:nickname/chats unauthenticated" do setup do user = insert(:user) - insert(:chat, user: user) + recipient = insert(:user) + CommonAPI.post_chat_message(user, recipient, "yo") %{conn: build_conn(), user: user} end From db80b9d630f9fc72ebc269cb24142501116c269a Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 17 Sep 2020 16:13:21 +0300 Subject: [PATCH 28/28] RichMedia: Fix log spam on failures and resetting TTL on cached errors --- lib/pleroma/web/rich_media/parser.ex | 67 +++++++++++++++++++--------- 1 file changed, 46 insertions(+), 21 deletions(-) diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex index 33f6f1fa1..c70d2fdba 100644 --- a/lib/pleroma/web/rich_media/parser.ex +++ b/lib/pleroma/web/rich_media/parser.ex @@ -20,36 +20,61 @@ def parse(url) do with {:ok, data} <- get_cached_or_parse(url), {:ok, _} <- set_ttl_based_on_image(data, url) do {:ok, data} - else - {:error, {:invalid_metadata, data}} = e -> - Logger.debug(fn -> "Incomplete or invalid metadata for #{url}: #{inspect(data)}" end) - e - - error -> - Logger.error(fn -> "Rich media error for #{url}: #{inspect(error)}" end) - error end end defp get_cached_or_parse(url) do - case Cachex.fetch!(:rich_media_cache, url, fn _ -> {:commit, parse_url(url)} end) do - {:ok, _data} = res -> - res + case Cachex.fetch(:rich_media_cache, url, fn -> + case parse_url(url) do + {:ok, _} = res -> + {:commit, res} - {:error, :body_too_large} = e -> - e + {:error, reason} = e -> + # Unfortunately we have to log errors here, instead of doing that + # along with ttl setting at the bottom. Otherwise we can get log spam + # if more than one process was waiting for the rich media card + # while it was generated. Ideally we would set ttl here as well, + # so we don't override it number_of_waiters_on_generation + # times, but one, obviously, can't set ttl for not-yet-created entry + # and Cachex doesn't support returning ttl from the fetch callback. + log_error(url, reason) + {:commit, e} + end + end) do + {action, res} when action in [:commit, :ok] -> + case res do + {:ok, _data} = res -> + res - {:error, {:content_type, _}} = e -> - e + {:error, reason} = e -> + if action == :commit, do: set_error_ttl(url, reason) + e + end - # The TTL is not set for the errors above, since they are unlikely to change - # with time - {:error, _} = e -> - ttl = Pleroma.Config.get([:rich_media, :failure_backoff], 60_000) - Cachex.expire(:rich_media_cache, url, ttl) - e + {:error, e} -> + {:error, {:cachex_error, e}} end end + + defp set_error_ttl(_url, :body_too_large), do: :ok + defp set_error_ttl(_url, {:content_type, _}), do: :ok + + # The TTL is not set for the errors above, since they are unlikely to change + # with time + + defp set_error_ttl(url, _reason) do + ttl = Pleroma.Config.get([:rich_media, :failure_backoff], 60_000) + Cachex.expire(:rich_media_cache, url, ttl) + :ok + end + + defp log_error(url, {:invalid_metadata, data}) do + Logger.debug(fn -> "Incomplete or invalid metadata for #{url}: #{inspect(data)}" end) + end + + defp log_error(url, reason) do + Logger.warn(fn -> "Rich media error for #{url}: #{inspect(reason)}" end) + end end @doc """