From ce387ce7306d944c11e63f0c86719afd5c2da2d2 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 28 Aug 2020 16:15:57 +0300 Subject: [PATCH 001/128] mix.exs: bump development version after 2.1.0 release --- mix.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index d7c408972..4de0c78db 100644 --- a/mix.exs +++ b/mix.exs @@ -4,7 +4,7 @@ defmodule Pleroma.Mixfile do def project do [ app: :pleroma, - version: version("2.1.0"), + version: version("2.1.50"), elixir: "~> 1.9", elixirc_paths: elixirc_paths(Mix.env()), compilers: [:phoenix, :gettext] ++ Mix.compilers(), From 35da3be9ddf2df042153978308178c94afe25737 Mon Sep 17 00:00:00 2001 From: Shpuld Shpludson Date: Fri, 28 Aug 2020 13:50:21 +0000 Subject: [PATCH 002/128] Add mention of sudo -Hu pleroma to docs --- docs/administration/updating.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/administration/updating.md b/docs/administration/updating.md index c994f3f16..ef2c9218c 100644 --- a/docs/administration/updating.md +++ b/docs/administration/updating.md @@ -18,9 +18,10 @@ su pleroma -s $SHELL -lc "./bin/pleroma_ctl migrate" 1. Go to the working directory of Pleroma (default is `/opt/pleroma`) 2. Run `git pull`. This pulls the latest changes from upstream. -3. Run `mix deps.get`. This pulls in any new dependencies. +3. Run `mix deps.get` [^1]. This pulls in any new dependencies. 4. Stop the Pleroma service. -5. Run `mix ecto.migrate`[^1]. This task performs database migrations, if there were any. +5. Run `mix ecto.migrate` [^1] [^2]. This task performs database migrations, if there were any. 6. Start the Pleroma service. -[^1]: Prefix with `MIX_ENV=prod` to run it using the production config file. +[^1]: Depending on which install guide you followed (for example on Debian/Ubuntu), you want to run `mix` tasks as `pleroma` user by adding `sudo -Hu pleroma` before the command. +[^2]: Prefix with `MIX_ENV=prod` to run it using the production config file. From f0fefc4f5c3aa4fa62f2edee72ee864a16e7176d Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Fri, 28 Aug 2020 18:17:44 +0300 Subject: [PATCH 003/128] marks notifications as read after mute --- lib/pleroma/notification.ex | 12 +++++ lib/pleroma/web/common_api/common_api.ex | 3 +- test/web/common_api/common_api_test.exs | 65 ++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index c1825f810..b952e81fa 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -648,4 +648,16 @@ def for_user_and_activity(user, activity) do ) |> Repo.one() end + + @spec mark_as_read(User.t(), Activity.t()) :: {integer(), nil | [term()]} + def mark_as_read(%User{id: id}, %Activity{data: %{"context" => context}}) do + from( + n in Notification, + join: a in assoc(n, :activity), + where: n.user_id == ^id, + where: n.seen == false, + where: fragment("?->>'context'", a.data) == ^context + ) + |> Repo.update_all(set: [seen: true]) + end end diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 5ad2b91c2..43e9e39a8 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -452,7 +452,8 @@ def unpin(id, user) do end def add_mute(user, activity) do - with {:ok, _} <- ThreadMute.add_mute(user.id, activity.data["context"]) do + with {:ok, _} <- ThreadMute.add_mute(user.id, activity.data["context"]), + _ <- Pleroma.Notification.mark_as_read(user, activity) do {:ok, activity} else {:error, _} -> {:error, dgettext("errors", "conversation is already muted")} diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs index 4ba6232dc..800db9a20 100644 --- a/test/web/common_api/common_api_test.exs +++ b/test/web/common_api/common_api_test.exs @@ -9,6 +9,7 @@ defmodule Pleroma.Web.CommonAPITest do alias Pleroma.Conversation.Participation alias Pleroma.Notification alias Pleroma.Object + alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Transmogrifier @@ -18,6 +19,7 @@ defmodule Pleroma.Web.CommonAPITest do import Pleroma.Factory import Mock + import Ecto.Query, only: [from: 2] require Pleroma.Constants @@ -808,6 +810,69 @@ test "should unpin when deleting a status", %{user: user, activity: activity} do [user: user, activity: activity] end + test "marks notifications as read after mute" do + author = insert(:user) + activity = insert(:note_activity, user: author) + + friend1 = insert(:user) + friend2 = insert(:user) + + {:ok, reply_activity} = + CommonAPI.post( + friend2, + %{ + status: "@#{author.nickname} @#{friend1.nickname} test reply", + in_reply_to_status_id: activity.id + } + ) + + {:ok, favorite_activity} = CommonAPI.favorite(friend2, activity.id) + {:ok, repeat_activity} = CommonAPI.repeat(activity.id, friend1) + + assert Repo.aggregate( + from(n in Notification, where: n.seen == false and n.user_id == ^friend1.id), + :count + ) == 1 + + unread_notifications = + Repo.all(from(n in Notification, where: n.seen == false, where: n.user_id == ^author.id)) + + assert Enum.any?(unread_notifications, fn n -> + n.type == "favourite" && n.activity_id == favorite_activity.id + end) + + assert Enum.any?(unread_notifications, fn n -> + n.type == "reblog" && n.activity_id == repeat_activity.id + end) + + assert Enum.any?(unread_notifications, fn n -> + n.type == "mention" && n.activity_id == reply_activity.id + end) + + {:ok, _} = CommonAPI.add_mute(author, activity) + assert CommonAPI.thread_muted?(author, activity) + + assert Repo.aggregate( + from(n in Notification, where: n.seen == false and n.user_id == ^friend1.id), + :count + ) == 1 + + read_notifications = + Repo.all(from(n in Notification, where: n.seen == true, where: n.user_id == ^author.id)) + + assert Enum.any?(read_notifications, fn n -> + n.type == "favourite" && n.activity_id == favorite_activity.id + end) + + assert Enum.any?(read_notifications, fn n -> + n.type == "reblog" && n.activity_id == repeat_activity.id + end) + + assert Enum.any?(read_notifications, fn n -> + n.type == "mention" && n.activity_id == reply_activity.id + end) + end + test "add mute", %{user: user, activity: activity} do {:ok, _} = CommonAPI.add_mute(user, activity) assert CommonAPI.thread_muted?(user, activity) From 858e9a59ed8537e42c490645633266e13474ae99 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Mon, 31 Aug 2020 09:54:16 +0300 Subject: [PATCH 004/128] mix.lock: bump fast_sanitize The update brings a better error message for when cmake isn't installed --- mix.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.lock b/mix.lock index 86d0a75d7..afb4b06db 100644 --- a/mix.lock +++ b/mix.lock @@ -42,8 +42,8 @@ "ex_machina": {:hex, :ex_machina, "2.4.0", "09a34c5d371bfb5f78399029194a8ff67aff340ebe8ba19040181af35315eabb", [:mix], [{:ecto, "~> 2.2 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}], "hexpm", "a20bc9ddc721b33ea913b93666c5d0bdca5cbad7a67540784ae277228832d72c"}, "ex_syslogger": {:hex, :ex_syslogger, "1.5.2", "72b6aa2d47a236e999171f2e1ec18698740f40af0bd02c8c650bf5f1fd1bac79", [:mix], [{:poison, ">= 1.5.0", [hex: :poison, repo: "hexpm", optional: true]}, {:syslog, "~> 1.1.0", [hex: :syslog, repo: "hexpm", optional: false]}], "hexpm", "ab9fab4136dbc62651ec6f16fa4842f10cf02ab4433fa3d0976c01be99398399"}, "excoveralls": {:hex, :excoveralls, "0.13.1", "b9f1697f7c9e0cfe15d1a1d737fb169c398803ffcbc57e672aa007e9fd42864c", [:mix], [{:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "b4bb550e045def1b4d531a37fb766cbbe1307f7628bf8f0414168b3f52021cce"}, - "fast_html": {:hex, :fast_html, "2.0.2", "1fabc408b2baa965cf6399a48796326f2721b21b397a3c667bb3bb88fb9559a4", [:make, :mix], [{:elixir_make, "~> 0.4", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.1.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}], "hexpm", "f077e2c1597a6e2678e6cacc64f456a6c6024eb4240092c46d4212496dc59aba"}, - "fast_sanitize": {:hex, :fast_sanitize, "0.2.1", "3302421a988992b6cae08e68f77069e167ff116444183f3302e3c36017a50558", [:mix], [{:fast_html, "~> 2.0", [hex: :fast_html, repo: "hexpm", optional: false]}, {:plug, "~> 1.8", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "bcd2c54e328128515edd1a8fb032fdea7e5581672ba161fc5962d21ecee92502"}, + "fast_html": {:hex, :fast_html, "2.0.3", "27289dea6c3a22952191a2d4e07f546c0de8a351484782c2e797dbae06a5dc8a", [:make, :mix], [{:elixir_make, "~> 0.4", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.1.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}], "hexpm", "17d41fa8afe4e912ffe74e13b87ddb085382cd2b7393636d338495c9a8a7b518"}, + "fast_sanitize": {:hex, :fast_sanitize, "0.2.2", "3cbbaebaea6043865dfb5b4ecb0f1af066ad410a51470e353714b10c42007b81", [:mix], [{:fast_html, "~> 2.0", [hex: :fast_html, repo: "hexpm", optional: false]}, {:plug, "~> 1.8", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "69f204db9250afa94a0d559d9110139850f57de2b081719fbafa1e9a89e94466"}, "flake_id": {:hex, :flake_id, "0.1.0", "7716b086d2e405d09b647121a166498a0d93d1a623bead243e1f74216079ccb3", [:mix], [{:base62, "~> 1.2", [hex: :base62, repo: "hexpm", optional: false]}, {:ecto, ">= 2.0.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm", "31fc8090fde1acd267c07c36ea7365b8604055f897d3a53dd967658c691bd827"}, "floki": {:hex, :floki, "0.27.0", "6b29a14283f1e2e8fad824bc930eaa9477c462022075df6bea8f0ad811c13599", [:mix], [{:html_entities, "~> 0.5.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm", "583b8c13697c37179f1f82443bcc7ad2f76fbc0bf4c186606eebd658f7f2631b"}, "gen_smtp": {:hex, :gen_smtp, "0.15.0", "9f51960c17769b26833b50df0b96123605a8024738b62db747fece14eb2fbfcc", [:rebar3], [], "hexpm", "29bd14a88030980849c7ed2447b8db6d6c9278a28b11a44cafe41b791205440f"}, From d91c4feebeb199f7c584f0a4292ce6f9cc331798 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 31 Aug 2020 11:02:54 +0200 Subject: [PATCH 005/128] Notification: Small refactor. --- lib/pleroma/notification.ex | 4 ++-- lib/pleroma/web/common_api/common_api.ex | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index b952e81fa..8868a910e 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -649,8 +649,8 @@ def for_user_and_activity(user, activity) do |> Repo.one() end - @spec mark_as_read(User.t(), Activity.t()) :: {integer(), nil | [term()]} - def mark_as_read(%User{id: id}, %Activity{data: %{"context" => context}}) do + @spec mark_context_as_read(User.t(), String.t()) :: {integer(), nil | [term()]} + def mark_context_as_read(%User{id: id}, context) do from( n in Notification, join: a in assoc(n, :activity), diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 43e9e39a8..4ab533658 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -453,7 +453,7 @@ def unpin(id, user) do def add_mute(user, activity) do with {:ok, _} <- ThreadMute.add_mute(user.id, activity.data["context"]), - _ <- Pleroma.Notification.mark_as_read(user, activity) do + _ <- Pleroma.Notification.mark_context_as_read(user, activity.data["context"]) do {:ok, activity} else {:error, _} -> {:error, dgettext("errors", "conversation is already muted")} From 0b621a834acf751332f4d202bd50d4ff3e789176 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 31 Aug 2020 16:48:17 +0200 Subject: [PATCH 006/128] Chats: Add cascading delete on both referenced users. Also remove the now-superfluous join in the chat controller, which was only used to filter out these cases. --- .../controllers/chat_controller.ex | 4 +--- .../20200831142509_chat_constraints.exs | 22 +++++++++++++++++++ test/chat_test.exs | 22 +++++++++++++++++++ 3 files changed, 45 insertions(+), 3 deletions(-) create mode 100644 priv/repo/migrations/20200831142509_chat_constraints.exs diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex index 1f2e953f7..e8a1746d4 100644 --- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex @@ -149,9 +149,7 @@ def index(%{assigns: %{user: %{id: user_id} = user}} = conn, _params) do from(c in Chat, where: c.user_id == ^user_id, where: c.recipient not in ^blocked_ap_ids, - order_by: [desc: c.updated_at], - inner_join: u in User, - on: u.ap_id == c.recipient + order_by: [desc: c.updated_at] ) |> Repo.all() diff --git a/priv/repo/migrations/20200831142509_chat_constraints.exs b/priv/repo/migrations/20200831142509_chat_constraints.exs new file mode 100644 index 000000000..868a40a45 --- /dev/null +++ b/priv/repo/migrations/20200831142509_chat_constraints.exs @@ -0,0 +1,22 @@ +defmodule Pleroma.Repo.Migrations.ChatConstraints do + use Ecto.Migration + + def change do + remove_orphans = """ + delete from chats where not exists(select id from users where ap_id = chats.recipient); + """ + + execute(remove_orphans) + + drop(constraint(:chats, "chats_user_id_fkey")) + + alter table(:chats) do + modify(:user_id, references(:users, type: :uuid, on_delete: :delete_all)) + + modify( + :recipient, + references(:users, column: :ap_id, type: :string, on_delete: :delete_all) + ) + end + end +end diff --git a/test/chat_test.exs b/test/chat_test.exs index 332f2180a..9e8a9ebf0 100644 --- a/test/chat_test.exs +++ b/test/chat_test.exs @@ -26,6 +26,28 @@ test "it creates a chat for a user and recipient" do assert chat.id end + test "deleting the user deletes the chat" do + user = insert(:user) + other_user = insert(:user) + + {:ok, chat} = Chat.bump_or_create(user.id, other_user.ap_id) + + Repo.delete(user) + + refute Chat.get_by_id(chat.id) + end + + test "deleting the recipient deletes the chat" do + user = insert(:user) + other_user = insert(:user) + + {:ok, chat} = Chat.bump_or_create(user.id, other_user.ap_id) + + Repo.delete(other_user) + + refute Chat.get_by_id(chat.id) + end + test "it returns and bumps a chat for a user and recipient if it already exists" do user = insert(:user) other_user = insert(:user) From dc3a418c270e48c6849159097f8bba57d2a2578e Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 1 Sep 2020 09:08:54 +0300 Subject: [PATCH 007/128] application.ex: disable warnings_as_errors at runtime see changed files for rationale --- lib/pleroma/application.ex | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index c0b5db9f1..005aba50a 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -39,6 +39,9 @@ def start(_type, _args) do # every time the application is restarted, so we disable module # conflicts at runtime Code.compiler_options(ignore_module_conflict: true) + # Disable warnings_as_errors at runtime, it breaks Phoenix live reload + # due to protocol consolidation warnings + Code.compiler_options(warnings_as_errors: false) Pleroma.Telemetry.Logger.attach() Config.Holder.save_default() Pleroma.HTML.compile_scrubbers() From 2ecc7d92308d624dc9edb50665d752a71f55f608 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 25 Aug 2020 00:38:10 +0200 Subject: [PATCH 008/128] transmogrifier: Remove mastodon emoji-format from emoji field --- lib/pleroma/web/activity_pub/transmogrifier.ex | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 76298c4a0..0831efadc 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -318,9 +318,6 @@ def fix_emoji(%{"tag" => tags} = object) when is_list(tags) do Map.put(mapping, name, data["icon"]["url"]) end) - # we merge mastodon and pleroma emoji into a single mapping, to allow for both wire formats - emoji = Map.merge(object["emoji"] || %{}, emoji) - Map.put(object, "emoji", emoji) end From a142da3e4f03f2bfee7af30cca59b0fdc82da73f Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 25 Aug 2020 01:16:12 +0200 Subject: [PATCH 009/128] Add new Emoji Ecto.Type and fix emoji in Question --- .../activity_pub/object_validators/emoji.ex | 34 +++++++++++++ .../object_validators/audio_validator.ex | 5 +- .../chat_message_validator.ex | 2 +- .../object_validators/event_validator.ex | 5 +- .../object_validators/note_validator.ex | 11 +++- .../object_validators/question_validator.ex | 5 +- .../chat_validation_test.exs | 1 + .../transmogrifier/question_handling_test.exs | 51 +++++++++++++++++++ 8 files changed, 105 insertions(+), 9 deletions(-) create mode 100644 lib/pleroma/ecto_type/activity_pub/object_validators/emoji.ex diff --git a/lib/pleroma/ecto_type/activity_pub/object_validators/emoji.ex b/lib/pleroma/ecto_type/activity_pub/object_validators/emoji.ex new file mode 100644 index 000000000..4aacc5c88 --- /dev/null +++ b/lib/pleroma/ecto_type/activity_pub/object_validators/emoji.ex @@ -0,0 +1,34 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.Emoji do + use Ecto.Type + + def type, do: :map + + def cast(data) when is_map(data) do + has_invalid_emoji? = + Enum.find(data, fn + {name, uri} when is_binary(name) and is_binary(uri) -> + # based on ObjectValidators.Uri.cast() + case URI.parse(uri) do + %URI{host: nil} -> true + %URI{host: ""} -> true + %URI{scheme: scheme} when scheme in ["https", "http"] -> false + _ -> true + end + + {_name, _uri} -> + true + end) + + if has_invalid_emoji?, do: :error, else: {:ok, data} + end + + def cast(_data), do: :error + + def dump(data), do: {:ok, data} + + def load(data), do: {:ok, data} +end diff --git a/lib/pleroma/web/activity_pub/object_validators/audio_validator.ex b/lib/pleroma/web/activity_pub/object_validators/audio_validator.ex index d1869f188..1a97c504a 100644 --- a/lib/pleroma/web/activity_pub/object_validators/audio_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/audio_validator.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioValidator do alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations + alias Pleroma.Web.ActivityPub.Transmogrifier import Ecto.Changeset @@ -33,8 +34,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioValidator do field(:attributedTo, ObjectValidators.ObjectID) field(:summary, :string) field(:published, ObjectValidators.DateTime) - # TODO: Write type - field(:emoji, :map, default: %{}) + field(:emoji, ObjectValidators.Emoji, default: %{}) field(:sensitive, :boolean, default: false) embeds_many(:attachment, AttachmentValidator) field(:replies_count, :integer, default: 0) @@ -83,6 +83,7 @@ defp fix(data) do data |> CommonFixes.fix_defaults() |> CommonFixes.fix_attribution() + |> Transmogrifier.fix_emoji() |> fix_url() end diff --git a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex index 91b475393..6acd4a771 100644 --- a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex @@ -22,7 +22,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator do field(:content, ObjectValidators.SafeText) field(:actor, ObjectValidators.ObjectID) field(:published, ObjectValidators.DateTime) - field(:emoji, :map, default: %{}) + field(:emoji, ObjectValidators.Emoji, default: %{}) embeds_one(:attachment, AttachmentValidator) end diff --git a/lib/pleroma/web/activity_pub/object_validators/event_validator.ex b/lib/pleroma/web/activity_pub/object_validators/event_validator.ex index 07e4821a4..0b4c99dc0 100644 --- a/lib/pleroma/web/activity_pub/object_validators/event_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/event_validator.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EventValidator do alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations + alias Pleroma.Web.ActivityPub.Transmogrifier import Ecto.Changeset @@ -39,8 +40,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EventValidator do field(:attributedTo, ObjectValidators.ObjectID) field(:published, ObjectValidators.DateTime) - # TODO: Write type - field(:emoji, :map, default: %{}) + field(:emoji, ObjectValidators.Emoji, default: %{}) field(:sensitive, :boolean, default: false) embeds_many(:attachment, AttachmentValidator) field(:replies_count, :integer, default: 0) @@ -74,6 +74,7 @@ defp fix(data) do data |> CommonFixes.fix_defaults() |> CommonFixes.fix_attribution() + |> Transmogrifier.fix_emoji() end def changeset(struct, data) do diff --git a/lib/pleroma/web/activity_pub/object_validators/note_validator.ex b/lib/pleroma/web/activity_pub/object_validators/note_validator.ex index 20e735619..ab4469a59 100644 --- a/lib/pleroma/web/activity_pub/object_validators/note_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/note_validator.ex @@ -6,6 +6,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do use Ecto.Schema alias Pleroma.EctoType.ActivityPub.ObjectValidators + alias Pleroma.Web.ActivityPub.Transmogrifier import Ecto.Changeset @@ -32,8 +33,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do field(:actor, ObjectValidators.ObjectID) field(:attributedTo, ObjectValidators.ObjectID) field(:published, ObjectValidators.DateTime) - # TODO: Write type - field(:emoji, :map, default: %{}) + field(:emoji, ObjectValidators.Emoji, default: %{}) field(:sensitive, :boolean, default: false) # TODO: Write type field(:attachment, {:array, :map}, default: []) @@ -53,7 +53,14 @@ def cast_and_validate(data) do |> validate_data() end + defp fix(data) do + data + |> Transmogrifier.fix_emoji() + end + def cast_data(data) do + data = fix(data) + %__MODULE__{} |> cast(data, __schema__(:fields)) end diff --git a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex index 712047424..934d3c1ea 100644 --- a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex @@ -10,6 +10,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations alias Pleroma.Web.ActivityPub.ObjectValidators.QuestionOptionsValidator + alias Pleroma.Web.ActivityPub.Transmogrifier import Ecto.Changeset @@ -35,8 +36,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do field(:attributedTo, ObjectValidators.ObjectID) field(:summary, :string) field(:published, ObjectValidators.DateTime) - # TODO: Write type - field(:emoji, :map, default: %{}) + field(:emoji, ObjectValidators.Emoji, default: %{}) field(:sensitive, :boolean, default: false) embeds_many(:attachment, AttachmentValidator) field(:replies_count, :integer, default: 0) @@ -85,6 +85,7 @@ defp fix(data) do data |> CommonFixes.fix_defaults() |> CommonFixes.fix_attribution() + |> Transmogrifier.fix_emoji() |> fix_closed() end diff --git a/test/web/activity_pub/object_validators/chat_validation_test.exs b/test/web/activity_pub/object_validators/chat_validation_test.exs index 50bf03515..16e4808e5 100644 --- a/test/web/activity_pub/object_validators/chat_validation_test.exs +++ b/test/web/activity_pub/object_validators/chat_validation_test.exs @@ -69,6 +69,7 @@ test "validates for a basic object we build", %{valid_chat_message: valid_chat_m assert {:ok, object, _meta} = ObjectValidator.validate(valid_chat_message, []) assert Map.put(valid_chat_message, "attachment", nil) == object + assert match?(%{"firefox" => _}, object["emoji"]) end test "validates for a basic object with an attachment", %{ diff --git a/test/web/activity_pub/transmogrifier/question_handling_test.exs b/test/web/activity_pub/transmogrifier/question_handling_test.exs index c82361828..74ee79543 100644 --- a/test/web/activity_pub/transmogrifier/question_handling_test.exs +++ b/test/web/activity_pub/transmogrifier/question_handling_test.exs @@ -106,6 +106,57 @@ test "Mastodon Question activity with HTML tags in plaintext" do assert Enum.sort(object.data["oneOf"]) == Enum.sort(options) end + test "Mastodon Question activity with custom emojis" do + options = [ + %{ + "type" => "Note", + "name" => ":blobcat:", + "replies" => %{"totalItems" => 0, "type" => "Collection"} + }, + %{ + "type" => "Note", + "name" => ":blobfox:", + "replies" => %{"totalItems" => 0, "type" => "Collection"} + } + ] + + tag = [ + %{ + "icon" => %{ + "type" => "Image", + "url" => "https://blob.cat/emoji/custom/blobcats/blobcat.png" + }, + "id" => "https://blob.cat/emoji/custom/blobcats/blobcat.png", + "name" => ":blobcat:", + "type" => "Emoji", + "updated" => "1970-01-01T00:00:00Z" + }, + %{ + "icon" => %{"type" => "Image", "url" => "https://blob.cat/emoji/blobfox/blobfox.png"}, + "id" => "https://blob.cat/emoji/blobfox/blobfox.png", + "name" => ":blobfox:", + "type" => "Emoji", + "updated" => "1970-01-01T00:00:00Z" + } + ] + + data = + File.read!("test/fixtures/mastodon-question-activity.json") + |> Poison.decode!() + |> Kernel.put_in(["object", "oneOf"], options) + |> Kernel.put_in(["object", "tag"], tag) + + {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data) + object = Object.normalize(activity, false) + + assert object.data["oneOf"] == options + + assert object.data["emoji"] == %{ + "blobcat" => "https://blob.cat/emoji/custom/blobcats/blobcat.png", + "blobfox" => "https://blob.cat/emoji/blobfox/blobfox.png" + } + end + test "returns an error if received a second time" do data = File.read!("test/fixtures/mastodon-question-activity.json") |> Poison.decode!() From b960cede9a3183098ac5eb330fbc4d1c14efc034 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 25 Aug 2020 02:18:33 +0200 Subject: [PATCH 010/128] common_fixes: Force inserting context and context_id --- .../web/activity_pub/object_validators/common_fixes.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex index 721749de0..720213d73 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex @@ -11,8 +11,8 @@ def fix_defaults(data) do Utils.create_context(data["context"] || data["conversation"]) data - |> Map.put_new("context", context) - |> Map.put_new("context_id", context_id) + |> Map.put("context", context) + |> Map.put("context_id", context_id) end def fix_attribution(data) do From d9a21e4784a83a0780b353c967520cd203f44f3f Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 25 Aug 2020 02:21:19 +0200 Subject: [PATCH 011/128] fetcher: Remove fix_object call for Question activities --- lib/pleroma/object/fetcher.ex | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 6fdbc8efd..d26c5adf5 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -36,8 +36,7 @@ defp maybe_reinject_internal_fields(_, new_data), do: new_data defp reinject_object(%Object{data: %{"type" => "Question"}} = object, new_data) do Logger.debug("Reinjecting object #{new_data["id"]}") - with new_data <- Transmogrifier.fix_object(new_data), - data <- maybe_reinject_internal_fields(object, new_data), + with data <- maybe_reinject_internal_fields(object, new_data), {:ok, data, _} <- ObjectValidator.validate(data, %{}), changeset <- Object.change(object, %{data: data}), changeset <- touch_changeset(changeset), From 126461942b63bbb74900f296ebcee72d2a33f3d2 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 1 Sep 2020 09:25:32 +0300 Subject: [PATCH 012/128] User table: ensure bio is always a string Gets rid of '|| ""' in multiple places and fixes #2067 --- lib/pleroma/user.ex | 4 ++-- lib/pleroma/web/activity_pub/activity_pub.ex | 2 +- lib/pleroma/web/auth/pleroma_authenticator.ex | 2 +- lib/pleroma/web/mastodon_api/views/account_view.ex | 2 +- lib/pleroma/web/metadata/opengraph.ex | 2 +- lib/pleroma/web/metadata/twitter_card.ex | 2 +- .../migrations/20200901061256_ensure_bio_is_string.exs | 7 +++++++ .../migrations/20200901061637_bio_set_not_null.exs | 10 ++++++++++ test/user_test.exs | 2 +- .../controllers/admin_api_controller_test.exs | 2 +- test/web/twitter_api/util_controller_test.exs | 2 +- 11 files changed, 27 insertions(+), 10 deletions(-) create mode 100644 priv/repo/migrations/20200901061256_ensure_bio_is_string.exs create mode 100644 priv/repo/migrations/20200901061637_bio_set_not_null.exs diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index d2ad9516f..94c96de8d 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -83,7 +83,7 @@ defmodule Pleroma.User do ] schema "users" do - field(:bio, :string) + field(:bio, :string, default: "") field(:raw_bio, :string) field(:email, :string) field(:name, :string) @@ -1587,7 +1587,7 @@ def purge_user_changeset(user) do # "Right to be forgotten" # https://gdpr.eu/right-to-be-forgotten/ change(user, %{ - bio: nil, + bio: "", raw_bio: nil, email: nil, name: nil, diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 624a508ae..333621413 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1224,7 +1224,7 @@ defp object_to_user_data(data) do name: data["name"], follower_address: data["followers"], following_address: data["following"], - bio: data["summary"], + bio: data["summary"] || "", actor_type: actor_type, also_known_as: Map.get(data, "alsoKnownAs", []), public_key: public_key, diff --git a/lib/pleroma/web/auth/pleroma_authenticator.ex b/lib/pleroma/web/auth/pleroma_authenticator.ex index 200ca03dc..c611b3e09 100644 --- a/lib/pleroma/web/auth/pleroma_authenticator.ex +++ b/lib/pleroma/web/auth/pleroma_authenticator.ex @@ -68,7 +68,7 @@ def create_from_registration( nickname = value([registration_attrs["nickname"], Registration.nickname(registration)]) email = value([registration_attrs["email"], Registration.email(registration)]) name = value([registration_attrs["name"], Registration.name(registration)]) || nickname - bio = value([registration_attrs["bio"], Registration.description(registration)]) + bio = value([registration_attrs["bio"], Registration.description(registration)]) || "" random_password = :crypto.strong_rand_bytes(64) |> Base.encode64() diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 864c0417f..d2a30a548 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -245,7 +245,7 @@ defp do_render("show.json", %{user: user} = opts) do followers_count: followers_count, following_count: following_count, statuses_count: user.note_count, - note: user.bio || "", + note: user.bio, url: user.uri || user.ap_id, avatar: image, avatar_static: image, diff --git a/lib/pleroma/web/metadata/opengraph.ex b/lib/pleroma/web/metadata/opengraph.ex index 68c871e71..bb1b23208 100644 --- a/lib/pleroma/web/metadata/opengraph.ex +++ b/lib/pleroma/web/metadata/opengraph.ex @@ -61,7 +61,7 @@ def build_tags(%{ @impl Provider def build_tags(%{user: user}) do - with truncated_bio = Utils.scrub_html_and_truncate(user.bio || "") do + with truncated_bio = Utils.scrub_html_and_truncate(user.bio) do [ {:meta, [ diff --git a/lib/pleroma/web/metadata/twitter_card.ex b/lib/pleroma/web/metadata/twitter_card.ex index 5d08ce422..df34b033f 100644 --- a/lib/pleroma/web/metadata/twitter_card.ex +++ b/lib/pleroma/web/metadata/twitter_card.ex @@ -40,7 +40,7 @@ def build_tags(%{activity_id: id, object: object, user: user}) do @impl Provider def build_tags(%{user: user}) do - with truncated_bio = Utils.scrub_html_and_truncate(user.bio || "") do + with truncated_bio = Utils.scrub_html_and_truncate(user.bio) do [ title_tag(user), {:meta, [property: "twitter:description", content: truncated_bio], []}, diff --git a/priv/repo/migrations/20200901061256_ensure_bio_is_string.exs b/priv/repo/migrations/20200901061256_ensure_bio_is_string.exs new file mode 100644 index 000000000..0e3bb3c81 --- /dev/null +++ b/priv/repo/migrations/20200901061256_ensure_bio_is_string.exs @@ -0,0 +1,7 @@ +defmodule Pleroma.Repo.Migrations.EnsureBioIsString do + use Ecto.Migration + + def change do + execute("update users set bio = '' where bio is null", "") + end +end diff --git a/priv/repo/migrations/20200901061637_bio_set_not_null.exs b/priv/repo/migrations/20200901061637_bio_set_not_null.exs new file mode 100644 index 000000000..e3a67d4e7 --- /dev/null +++ b/priv/repo/migrations/20200901061637_bio_set_not_null.exs @@ -0,0 +1,10 @@ +defmodule Pleroma.Repo.Migrations.BioSetNotNull do + use Ecto.Migration + + def change do + execute( + "alter table users alter column bio set not null", + "alter table users alter column bio drop not null" + ) + end +end diff --git a/test/user_test.exs b/test/user_test.exs index 3cf248659..50f72549e 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -1466,7 +1466,7 @@ test "delete/1 purges a user when they wouldn't be fully deleted" do user = User.get_by_id(user.id) assert %User{ - bio: nil, + bio: "", raw_bio: nil, email: nil, name: nil, diff --git a/test/web/admin_api/controllers/admin_api_controller_test.exs b/test/web/admin_api/controllers/admin_api_controller_test.exs index dbf478edf..3bc88c6a9 100644 --- a/test/web/admin_api/controllers/admin_api_controller_test.exs +++ b/test/web/admin_api/controllers/admin_api_controller_test.exs @@ -203,7 +203,7 @@ test "single user", %{admin: admin, conn: conn} do assert user.note_count == 0 assert user.follower_count == 0 assert user.following_count == 0 - assert user.bio == nil + assert user.bio == "" assert user.name == nil assert called(Pleroma.Web.Federator.publish(:_)) diff --git a/test/web/twitter_api/util_controller_test.exs b/test/web/twitter_api/util_controller_test.exs index 354d77b56..d164127ee 100644 --- a/test/web/twitter_api/util_controller_test.exs +++ b/test/web/twitter_api/util_controller_test.exs @@ -594,7 +594,7 @@ test "with proper permissions and valid password", %{conn: conn, user: user} do user = User.get_by_id(user.id) assert user.deactivated == true assert user.name == nil - assert user.bio == nil + assert user.bio == "" assert user.password_hash == nil end end From d8728580468ecf876e531440fa31aef6a3e33f7b Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 1 Sep 2020 12:48:39 +0200 Subject: [PATCH 013/128] Fix removing an account from a list Mastodon (Frontend) changed a different method for deletes, keeping old format as mastodon documentation is too loose --- .../web/api_spec/operations/list_operation.ex | 20 +++++++++++++------ .../controllers/list_controller.ex | 13 ++++++++++++ .../controllers/list_controller_test.exs | 20 ++++++++++++++++++- 3 files changed, 46 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/web/api_spec/operations/list_operation.ex b/lib/pleroma/web/api_spec/operations/list_operation.ex index c88ed5dd0..15039052e 100644 --- a/lib/pleroma/web/api_spec/operations/list_operation.ex +++ b/lib/pleroma/web/api_spec/operations/list_operation.ex @@ -114,7 +114,7 @@ def add_to_list_operation do description: "Add accounts to the given list.", operationId: "ListController.add_to_list", parameters: [id_param()], - requestBody: add_remove_accounts_request(), + requestBody: add_remove_accounts_request(true), security: [%{"oAuth" => ["write:lists"]}], responses: %{ 200 => Operation.response("Empty object", "application/json", %Schema{type: :object}) @@ -127,8 +127,16 @@ def remove_from_list_operation do tags: ["Lists"], summary: "Remove accounts from list", operationId: "ListController.remove_from_list", - parameters: [id_param()], - requestBody: add_remove_accounts_request(), + parameters: [ + id_param(), + Operation.parameter( + :account_ids, + :query, + %Schema{type: :array, items: %Schema{type: :string}}, + "Array of account IDs" + ) + ], + requestBody: add_remove_accounts_request(false), security: [%{"oAuth" => ["write:lists"]}], responses: %{ 200 => Operation.response("Empty object", "application/json", %Schema{type: :object}) @@ -171,7 +179,7 @@ defp create_update_request do ) end - defp add_remove_accounts_request do + defp add_remove_accounts_request(required) when is_boolean(required) do request_body( "Parameters", %Schema{ @@ -180,9 +188,9 @@ defp add_remove_accounts_request do properties: %{ account_ids: %Schema{type: :array, description: "Array of account IDs", items: FlakeID} }, - required: [:account_ids] + required: required && [:account_ids] }, - required: true + required: required ) end end diff --git a/lib/pleroma/web/mastodon_api/controllers/list_controller.ex b/lib/pleroma/web/mastodon_api/controllers/list_controller.ex index acdc76fd2..4df13cb81 100644 --- a/lib/pleroma/web/mastodon_api/controllers/list_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/list_controller.ex @@ -86,6 +86,19 @@ def remove_from_list( json(conn, %{}) end + def remove_from_list( + %{assigns: %{list: list}, params: %{account_ids: account_ids}} = conn, + _ + ) do + Enum.each(account_ids, fn account_id -> + with %User{} = followed <- User.get_cached_by_id(account_id) do + Pleroma.List.unfollow(list, followed) + end + end) + + json(conn, %{}) + end + defp list_by_id_and_user(%{assigns: %{user: user}, params: %{id: id}} = conn, _) do case Pleroma.List.get(id, user) do %Pleroma.List{} = list -> assign(conn, :list, list) diff --git a/test/web/mastodon_api/controllers/list_controller_test.exs b/test/web/mastodon_api/controllers/list_controller_test.exs index 57a9ef4a4..091ec006c 100644 --- a/test/web/mastodon_api/controllers/list_controller_test.exs +++ b/test/web/mastodon_api/controllers/list_controller_test.exs @@ -67,7 +67,7 @@ test "adding users to a list" do assert following == [other_user.follower_address] end - test "removing users from a list" do + test "removing users from a list, body params" do %{user: user, conn: conn} = oauth_access(["write:lists"]) other_user = insert(:user) third_user = insert(:user) @@ -85,6 +85,24 @@ test "removing users from a list" do assert following == [third_user.follower_address] end + test "removing users from a list, query params" do + %{user: user, conn: conn} = oauth_access(["write:lists"]) + other_user = insert(:user) + third_user = insert(:user) + {:ok, list} = Pleroma.List.create("name", user) + {:ok, list} = Pleroma.List.follow(list, other_user) + {:ok, list} = Pleroma.List.follow(list, third_user) + + assert %{} == + conn + |> put_req_header("content-type", "application/json") + |> delete("/api/v1/lists/#{list.id}/accounts?account_ids[]=#{other_user.id}") + |> json_response_and_validate_schema(:ok) + + %Pleroma.List{following: following} = Pleroma.List.get(list.id, user) + assert following == [third_user.follower_address] + end + test "listing users in a list" do %{user: user, conn: conn} = oauth_access(["read:lists"]) other_user = insert(:user) From 03d06062ab8feffbf7b03ecb7ff86c4a42af79ef Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 1 Sep 2020 19:12:45 +0300 Subject: [PATCH 014/128] don't fail on url fetch --- lib/pleroma/web/rich_media/parser.ex | 44 ++++++++++--------- .../rich_media/parsers/ttl/aws_signed_url.ex | 15 +++---- test/web/rich_media/aws_signed_url_test.exs | 2 +- test/web/rich_media/parser_test.exs | 26 ++++++++--- 4 files changed, 49 insertions(+), 38 deletions(-) diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex index ca592833f..e9aa2dd03 100644 --- a/lib/pleroma/web/rich_media/parser.ex +++ b/lib/pleroma/web/rich_media/parser.ex @@ -3,6 +3,8 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.RichMedia.Parser do + require Logger + defp parsers do Pleroma.Config.get([:rich_media, :parsers]) end @@ -10,18 +12,19 @@ defp parsers do def parse(nil), do: {:error, "No URL provided"} if Pleroma.Config.get(:env) == :test do + @spec parse(String.t()) :: {:ok, map()} | {:error, any()} def parse(url), do: parse_url(url) else + @spec parse(String.t()) :: {:ok, map()} | {:error, any()} def parse(url) do - try do - Cachex.fetch!(:rich_media_cache, url, fn _ -> - {:commit, parse_url(url)} - end) - |> set_ttl_based_on_image(url) - rescue - e -> - {:error, "Cachex error: #{inspect(e)}"} - end + Cachex.fetch!(:rich_media_cache, url, fn _ -> + with {:ok, data} <- parse_url(url) do + {:commit, {:ok, data}} + else + error -> {:ignore, error} + end + end) + |> set_ttl_based_on_image(url) end end @@ -47,9 +50,11 @@ def ttl(data, url) do config :pleroma, :rich_media, ttl_setters: [MyModule] """ + @spec set_ttl_based_on_image({:ok, map()} | {:error, any()}, String.t()) :: + {:ok, map()} | {:error, any()} def set_ttl_based_on_image({:ok, data}, url) do with {:ok, nil} <- Cachex.ttl(:rich_media_cache, url), - ttl when is_number(ttl) <- get_ttl_from_image(data, url) do + {:ok, ttl} when is_number(ttl) <- get_ttl_from_image(data, url) do Cachex.expire_at(:rich_media_cache, url, ttl * 1000) {:ok, data} else @@ -58,8 +63,14 @@ def set_ttl_based_on_image({:ok, data}, url) do end end + def set_ttl_based_on_image({:error, _} = error, _) do + Logger.error("parsing error: #{inspect(error)}") + error + end + defp get_ttl_from_image(data, url) do - Pleroma.Config.get([:rich_media, :ttl_setters]) + [:rich_media, :ttl_setters] + |> Pleroma.Config.get() |> Enum.reduce({:ok, nil}, fn module, {:ok, _ttl} -> module.ttl(data, url) @@ -70,23 +81,16 @@ defp get_ttl_from_image(data, url) do end defp parse_url(url) do - try do - {:ok, %Tesla.Env{body: html}} = Pleroma.Web.RichMedia.Helpers.rich_media_get(url) - + with {:ok, %Tesla.Env{body: html}} <- Pleroma.Web.RichMedia.Helpers.rich_media_get(url), + {:ok, html} <- Floki.parse_document(html) do html - |> parse_html() |> maybe_parse() |> Map.put("url", url) |> clean_parsed_data() |> check_parsed_data() - rescue - e -> - {:error, "Parsing error: #{inspect(e)} #{inspect(__STACKTRACE__)}"} end end - defp parse_html(html), do: Floki.parse_document!(html) - defp maybe_parse(html) do Enum.reduce_while(parsers(), %{}, fn parser, acc -> case parser.parse(html, acc) do diff --git a/lib/pleroma/web/rich_media/parsers/ttl/aws_signed_url.ex b/lib/pleroma/web/rich_media/parsers/ttl/aws_signed_url.ex index 0dc1efdaf..c5aaea2d4 100644 --- a/lib/pleroma/web/rich_media/parsers/ttl/aws_signed_url.ex +++ b/lib/pleroma/web/rich_media/parsers/ttl/aws_signed_url.ex @@ -10,20 +10,15 @@ def ttl(data, _url) do |> parse_query_params() |> format_query_params() |> get_expiration_timestamp() + else + {:error, "Not aws signed url #{inspect(image)}"} end end - defp is_aws_signed_url(""), do: nil - defp is_aws_signed_url(nil), do: nil - - defp is_aws_signed_url(image) when is_binary(image) do + defp is_aws_signed_url(image) when is_binary(image) and image != "" do %URI{host: host, query: query} = URI.parse(image) - if String.contains?(host, "amazonaws.com") and String.contains?(query, "X-Amz-Expires") do - image - else - nil - end + String.contains?(host, "amazonaws.com") and String.contains?(query, "X-Amz-Expires") end defp is_aws_signed_url(_), do: nil @@ -46,6 +41,6 @@ defp get_expiration_timestamp(params) when is_map(params) do |> Map.get("X-Amz-Date") |> Timex.parse("{ISO:Basic:Z}") - Timex.to_unix(date) + String.to_integer(Map.get(params, "X-Amz-Expires")) + {:ok, Timex.to_unix(date) + String.to_integer(Map.get(params, "X-Amz-Expires"))} end end diff --git a/test/web/rich_media/aws_signed_url_test.exs b/test/web/rich_media/aws_signed_url_test.exs index b30f4400e..a21f3c935 100644 --- a/test/web/rich_media/aws_signed_url_test.exs +++ b/test/web/rich_media/aws_signed_url_test.exs @@ -21,7 +21,7 @@ test "s3 signed url is parsed correct for expiration time" do expire_time = Timex.parse!(timestamp, "{ISO:Basic:Z}") |> Timex.to_unix() |> Kernel.+(valid_till) - assert expire_time == Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl.ttl(metadata, url) + assert {:ok, expire_time} == Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl.ttl(metadata, url) end test "s3 signed url is parsed and correct ttl is set for rich media" do diff --git a/test/web/rich_media/parser_test.exs b/test/web/rich_media/parser_test.exs index 420a612c6..1e09cbf84 100644 --- a/test/web/rich_media/parser_test.exs +++ b/test/web/rich_media/parser_test.exs @@ -5,6 +5,8 @@ defmodule Pleroma.Web.RichMedia.ParserTest do use ExUnit.Case, async: true + alias Pleroma.Web.RichMedia.Parser + setup do Tesla.Mock.mock(fn %{ @@ -48,23 +50,29 @@ defmodule Pleroma.Web.RichMedia.ParserTest do %{method: :get, url: "http://example.com/empty"} -> %Tesla.Env{status: 200, body: "hello"} + + %{method: :get, url: "http://example.com/malformed"} -> + %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/malformed-data.html")} + + %{method: :get, url: "http://example.com/error"} -> + {:error, :overload} end) :ok end test "returns error when no metadata present" do - assert {:error, _} = Pleroma.Web.RichMedia.Parser.parse("http://example.com/empty") + assert {:error, _} = Parser.parse("http://example.com/empty") end test "doesn't just add a title" do - assert Pleroma.Web.RichMedia.Parser.parse("http://example.com/non-ogp") == + assert Parser.parse("http://example.com/non-ogp") == {:error, "Found metadata was invalid or incomplete: %{\"url\" => \"http://example.com/non-ogp\"}"} end test "parses ogp" do - assert Pleroma.Web.RichMedia.Parser.parse("http://example.com/ogp") == + assert Parser.parse("http://example.com/ogp") == {:ok, %{ "image" => "http://ia.media-imdb.com/images/rock.jpg", @@ -77,7 +85,7 @@ test "parses ogp" do end test "falls back to when ogp:title is missing" do - assert Pleroma.Web.RichMedia.Parser.parse("http://example.com/ogp-missing-title") == + assert Parser.parse("http://example.com/ogp-missing-title") == {:ok, %{ "image" => "http://ia.media-imdb.com/images/rock.jpg", @@ -90,7 +98,7 @@ test "falls back to <title> when ogp:title is missing" do end test "parses twitter card" do - assert Pleroma.Web.RichMedia.Parser.parse("http://example.com/twitter-card") == + assert Parser.parse("http://example.com/twitter-card") == {:ok, %{ "card" => "summary", @@ -103,7 +111,7 @@ test "parses twitter card" do end test "parses OEmbed" do - assert Pleroma.Web.RichMedia.Parser.parse("http://example.com/oembed") == + assert Parser.parse("http://example.com/oembed") == {:ok, %{ "author_name" => "‮‭‬bees‬", @@ -132,6 +140,10 @@ test "parses OEmbed" do end test "rejects invalid OGP data" do - assert {:error, _} = Pleroma.Web.RichMedia.Parser.parse("http://example.com/malformed") + assert {:error, _} = Parser.parse("http://example.com/malformed") + end + + test "returns error if getting page was not successful" do + assert {:error, :overload} = Parser.parse("http://example.com/error") end end From 0a9c63fb4351ed29a521697f2c584b0ae007696c Mon Sep 17 00:00:00 2001 From: Sean King <seanking2919@protonmail.com> Date: Tue, 1 Sep 2020 12:20:32 -0600 Subject: [PATCH 015/128] Fix frontend install mix task bug --- lib/mix/tasks/pleroma/frontend.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/mix/tasks/pleroma/frontend.ex b/lib/mix/tasks/pleroma/frontend.ex index 2adbf8d72..0a48be1fe 100644 --- a/lib/mix/tasks/pleroma/frontend.ex +++ b/lib/mix/tasks/pleroma/frontend.ex @@ -69,7 +69,7 @@ def run(["install", frontend | args]) do fe_label = "#{frontend} (#{ref})" - tmp_dir = Path.join(dest, "tmp") + tmp_dir = Path.join([instance_static_dir, "frontends", "tmp"]) with {_, :ok} <- {:download_or_unzip, download_or_unzip(frontend_info, tmp_dir, options[:file])}, @@ -133,6 +133,7 @@ defp download_build(frontend_info, dest) do defp install_frontend(frontend_info, source, dest) do from = frontend_info["build_dir"] || "dist" + File.rm_rf!(dest) File.mkdir_p!(dest) File.cp_r!(Path.join([source, from]), dest) :ok From 868057871ac041346d8367181f00f0b127b33945 Mon Sep 17 00:00:00 2001 From: Karol Kosek <krkk@krkk.ct8.pl> Date: Tue, 1 Sep 2020 19:56:32 +0200 Subject: [PATCH 016/128] search: fix 'following' query parameter The parameter included the accounts that are following you (followers) instead of those you are actually following. Co-Authored-By: Haelwenn (lanodan) Monnier <contact@hacktivis.me> --- CHANGELOG.md | 5 +++++ lib/pleroma/user/search.ex | 2 +- test/user_search_test.exs | 24 ++++++++++++------------ 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0850deed7..07bc6d77c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## unreleased-patch - ??? + +### Fixed +- Mastodon API: Search parameter `following` now correctly returns the followings rather than the followers + ## [2.1.0] - 2020-08-28 ### Changed diff --git a/lib/pleroma/user/search.ex b/lib/pleroma/user/search.ex index d4fd31069..adbef7fb8 100644 --- a/lib/pleroma/user/search.ex +++ b/lib/pleroma/user/search.ex @@ -116,7 +116,7 @@ defp trigram_rank(query, query_string) do end defp base_query(_user, false), do: User - defp base_query(user, true), do: User.get_followers_query(user) + defp base_query(user, true), do: User.get_friends_query(user) defp filter_invisible_users(query) do from(q in query, where: q.invisible == false) diff --git a/test/user_search_test.exs b/test/user_search_test.exs index 559ba5966..01976bf58 100644 --- a/test/user_search_test.exs +++ b/test/user_search_test.exs @@ -109,22 +109,22 @@ test "finds users, boosting ranks of friends and followers" do Enum.map(User.search("doe", resolve: false, for_user: u1), & &1.id) == [] end - test "finds followers of user by partial name" do - u1 = insert(:user) - u2 = insert(:user, %{name: "Jimi"}) - follower_jimi = insert(:user, %{name: "Jimi Hendrix"}) - follower_lizz = insert(:user, %{name: "Lizz Wright"}) - friend = insert(:user, %{name: "Jimi"}) + test "finds followings of user by partial name" do + lizz = insert(:user, %{name: "Lizz"}) + jimi = insert(:user, %{name: "Jimi"}) + following_lizz = insert(:user, %{name: "Jimi Hendrix"}) + following_jimi = insert(:user, %{name: "Lizz Wright"}) + follower_lizz = insert(:user, %{name: "Jimi"}) - {:ok, follower_jimi} = User.follow(follower_jimi, u1) - {:ok, _follower_lizz} = User.follow(follower_lizz, u2) - {:ok, u1} = User.follow(u1, friend) + {:ok, lizz} = User.follow(lizz, following_lizz) + {:ok, _jimi} = User.follow(jimi, following_jimi) + {:ok, _follower_lizz} = User.follow(follower_lizz, lizz) - assert Enum.map(User.search("jimi", following: true, for_user: u1), & &1.id) == [ - follower_jimi.id + assert Enum.map(User.search("jimi", following: true, for_user: lizz), & &1.id) == [ + following_lizz.id ] - assert User.search("lizz", following: true, for_user: u1) == [] + assert User.search("lizz", following: true, for_user: lizz) == [] end test "find local and remote users for authenticated users" do From c17d83cd7330d5c874f647a5224a3a130ed88fb0 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov <alex.strizhakov@gmail.com> Date: Wed, 19 Aug 2020 10:22:59 +0300 Subject: [PATCH 017/128] improvements and fixes for http requests - fix for gun worker termination in some circumstances - pool for http clients (ex_aws, tzdata) - default pool timeouts for gun - gun retries on gun_down messages - s3 upload timeout if streaming enabled --- config/config.exs | 12 +++++--- lib/pleroma/gun/connection_pool/worker.ex | 35 ++++++++++++++++------- lib/pleroma/http/adapter_helper.ex | 16 +++++++---- lib/pleroma/http/adapter_helper/gun.ex | 18 ++++++------ lib/pleroma/http/ex_aws.ex | 2 ++ lib/pleroma/http/tzdata.ex | 4 +++ lib/pleroma/uploaders/s3.ex | 19 ++++++++---- 7 files changed, 70 insertions(+), 36 deletions(-) diff --git a/config/config.exs b/config/config.exs index 246712b9f..ea8869d48 100644 --- a/config/config.exs +++ b/config/config.exs @@ -740,19 +740,23 @@ config :pleroma, :pools, federation: [ size: 50, - max_waiting: 10 + max_waiting: 10, + timeout: 10_000 ], media: [ size: 50, - max_waiting: 10 + max_waiting: 10, + timeout: 10_000 ], upload: [ size: 25, - max_waiting: 5 + max_waiting: 5, + timeout: 15_000 ], default: [ size: 10, - max_waiting: 2 + max_waiting: 2, + timeout: 5_000 ] config :pleroma, :hackney_pools, diff --git a/lib/pleroma/gun/connection_pool/worker.ex b/lib/pleroma/gun/connection_pool/worker.ex index fec9d0efa..c36332817 100644 --- a/lib/pleroma/gun/connection_pool/worker.ex +++ b/lib/pleroma/gun/connection_pool/worker.ex @@ -83,17 +83,25 @@ def handle_call(:remove_client, {client_pid, _}, %{key: key} = state) do end) {ref, state} = pop_in(state.client_monitors[client_pid]) - Process.demonitor(ref) - - timer = - if used_by == [] do - max_idle = Pleroma.Config.get([:connections_pool, :max_idle_time], 30_000) - Process.send_after(self(), :idle_close, max_idle) + # DOWN message can receive right after `remove_client` call and cause worker to terminate + state = + if is_nil(ref) do + state else - nil + Process.demonitor(ref) + + timer = + if used_by == [] do + max_idle = Pleroma.Config.get([:connections_pool, :max_idle_time], 30_000) + Process.send_after(self(), :idle_close, max_idle) + else + nil + end + + %{state | timer: timer} end - {:reply, :ok, %{state | timer: timer}, :hibernate} + {:reply, :ok, state, :hibernate} end @impl true @@ -103,16 +111,21 @@ def handle_info(:idle_close, state) do {:stop, :normal, state} end + @impl true + def handle_info({:gun_up, _pid, _protocol}, state) do + {:noreply, state, :hibernate} + end + # Gracefully shutdown if the connection got closed without any streams left @impl true def handle_info({:gun_down, _pid, _protocol, _reason, []}, state) do {:stop, :normal, state} end - # Otherwise, shutdown with an error + # Otherwise, wait for retry @impl true - def handle_info({:gun_down, _pid, _protocol, _reason, _killed_streams} = down_message, state) do - {:stop, {:error, down_message}, state} + def handle_info({:gun_down, _pid, _protocol, _reason, _killed_streams}, state) do + {:noreply, state, :hibernate} end @impl true diff --git a/lib/pleroma/http/adapter_helper.ex b/lib/pleroma/http/adapter_helper.ex index 9ec3836b0..740e6e9ff 100644 --- a/lib/pleroma/http/adapter_helper.ex +++ b/lib/pleroma/http/adapter_helper.ex @@ -10,6 +10,7 @@ defmodule Pleroma.HTTP.AdapterHelper do @type proxy_type() :: :socks4 | :socks5 @type host() :: charlist() | :inet.ip_address() + @type pool() :: :federation | :upload | :media | :default alias Pleroma.Config alias Pleroma.HTTP.AdapterHelper @@ -44,14 +45,13 @@ def maybe_add_proxy(opts, proxy), do: Keyword.put_new(opts, :proxy, proxy) @spec options(URI.t(), keyword()) :: keyword() def options(%URI{} = uri, opts \\ []) do @defaults - |> put_timeout() |> Keyword.merge(opts) + |> put_timeout() |> adapter_helper().options(uri) end - # For Hackney, this is the time a connection can stay idle in the pool. - # For Gun, this is the timeout to receive a message from Gun. - defp put_timeout(opts) do + @spec pool_timeout(pool()) :: non_neg_integer() + def pool_timeout(pool) do {config_key, default} = if adapter() == Tesla.Adapter.Gun do {:pools, Config.get([:pools, :default, :timeout], 5_000)} @@ -59,9 +59,13 @@ defp put_timeout(opts) do {:hackney_pools, 10_000} end - timeout = Config.get([config_key, opts[:pool], :timeout], default) + Config.get([config_key, pool, :timeout], default) + end - Keyword.merge(opts, timeout: timeout) + # For Hackney, this is the time a connection can stay idle in the pool. + # For Gun, this is the timeout to receive a message from Gun. + defp put_timeout(opts) do + Keyword.put_new(opts, :timeout, pool_timeout(opts[:pool])) end def get_conn(uri, opts), do: adapter_helper().get_conn(uri, opts) diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex index b4ff8306c..db0a298b3 100644 --- a/lib/pleroma/http/adapter_helper/gun.ex +++ b/lib/pleroma/http/adapter_helper/gun.ex @@ -5,6 +5,7 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do @behaviour Pleroma.HTTP.AdapterHelper + alias Pleroma.Config alias Pleroma.Gun.ConnectionPool alias Pleroma.HTTP.AdapterHelper @@ -14,7 +15,7 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do connect_timeout: 5_000, domain_lookup_timeout: 5_000, tls_handshake_timeout: 5_000, - retry: 0, + retry: 1, retry_timeout: 1000, await_up_timeout: 5_000 ] @@ -22,10 +23,11 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do @spec options(keyword(), URI.t()) :: keyword() def options(incoming_opts \\ [], %URI{} = uri) do proxy = - Pleroma.Config.get([:http, :proxy_url]) + [:http, :proxy_url] + |> Config.get() |> AdapterHelper.format_proxy() - config_opts = Pleroma.Config.get([:http, :adapter], []) + config_opts = Config.get([:http, :adapter], []) @defaults |> Keyword.merge(config_opts) @@ -37,8 +39,7 @@ def options(incoming_opts \\ [], %URI{} = uri) do defp add_scheme_opts(opts, %{scheme: "http"}), do: opts defp add_scheme_opts(opts, %{scheme: "https"}) do - opts - |> Keyword.put(:certificates_verification, true) + Keyword.put(opts, :certificates_verification, true) end @spec get_conn(URI.t(), keyword()) :: {:ok, keyword()} | {:error, atom()} @@ -51,11 +52,11 @@ def get_conn(uri, opts) do @prefix Pleroma.Gun.ConnectionPool def limiter_setup do - wait = Pleroma.Config.get([:connections_pool, :connection_acquisition_wait]) - retries = Pleroma.Config.get([:connections_pool, :connection_acquisition_retries]) + wait = Config.get([:connections_pool, :connection_acquisition_wait]) + retries = Config.get([:connections_pool, :connection_acquisition_retries]) :pools - |> Pleroma.Config.get([]) + |> Config.get([]) |> Enum.each(fn {name, opts} -> max_running = Keyword.get(opts, :size, 50) max_waiting = Keyword.get(opts, :max_waiting, 10) @@ -69,7 +70,6 @@ def limiter_setup do case result do :ok -> :ok {:error, :existing} -> :ok - e -> raise e end end) diff --git a/lib/pleroma/http/ex_aws.ex b/lib/pleroma/http/ex_aws.ex index e53e64077..2fe8beafc 100644 --- a/lib/pleroma/http/ex_aws.ex +++ b/lib/pleroma/http/ex_aws.ex @@ -11,6 +11,8 @@ defmodule Pleroma.HTTP.ExAws do @impl true def request(method, url, body \\ "", headers \\ [], http_opts \\ []) do + http_opts = Keyword.put(http_opts, :adapter, pool: :upload) + case HTTP.request(method, url, body, headers, http_opts) do {:ok, env} -> {:ok, %{status_code: env.status, headers: env.headers, body: env.body}} diff --git a/lib/pleroma/http/tzdata.ex b/lib/pleroma/http/tzdata.ex index 34bb253a7..7dd3352ff 100644 --- a/lib/pleroma/http/tzdata.ex +++ b/lib/pleroma/http/tzdata.ex @@ -11,6 +11,8 @@ defmodule Pleroma.HTTP.Tzdata do @impl true def get(url, headers, options) do + options = Keyword.put(options, :adapter, pool: :upload) + with {:ok, %Tesla.Env{} = env} <- HTTP.get(url, headers, options) do {:ok, {env.status, env.headers, env.body}} end @@ -18,6 +20,8 @@ def get(url, headers, options) do @impl true def head(url, headers, options) do + options = Keyword.put(options, :adapter, pool: :upload) + with {:ok, %Tesla.Env{} = env} <- HTTP.head(url, headers, options) do {:ok, {env.status, env.headers}} end diff --git a/lib/pleroma/uploaders/s3.ex b/lib/pleroma/uploaders/s3.ex index a13ff23b6..ed9794ca2 100644 --- a/lib/pleroma/uploaders/s3.ex +++ b/lib/pleroma/uploaders/s3.ex @@ -46,12 +46,19 @@ def put_file(%Pleroma.Upload{} = upload) do op = if streaming do - upload.tempfile - |> ExAws.S3.Upload.stream_file() - |> ExAws.S3.upload(bucket, s3_name, [ - {:acl, :public_read}, - {:content_type, upload.content_type} - ]) + op = + upload.tempfile + |> ExAws.S3.Upload.stream_file() + |> ExAws.S3.upload(bucket, s3_name, [ + {:acl, :public_read}, + {:content_type, upload.content_type} + ]) + + # set s3 upload timeout to respect :upload pool timeout + # timeout should be slightly larger, so s3 can retry upload on fail + timeout = Pleroma.HTTP.AdapterHelper.pool_timeout(:upload) + 1_000 + opts = Keyword.put(op.opts, :timeout, timeout) + Map.put(op, :opts, opts) else {:ok, file_data} = File.read(upload.tempfile) From 5e8adf91b46c3f23ec423d53afccbb062df4a241 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov <alex.strizhakov@gmail.com> Date: Tue, 1 Sep 2020 11:30:56 +0300 Subject: [PATCH 018/128] don't overwrite passed pool option in http clients --- lib/pleroma/http/ex_aws.ex | 2 +- lib/pleroma/http/tzdata.ex | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/http/ex_aws.ex b/lib/pleroma/http/ex_aws.ex index 2fe8beafc..c3f335c73 100644 --- a/lib/pleroma/http/ex_aws.ex +++ b/lib/pleroma/http/ex_aws.ex @@ -11,7 +11,7 @@ defmodule Pleroma.HTTP.ExAws do @impl true def request(method, url, body \\ "", headers \\ [], http_opts \\ []) do - http_opts = Keyword.put(http_opts, :adapter, pool: :upload) + http_opts = Keyword.put_new(http_opts, :adapter, pool: :upload) case HTTP.request(method, url, body, headers, http_opts) do {:ok, env} -> diff --git a/lib/pleroma/http/tzdata.ex b/lib/pleroma/http/tzdata.ex index 7dd3352ff..356799aab 100644 --- a/lib/pleroma/http/tzdata.ex +++ b/lib/pleroma/http/tzdata.ex @@ -11,7 +11,7 @@ defmodule Pleroma.HTTP.Tzdata do @impl true def get(url, headers, options) do - options = Keyword.put(options, :adapter, pool: :upload) + options = Keyword.put_new(options, :adapter, pool: :upload) with {:ok, %Tesla.Env{} = env} <- HTTP.get(url, headers, options) do {:ok, {env.status, env.headers, env.body}} @@ -20,7 +20,7 @@ def get(url, headers, options) do @impl true def head(url, headers, options) do - options = Keyword.put(options, :adapter, pool: :upload) + options = Keyword.put_new(options, :adapter, pool: :upload) with {:ok, %Tesla.Env{} = env} <- HTTP.head(url, headers, options) do {:ok, {env.status, env.headers}} From 79f65b4374908a32ebf39db176a30a01152a9141 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov <alex.strizhakov@gmail.com> Date: Wed, 2 Sep 2020 09:16:51 +0300 Subject: [PATCH 019/128] correct pool and uniform headers format --- lib/mix/tasks/pleroma/frontend.ex | 4 +++- lib/pleroma/instances/instance.ex | 4 +++- lib/pleroma/object/fetcher.ex | 6 +++--- lib/pleroma/web/rich_media/helpers.ex | 2 +- lib/pleroma/web/web_finger/web_finger.ex | 4 ++-- test/support/http_request_mock.ex | 4 ++-- 6 files changed, 14 insertions(+), 10 deletions(-) diff --git a/lib/mix/tasks/pleroma/frontend.ex b/lib/mix/tasks/pleroma/frontend.ex index 2adbf8d72..484af6da7 100644 --- a/lib/mix/tasks/pleroma/frontend.ex +++ b/lib/mix/tasks/pleroma/frontend.ex @@ -124,7 +124,9 @@ defp download_build(frontend_info, dest) do url = String.replace(frontend_info["build_url"], "${ref}", frontend_info["ref"]) with {:ok, %{status: 200, body: zip_body}} <- - Pleroma.HTTP.get(url, [], timeout: 120_000, recv_timeout: 120_000) do + Pleroma.HTTP.get(url, [], + adapter: [pool: :media, timeout: 120_000, recv_timeout: 120_000] + ) do unzip(zip_body, dest) else e -> {:error, e} diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index a1f935232..711c42158 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -150,7 +150,9 @@ def get_or_update_favicon(%URI{host: host} = instance_uri) do defp scrape_favicon(%URI{} = instance_uri) do try do with {:ok, %Tesla.Env{body: html}} <- - Pleroma.HTTP.get(to_string(instance_uri), [{:Accept, "text/html"}]), + Pleroma.HTTP.get(to_string(instance_uri), [{"accept", "text/html"}], + adapter: [pool: :media] + ), favicon_rel <- html |> Floki.parse_document!() diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 6fdbc8efd..374d8704a 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -164,12 +164,12 @@ defp make_signature(id, date) do date: date }) - [{"signature", signature}] + {"signature", signature} end defp sign_fetch(headers, id, date) do if Pleroma.Config.get([:activitypub, :sign_object_fetches]) do - headers ++ make_signature(id, date) + [make_signature(id, date) | headers] else headers end @@ -177,7 +177,7 @@ defp sign_fetch(headers, id, date) do defp maybe_date_fetch(headers, date) do if Pleroma.Config.get([:activitypub, :sign_object_fetches]) do - headers ++ [{"date", date}] + [{"date", date} | headers] else headers end diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index 6210f2c5a..2fb482b51 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -96,6 +96,6 @@ def rich_media_get(url) do @rich_media_options end - Pleroma.HTTP.get(url, headers, options) + Pleroma.HTTP.get(url, headers, adapter: options) end end diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index c4051e63e..6629f5356 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -136,12 +136,12 @@ def get_template_from_xml(body) do def find_lrdd_template(domain) do with {:ok, %{status: status, body: body}} when status in 200..299 <- - HTTP.get("http://#{domain}/.well-known/host-meta", []) do + HTTP.get("http://#{domain}/.well-known/host-meta") do get_template_from_xml(body) else _ -> with {:ok, %{body: body, status: status}} when status in 200..299 <- - HTTP.get("https://#{domain}/.well-known/host-meta", []) do + HTTP.get("https://#{domain}/.well-known/host-meta") do get_template_from_xml(body) else e -> {:error, "Can't find LRDD template: #{inspect(e)}"} diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex index eeeba7880..a0ebf65d9 100644 --- a/test/support/http_request_mock.ex +++ b/test/support/http_request_mock.ex @@ -1350,11 +1350,11 @@ def get("https://relay.mastodon.host/actor", _, _, _) do {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/relay/relay.json")}} end - def get("http://localhost:4001/", _, "", Accept: "text/html") do + def get("http://localhost:4001/", _, "", [{"accept", "text/html"}]) do {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/7369654.html")}} end - def get("https://osada.macgirvin.com/", _, "", Accept: "text/html") do + def get("https://osada.macgirvin.com/", _, "", [{"accept", "text/html"}]) do {:ok, %Tesla.Env{ status: 200, From 1c57ef44983e150f3cc016290fe99f159eb79cb0 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov <alex.strizhakov@gmail.com> Date: Wed, 2 Sep 2020 10:33:43 +0300 Subject: [PATCH 020/128] default pool for tz_data client --- lib/pleroma/http/tzdata.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/http/tzdata.ex b/lib/pleroma/http/tzdata.ex index 356799aab..4539ac359 100644 --- a/lib/pleroma/http/tzdata.ex +++ b/lib/pleroma/http/tzdata.ex @@ -11,7 +11,7 @@ defmodule Pleroma.HTTP.Tzdata do @impl true def get(url, headers, options) do - options = Keyword.put_new(options, :adapter, pool: :upload) + options = Keyword.put_new(options, :adapter, pool: :default) with {:ok, %Tesla.Env{} = env} <- HTTP.get(url, headers, options) do {:ok, {env.status, env.headers, env.body}} @@ -20,7 +20,7 @@ def get(url, headers, options) do @impl true def head(url, headers, options) do - options = Keyword.put_new(options, :adapter, pool: :upload) + options = Keyword.put_new(options, :adapter, pool: :default) with {:ok, %Tesla.Env{} = env} <- HTTP.head(url, headers, options) do {:ok, {env.status, env.headers}} From 84fbf1616104c09e0f4f5442d86ca2c573ae4056 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov <alex.strizhakov@gmail.com> Date: Wed, 2 Sep 2020 10:50:51 +0300 Subject: [PATCH 021/128] timeout option moved to gun adapter helper --- lib/pleroma/http/adapter_helper.ex | 23 ++--------------------- lib/pleroma/http/adapter_helper/gun.ex | 15 +++++++++++++++ lib/pleroma/uploaders/s3.ex | 14 +++++++++----- 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/lib/pleroma/http/adapter_helper.ex b/lib/pleroma/http/adapter_helper.ex index 740e6e9ff..0728cbaa2 100644 --- a/lib/pleroma/http/adapter_helper.ex +++ b/lib/pleroma/http/adapter_helper.ex @@ -10,9 +10,7 @@ defmodule Pleroma.HTTP.AdapterHelper do @type proxy_type() :: :socks4 | :socks5 @type host() :: charlist() | :inet.ip_address() - @type pool() :: :federation | :upload | :media | :default - alias Pleroma.Config alias Pleroma.HTTP.AdapterHelper require Logger @@ -46,29 +44,12 @@ def maybe_add_proxy(opts, proxy), do: Keyword.put_new(opts, :proxy, proxy) def options(%URI{} = uri, opts \\ []) do @defaults |> Keyword.merge(opts) - |> put_timeout() |> adapter_helper().options(uri) end - @spec pool_timeout(pool()) :: non_neg_integer() - def pool_timeout(pool) do - {config_key, default} = - if adapter() == Tesla.Adapter.Gun do - {:pools, Config.get([:pools, :default, :timeout], 5_000)} - else - {:hackney_pools, 10_000} - end - - Config.get([config_key, pool, :timeout], default) - end - - # For Hackney, this is the time a connection can stay idle in the pool. - # For Gun, this is the timeout to receive a message from Gun. - defp put_timeout(opts) do - Keyword.put_new(opts, :timeout, pool_timeout(opts[:pool])) - end - + @spec get_conn(URI.t(), keyword()) :: {:ok, keyword()} | {:error, atom()} def get_conn(uri, opts), do: adapter_helper().get_conn(uri, opts) + defp adapter, do: Application.get_env(:tesla, :adapter) defp adapter_helper do diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex index db0a298b3..02e20f2d1 100644 --- a/lib/pleroma/http/adapter_helper/gun.ex +++ b/lib/pleroma/http/adapter_helper/gun.ex @@ -20,6 +20,8 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do await_up_timeout: 5_000 ] + @type pool() :: :federation | :upload | :media | :default + @spec options(keyword(), URI.t()) :: keyword() def options(incoming_opts \\ [], %URI{} = uri) do proxy = @@ -34,6 +36,7 @@ def options(incoming_opts \\ [], %URI{} = uri) do |> add_scheme_opts(uri) |> AdapterHelper.maybe_add_proxy(proxy) |> Keyword.merge(incoming_opts) + |> put_timeout() end defp add_scheme_opts(opts, %{scheme: "http"}), do: opts @@ -42,6 +45,18 @@ defp add_scheme_opts(opts, %{scheme: "https"}) do Keyword.put(opts, :certificates_verification, true) end + defp put_timeout(opts) do + # this is the timeout to receive a message from Gun + Keyword.put_new(opts, :timeout, pool_timeout(opts[:pool])) + end + + @spec pool_timeout(pool()) :: non_neg_integer() + def pool_timeout(pool) do + default = Config.get([:pools, :default, :timeout], 5_000) + + Config.get([:pools, pool, :timeout], default) + end + @spec get_conn(URI.t(), keyword()) :: {:ok, keyword()} | {:error, atom()} def get_conn(uri, opts) do case ConnectionPool.get_conn(uri, opts) do diff --git a/lib/pleroma/uploaders/s3.ex b/lib/pleroma/uploaders/s3.ex index ed9794ca2..6dbef9085 100644 --- a/lib/pleroma/uploaders/s3.ex +++ b/lib/pleroma/uploaders/s3.ex @@ -54,11 +54,15 @@ def put_file(%Pleroma.Upload{} = upload) do {:content_type, upload.content_type} ]) - # set s3 upload timeout to respect :upload pool timeout - # timeout should be slightly larger, so s3 can retry upload on fail - timeout = Pleroma.HTTP.AdapterHelper.pool_timeout(:upload) + 1_000 - opts = Keyword.put(op.opts, :timeout, timeout) - Map.put(op, :opts, opts) + if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun do + # set s3 upload timeout to respect :upload pool timeout + # timeout should be slightly larger, so s3 can retry upload on fail + timeout = Pleroma.HTTP.AdapterHelper.Gun.pool_timeout(:upload) + 1_000 + opts = Keyword.put(op.opts, :timeout, timeout) + Map.put(op, :opts, opts) + else + op + end else {:ok, file_data} = File.read(upload.tempfile) From 46236d1d873473d95b11cd7bfdcaa284ea55a9ad Mon Sep 17 00:00:00 2001 From: rinpatch <rinpatch@sdf.org> Date: Wed, 2 Sep 2020 12:42:25 +0300 Subject: [PATCH 022/128] html.ex: optimize external url extraction By using a :not() selector and only extracting attributes from the first match. --- lib/pleroma/html.ex | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/html.ex b/lib/pleroma/html.ex index dc1b9b840..20b02f091 100644 --- a/lib/pleroma/html.ex +++ b/lib/pleroma/html.ex @@ -109,8 +109,9 @@ def extract_first_external_url(object, content) do result = content |> Floki.parse_fragment!() - |> Floki.filter_out("a.mention,a.hashtag,a.attachment,a[rel~=\"tag\"]") - |> Floki.attribute("a", "href") + |> Floki.find("a:not(.mention,.hashtag,.attachment,[rel~=\"tag\"])") + |> Enum.take(1) + |> Floki.attribute("href") |> Enum.at(0) {:commit, {:ok, result}} From 19691389b92e617f1edad7d4e3168fe917d0a675 Mon Sep 17 00:00:00 2001 From: rinpatch <rinpatch@sdf.org> Date: Wed, 2 Sep 2020 14:21:28 +0300 Subject: [PATCH 023/128] Rich media: Add failure tracking --- CHANGELOG.md | 3 ++ config/config.exs | 1 + config/description.exs | 7 +++ docs/configuration/cheatsheet.md | 1 + lib/pleroma/web/rich_media/parser.ex | 56 ++++++++++++--------- test/web/rich_media/aws_signed_url_test.exs | 2 +- 6 files changed, 46 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07bc6d77c..4424c060f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## unreleased-patch - ??? +### Added +- Rich media failure tracking (along with `:failure_backoff` option) + ### Fixed - Mastodon API: Search parameter `following` now correctly returns the followings rather than the followers diff --git a/config/config.exs b/config/config.exs index ea8869d48..ed37b93c0 100644 --- a/config/config.exs +++ b/config/config.exs @@ -412,6 +412,7 @@ Pleroma.Web.RichMedia.Parsers.TwitterCard, Pleroma.Web.RichMedia.Parsers.OEmbed ], + failure_backoff: 60_000, ttl_setters: [Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl] config :pleroma, :media_proxy, diff --git a/config/description.exs b/config/description.exs index 29a657333..5e08ba109 100644 --- a/config/description.exs +++ b/config/description.exs @@ -2385,6 +2385,13 @@ suggestions: [ Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl ] + }, + %{ + key: :failure_backoff, + type: :integer, + description: + "Amount of milliseconds after request failure, during which the request will not be retried.", + suggestions: [60_000] } ] }, diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 2f440adf4..a9a650fab 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -361,6 +361,7 @@ config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Http, * `ignore_hosts`: list of hosts which will be ignored by the metadata parser. For example `["accounts.google.com", "xss.website"]`, defaults to `[]`. * `ignore_tld`: list TLDs (top-level domains) which will ignore for parse metadata. default is ["local", "localdomain", "lan"]. * `parsers`: list of Rich Media parsers. +* `failure_backoff`: Amount of milliseconds after request failure, during which the request will not be retried. ## HTTP server diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex index e9aa2dd03..e98c743ca 100644 --- a/lib/pleroma/web/rich_media/parser.ex +++ b/lib/pleroma/web/rich_media/parser.ex @@ -17,14 +17,25 @@ def parse(url), do: parse_url(url) else @spec parse(String.t()) :: {:ok, map()} | {:error, any()} def parse(url) do - Cachex.fetch!(:rich_media_cache, url, fn _ -> - with {:ok, data} <- parse_url(url) do - {:commit, {:ok, data}} - else - error -> {:ignore, error} - end - end) - |> set_ttl_based_on_image(url) + with {:ok, data} <- get_cached_or_parse(url), + {:ok, _} <- set_ttl_based_on_image(data, url) do + {:ok, data} + else + error -> + Logger.error(fn -> "Rich media error: #{inspect(error)}" end) + end + end + + defp get_cached_or_parse(url) do + case Cachex.fetch!(:rich_media_cache, url, fn _ -> {:commit, parse_url(url)} end) do + {:ok, _data} = res -> + res + + {:error, _} = e -> + ttl = Pleroma.Config.get([:rich_media, :failure_backoff], 60_000) + Cachex.expire(:rich_media_cache, url, ttl) + e + end end end @@ -50,22 +61,21 @@ def ttl(data, url) do config :pleroma, :rich_media, ttl_setters: [MyModule] """ - @spec set_ttl_based_on_image({:ok, map()} | {:error, any()}, String.t()) :: - {:ok, map()} | {:error, any()} - def set_ttl_based_on_image({:ok, data}, url) do - with {:ok, nil} <- Cachex.ttl(:rich_media_cache, url), - {:ok, ttl} when is_number(ttl) <- get_ttl_from_image(data, url) do - Cachex.expire_at(:rich_media_cache, url, ttl * 1000) - {:ok, data} - else - _ -> - {:ok, data} - end - end + @spec set_ttl_based_on_image(map(), String.t()) :: + {:ok, Integer.t() | :noop} | {:error, :no_key} + def set_ttl_based_on_image(data, url) do + case get_ttl_from_image(data, url) do + {:ok, ttl} when is_number(ttl) -> + ttl = ttl * 1000 - def set_ttl_based_on_image({:error, _} = error, _) do - Logger.error("parsing error: #{inspect(error)}") - error + case Cachex.expire_at(:rich_media_cache, url, ttl) do + {:ok, true} -> {:ok, ttl} + {:ok, false} -> {:error, :no_key} + end + + _ -> + {:ok, :noop} + end end defp get_ttl_from_image(data, url) do diff --git a/test/web/rich_media/aws_signed_url_test.exs b/test/web/rich_media/aws_signed_url_test.exs index a21f3c935..1ceae1a31 100644 --- a/test/web/rich_media/aws_signed_url_test.exs +++ b/test/web/rich_media/aws_signed_url_test.exs @@ -55,7 +55,7 @@ test "s3 signed url is parsed and correct ttl is set for rich media" do Cachex.put(:rich_media_cache, url, metadata) - Pleroma.Web.RichMedia.Parser.set_ttl_based_on_image({:ok, metadata}, url) + Pleroma.Web.RichMedia.Parser.set_ttl_based_on_image(metadata, url) {:ok, cache_ttl} = Cachex.ttl(:rich_media_cache, url) From a11f23c130331d98db941c3edcc4d2dcf139bbc6 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov <alex.strizhakov@gmail.com> Date: Wed, 2 Sep 2020 15:45:47 +0300 Subject: [PATCH 024/128] user agent if Endpoint is not started yet --- lib/pleroma/application.ex | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 005aba50a..33b1e3872 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -22,13 +22,18 @@ def named_version, do: @name <> " " <> @version def repository, do: @repository def user_agent do - case Config.get([:http, :user_agent], :default) do - :default -> - info = "#{Pleroma.Web.base_url()} <#{Config.get([:instance, :email], "")}>" - named_version() <> "; " <> info + if Process.whereis(Pleroma.Web.Endpoint) do + case Config.get([:http, :user_agent], :default) do + :default -> + info = "#{Pleroma.Web.base_url()} <#{Config.get([:instance, :email], "")}>" + named_version() <> "; " <> info - custom -> - custom + custom -> + custom + end + else + # fallback, if endpoint is not started yet + "Pleroma Data Loader" end end From d48fc90978eee46c8fba00a80082d14fc31a0eec Mon Sep 17 00:00:00 2001 From: rinpatch <rinpatch@sdf.org> Date: Wed, 2 Sep 2020 13:44:08 +0300 Subject: [PATCH 025/128] StatusView: Start fetching rich media cards as soon as possible --- CHANGELOG.md | 2 ++ .../web/mastodon_api/views/status_view.ex | 16 ++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4424c060f..54685cfb8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Fixed - Mastodon API: Search parameter `following` now correctly returns the followings rather than the followers +- Mastodon API: Timelines hanging for (`number of posts with links * rich media timeout`) in the worst case. + Reduced to just rich media timeout. ## [2.1.0] - 2020-08-28 diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 01b8bb6bb..3fe1967be 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -23,6 +23,17 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do import Pleroma.Web.ActivityPub.Visibility, only: [get_visibility: 1, visible_for_user?: 2] + # This is a naive way to do this, just spawning a process per activity + # to fetch the preview. However it should be fine considering + # pagination is restricted to 40 activities at a time + defp fetch_rich_media_for_activities(activities) do + Enum.each(activities, fn activity -> + spawn(fn -> + Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) + end) + end) + end + # TODO: Add cached version. defp get_replied_to_activities([]), do: %{} @@ -80,6 +91,11 @@ def render("index.json", opts) do # To do: check AdminAPIControllerTest on the reasons behind nil activities in the list activities = Enum.filter(opts.activities, & &1) + + # Start fetching rich media before doing anything else, so that later calls to get the cards + # only block for timeout in the worst case, as opposed to + # length(activities_with_links) * timeout + fetch_rich_media_for_activities(activities) replied_to_activities = get_replied_to_activities(activities) parent_activities = From cbf7f0e02943f44a73f4418b8c6a8bada06331d8 Mon Sep 17 00:00:00 2001 From: Mark Felder <feld@FreeBSD.org> Date: Wed, 2 Sep 2020 09:09:13 -0500 Subject: [PATCH 026/128] Disallow password resets for deactivated accounts. Ensure all responses to password reset events are identical. --- CHANGELOG.md | 1 + .../controllers/auth_controller.ex | 14 ++++-------- lib/pleroma/web/twitter_api/twitter_api.ex | 13 ++--------- .../controllers/auth_controller_test.exs | 22 ++++++++++++++----- 4 files changed, 23 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 54685cfb8..cdd5b94a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Search parameter `following` now correctly returns the followings rather than the followers - Mastodon API: Timelines hanging for (`number of posts with links * rich media timeout`) in the worst case. Reduced to just rich media timeout. +- Password resets no longer processed for deactivated accounts ## [2.1.0] - 2020-08-28 diff --git a/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex b/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex index 753b3db3e..9f09550e1 100644 --- a/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex @@ -59,17 +59,11 @@ def logout(conn, _) do def password_reset(conn, params) do nickname_or_email = params["email"] || params["nickname"] - with {:ok, _} <- TwitterAPI.password_reset(nickname_or_email) do - conn - |> put_status(:no_content) - |> json("") - else - {:error, "unknown user"} -> - send_resp(conn, :not_found, "") + TwitterAPI.password_reset(nickname_or_email) - {:error, _} -> - send_resp(conn, :bad_request, "") - end + conn + |> put_status(:no_content) + |> json("") end defp local_mastodon_root_path(conn) do diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex index 2294d9d0d..5d7948507 100644 --- a/lib/pleroma/web/twitter_api/twitter_api.ex +++ b/lib/pleroma/web/twitter_api/twitter_api.ex @@ -72,7 +72,7 @@ defp maybe_notify_admins(%User{} = account) do def password_reset(nickname_or_email) do with true <- is_binary(nickname_or_email), - %User{local: true, email: email} = user when is_binary(email) <- + %User{local: true, email: email, deactivated: false} = user when is_binary(email) <- User.get_by_nickname_or_email(nickname_or_email), {:ok, token_record} <- Pleroma.PasswordResetToken.create_token(user) do user @@ -81,17 +81,8 @@ def password_reset(nickname_or_email) do {:ok, :enqueued} else - false -> - {:error, "bad user identifier"} - - %User{local: true, email: nil} -> + _ -> {:ok, :noop} - - %User{local: false} -> - {:error, "remote user"} - - nil -> - {:error, "unknown user"} end end diff --git a/test/web/mastodon_api/controllers/auth_controller_test.exs b/test/web/mastodon_api/controllers/auth_controller_test.exs index a485f8e41..4fa95fce1 100644 --- a/test/web/mastodon_api/controllers/auth_controller_test.exs +++ b/test/web/mastodon_api/controllers/auth_controller_test.exs @@ -122,17 +122,27 @@ test "it doesn't fail when a user has no email", %{conn: conn} do {:ok, user: user} end - test "it returns 404 when user is not found", %{conn: conn, user: user} do + test "it returns 204 when user is not found", %{conn: conn, user: user} do conn = post(conn, "/auth/password?email=nonexisting_#{user.email}") - assert conn.status == 404 - assert conn.resp_body == "" + + assert conn + |> json_response(:no_content) end - test "it returns 400 when user is not local", %{conn: conn, user: user} do + test "it returns 204 when user is not local", %{conn: conn, user: user} do {:ok, user} = Repo.update(Ecto.Changeset.change(user, local: false)) conn = post(conn, "/auth/password?email=#{user.email}") - assert conn.status == 400 - assert conn.resp_body == "" + + assert conn + |> json_response(:no_content) + end + + test "it returns 204 when user is deactivated", %{conn: conn, user: user} do + {:ok, user} = Repo.update(Ecto.Changeset.change(user, deactivated: true, local: true)) + conn = post(conn, "/auth/password?email=#{user.email}") + + assert conn + |> json_response(:no_content) end end From 581f382e712dc50fa51d1ab211f13b8843dcb448 Mon Sep 17 00:00:00 2001 From: lain <lain@soykaf.club> Date: Wed, 2 Sep 2020 18:32:00 +0200 Subject: [PATCH 027/128] ListController: DRY up stuff. --- .../mastodon_api/controllers/list_controller.ex | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/controllers/list_controller.ex b/lib/pleroma/web/mastodon_api/controllers/list_controller.ex index 4df13cb81..5daeaa780 100644 --- a/lib/pleroma/web/mastodon_api/controllers/list_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/list_controller.ex @@ -73,19 +73,6 @@ def add_to_list(%{assigns: %{list: list}, body_params: %{account_ids: account_id end # DELETE /api/v1/lists/:id/accounts - def remove_from_list( - %{assigns: %{list: list}, body_params: %{account_ids: account_ids}} = conn, - _ - ) do - Enum.each(account_ids, fn account_id -> - with %User{} = followed <- User.get_cached_by_id(account_id) do - Pleroma.List.unfollow(list, followed) - end - end) - - json(conn, %{}) - end - def remove_from_list( %{assigns: %{list: list}, params: %{account_ids: account_ids}} = conn, _ @@ -99,6 +86,10 @@ def remove_from_list( json(conn, %{}) end + def remove_from_list(%{body_params: params} = conn, _) do + remove_from_list(%{conn | params: params}, %{}) + end + defp list_by_id_and_user(%{assigns: %{user: user}, params: %{id: id}} = conn, _) do case Pleroma.List.get(id, user) do %Pleroma.List{} = list -> assign(conn, :list, list) From 5da367760748f7c4534b84dcb9286d715110472e Mon Sep 17 00:00:00 2001 From: lain <lain@soykaf.club> Date: Thu, 3 Sep 2020 11:40:17 +0200 Subject: [PATCH 028/128] Frontend mix task: Add tests. --- test/tasks/frontend_test.exs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/test/tasks/frontend_test.exs b/test/tasks/frontend_test.exs index 0ca2b9a28..022ae51be 100644 --- a/test/tasks/frontend_test.exs +++ b/test/tasks/frontend_test.exs @@ -48,11 +48,18 @@ test "it also works given a file" do } }) + folder = Path.join([@dir, "frontends", "pleroma", "fantasy"]) + previously_existing = Path.join([folder, "temp"]) + File.mkdir_p!(folder) + File.write!(previously_existing, "yey") + assert File.exists?(previously_existing) + capture_io(fn -> Frontend.run(["install", "pleroma", "--file", "test/fixtures/tesla_mock/frontend.zip"]) end) - assert File.exists?(Path.join([@dir, "frontends", "pleroma", "fantasy", "test.txt"])) + assert File.exists?(Path.join([folder, "test.txt"])) + refute File.exists?(previously_existing) end test "it downloads and unzips unknown frontends" do From d34fe2840d969c30b393cfb73e34b6301027c776 Mon Sep 17 00:00:00 2001 From: rinpatch <rinpatch@sdf.org> Date: Thu, 3 Sep 2020 23:15:22 +0300 Subject: [PATCH 029/128] HTTP: radically simplify pool checkin/checkout Use a custom tesla middleware instead of adapter helper function + custom redirect middleware. This will also fix "Client died before releasing the connection" messages when the request pool is overloaded. Since the checkout is now done after passing ConcurrentLimiter. This is technically less efficient, since the connection needs to be checked in/out every time the middleware is left or entered respectively. But I don't think the nanoseconds we might lose on redirects to the same host are worth the complexity. --- lib/pleroma/http/adapter_helper.ex | 4 - lib/pleroma/http/adapter_helper/gun.ex | 9 -- lib/pleroma/http/adapter_helper/hackney.ex | 3 - lib/pleroma/http/http.ex | 33 +++--- .../tesla/middleware/connection_pool.ex | 35 ++++++ .../tesla/middleware/follow_redirects.ex | 110 ------------------ 6 files changed, 48 insertions(+), 146 deletions(-) create mode 100644 lib/pleroma/tesla/middleware/connection_pool.ex delete mode 100644 lib/pleroma/tesla/middleware/follow_redirects.ex diff --git a/lib/pleroma/http/adapter_helper.ex b/lib/pleroma/http/adapter_helper.ex index 0728cbaa2..d72297323 100644 --- a/lib/pleroma/http/adapter_helper.ex +++ b/lib/pleroma/http/adapter_helper.ex @@ -19,7 +19,6 @@ defmodule Pleroma.HTTP.AdapterHelper do | {Connection.proxy_type(), Connection.host(), pos_integer()} @callback options(keyword(), URI.t()) :: keyword() - @callback get_conn(URI.t(), keyword()) :: {:ok, term()} | {:error, term()} @spec format_proxy(String.t() | tuple() | nil) :: proxy() | nil def format_proxy(nil), do: nil @@ -47,9 +46,6 @@ def options(%URI{} = uri, opts \\ []) do |> adapter_helper().options(uri) end - @spec get_conn(URI.t(), keyword()) :: {:ok, keyword()} | {:error, atom()} - def get_conn(uri, opts), do: adapter_helper().get_conn(uri, opts) - defp adapter, do: Application.get_env(:tesla, :adapter) defp adapter_helper do diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex index 02e20f2d1..4a967d8f2 100644 --- a/lib/pleroma/http/adapter_helper/gun.ex +++ b/lib/pleroma/http/adapter_helper/gun.ex @@ -6,7 +6,6 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do @behaviour Pleroma.HTTP.AdapterHelper alias Pleroma.Config - alias Pleroma.Gun.ConnectionPool alias Pleroma.HTTP.AdapterHelper require Logger @@ -57,14 +56,6 @@ def pool_timeout(pool) do Config.get([:pools, pool, :timeout], default) end - @spec get_conn(URI.t(), keyword()) :: {:ok, keyword()} | {:error, atom()} - def get_conn(uri, opts) do - case ConnectionPool.get_conn(uri, opts) do - {:ok, conn_pid} -> {:ok, Keyword.merge(opts, conn: conn_pid, close_conn: false)} - err -> err - end - end - @prefix Pleroma.Gun.ConnectionPool def limiter_setup do wait = Config.get([:connections_pool, :connection_acquisition_wait]) diff --git a/lib/pleroma/http/adapter_helper/hackney.ex b/lib/pleroma/http/adapter_helper/hackney.ex index cd569422b..f47a671ad 100644 --- a/lib/pleroma/http/adapter_helper/hackney.ex +++ b/lib/pleroma/http/adapter_helper/hackney.ex @@ -23,7 +23,4 @@ def options(connection_opts \\ [], %URI{} = uri) do end defp add_scheme_opts(opts, _), do: opts - - @spec get_conn(URI.t(), keyword()) :: {:ok, keyword()} - def get_conn(_uri, opts), do: {:ok, opts} end diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index b37b3fa89..7bc73f4a0 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -62,28 +62,21 @@ def request(method, url, body, headers, options) when is_binary(url) do uri = URI.parse(url) adapter_opts = AdapterHelper.options(uri, options[:adapter] || []) - case AdapterHelper.get_conn(uri, adapter_opts) do - {:ok, adapter_opts} -> - options = put_in(options[:adapter], adapter_opts) - params = options[:params] || [] - request = build_request(method, headers, options, url, body, params) + options = put_in(options[:adapter], adapter_opts) + params = options[:params] || [] + request = build_request(method, headers, options, url, body, params) - adapter = Application.get_env(:tesla, :adapter) + adapter = Application.get_env(:tesla, :adapter) - client = Tesla.client(adapter_middlewares(adapter), adapter) + client = Tesla.client(adapter_middlewares(adapter), adapter) - maybe_limit( - fn -> - request(client, request) - end, - adapter, - adapter_opts - ) - - # Connection release is handled in a custom FollowRedirects middleware - err -> - err - end + maybe_limit( + fn -> + request(client, request) + end, + adapter, + adapter_opts + ) end @spec request(Client.t(), keyword()) :: {:ok, Env.t()} | {:error, any()} @@ -110,7 +103,7 @@ defp maybe_limit(fun, _, _) do end defp adapter_middlewares(Tesla.Adapter.Gun) do - [Pleroma.HTTP.Middleware.FollowRedirects] + [Tesla.Middleware.FollowRedirects, Pleroma.Tesla.Middleware.ConnectionPool] end defp adapter_middlewares(_), do: [] diff --git a/lib/pleroma/tesla/middleware/connection_pool.ex b/lib/pleroma/tesla/middleware/connection_pool.ex new file mode 100644 index 000000000..a435ab4cc --- /dev/null +++ b/lib/pleroma/tesla/middleware/connection_pool.ex @@ -0,0 +1,35 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Tesla.Middleware.ConnectionPool do + @moduledoc """ + Middleware to get/release connections from `Pleroma.Gun.ConnectionPool` + """ + + @behaviour Tesla.Middleware + + alias Pleroma.Gun.ConnectionPool + + @impl Tesla.Middleware + def call(%Tesla.Env{url: url, opts: opts} = env, next, _) do + uri = URI.parse(url) + + case ConnectionPool.get_conn(uri, opts[:adapter]) do + {:ok, conn_pid} -> + adapter_opts = Keyword.merge(opts[:adapter], conn: conn_pid, close_conn: false) + opts = Keyword.put(opts, :adapter, adapter_opts) + env = %{env | opts: opts} + res = Tesla.run(env, next) + + unless opts[:adapter][:body_as] == :chunks do + ConnectionPool.release_conn(conn_pid) + end + + res + + err -> + err + end + end +end diff --git a/lib/pleroma/tesla/middleware/follow_redirects.ex b/lib/pleroma/tesla/middleware/follow_redirects.ex deleted file mode 100644 index 5a7032215..000000000 --- a/lib/pleroma/tesla/middleware/follow_redirects.ex +++ /dev/null @@ -1,110 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2015-2020 Tymon Tobolski <https://github.com/teamon/tesla/blob/master/lib/tesla/middleware/follow_redirects.ex> -# Copyright © 2020 Pleroma Authors <https://pleroma.social/> -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.HTTP.Middleware.FollowRedirects do - @moduledoc """ - Pool-aware version of https://github.com/teamon/tesla/blob/master/lib/tesla/middleware/follow_redirects.ex - - Follow 3xx redirects - ## Options - - `:max_redirects` - limit number of redirects (default: `5`) - """ - - alias Pleroma.Gun.ConnectionPool - - @behaviour Tesla.Middleware - - @max_redirects 5 - @redirect_statuses [301, 302, 303, 307, 308] - - @impl Tesla.Middleware - def call(env, next, opts \\ []) do - max = Keyword.get(opts, :max_redirects, @max_redirects) - - redirect(env, next, max) - end - - defp redirect(env, next, left) do - opts = env.opts[:adapter] - - case Tesla.run(env, next) do - {:ok, %{status: status} = res} when status in @redirect_statuses and left > 0 -> - release_conn(opts) - - case Tesla.get_header(res, "location") do - nil -> - {:ok, res} - - location -> - location = parse_location(location, res) - - case get_conn(location, opts) do - {:ok, opts} -> - %{env | opts: Keyword.put(env.opts, :adapter, opts)} - |> new_request(res.status, location) - |> redirect(next, left - 1) - - e -> - e - end - end - - {:ok, %{status: status}} when status in @redirect_statuses -> - release_conn(opts) - {:error, {__MODULE__, :too_many_redirects}} - - {:error, _} = e -> - release_conn(opts) - e - - other -> - unless opts[:body_as] == :chunks do - release_conn(opts) - end - - other - end - end - - defp get_conn(location, opts) do - uri = URI.parse(location) - - case ConnectionPool.get_conn(uri, opts) do - {:ok, conn} -> - {:ok, Keyword.merge(opts, conn: conn)} - - e -> - e - end - end - - defp release_conn(opts) do - ConnectionPool.release_conn(opts[:conn]) - end - - # The 303 (See Other) redirect was added in HTTP/1.1 to indicate that the originally - # requested resource is not available, however a related resource (or another redirect) - # available via GET is available at the specified location. - # https://tools.ietf.org/html/rfc7231#section-6.4.4 - defp new_request(env, 303, location), do: %{env | url: location, method: :get, query: []} - - # The 307 (Temporary Redirect) status code indicates that the target - # resource resides temporarily under a different URI and the user agent - # MUST NOT change the request method (...) - # https://tools.ietf.org/html/rfc7231#section-6.4.7 - defp new_request(env, 307, location), do: %{env | url: location} - - defp new_request(env, _, location), do: %{env | url: location, query: []} - - defp parse_location("https://" <> _rest = location, _env), do: location - defp parse_location("http://" <> _rest = location, _env), do: location - - defp parse_location(location, env) do - env.url - |> URI.parse() - |> URI.merge(location) - |> URI.to_string() - end -end From bce22937dc5c7545b9f104daf4b8ef9afaa65e75 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me> Date: Fri, 4 Sep 2020 09:15:58 +0200 Subject: [PATCH 030/128] mix.lock: Bump fast_html This update fixes an incorrect push to Hex that reverted the gcc-10 fix --- mix.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.lock b/mix.lock index afb4b06db..874ea9215 100644 --- a/mix.lock +++ b/mix.lock @@ -42,7 +42,7 @@ "ex_machina": {:hex, :ex_machina, "2.4.0", "09a34c5d371bfb5f78399029194a8ff67aff340ebe8ba19040181af35315eabb", [:mix], [{:ecto, "~> 2.2 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}], "hexpm", "a20bc9ddc721b33ea913b93666c5d0bdca5cbad7a67540784ae277228832d72c"}, "ex_syslogger": {:hex, :ex_syslogger, "1.5.2", "72b6aa2d47a236e999171f2e1ec18698740f40af0bd02c8c650bf5f1fd1bac79", [:mix], [{:poison, ">= 1.5.0", [hex: :poison, repo: "hexpm", optional: true]}, {:syslog, "~> 1.1.0", [hex: :syslog, repo: "hexpm", optional: false]}], "hexpm", "ab9fab4136dbc62651ec6f16fa4842f10cf02ab4433fa3d0976c01be99398399"}, "excoveralls": {:hex, :excoveralls, "0.13.1", "b9f1697f7c9e0cfe15d1a1d737fb169c398803ffcbc57e672aa007e9fd42864c", [:mix], [{:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "b4bb550e045def1b4d531a37fb766cbbe1307f7628bf8f0414168b3f52021cce"}, - "fast_html": {:hex, :fast_html, "2.0.3", "27289dea6c3a22952191a2d4e07f546c0de8a351484782c2e797dbae06a5dc8a", [:make, :mix], [{:elixir_make, "~> 0.4", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.1.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}], "hexpm", "17d41fa8afe4e912ffe74e13b87ddb085382cd2b7393636d338495c9a8a7b518"}, + "fast_html": {:hex, :fast_html, "2.0.4", "4910ee49f2f6b19692e3bf30bf97f1b6b7dac489cd6b0f34cd0fe3042c56ba30", [:make, :mix], [{:elixir_make, "~> 0.4", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.1.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}], "hexpm", "3bb49d541dfc02ad5e425904f53376d758c09f89e521afc7d2b174b3227761ea"}, "fast_sanitize": {:hex, :fast_sanitize, "0.2.2", "3cbbaebaea6043865dfb5b4ecb0f1af066ad410a51470e353714b10c42007b81", [:mix], [{:fast_html, "~> 2.0", [hex: :fast_html, repo: "hexpm", optional: false]}, {:plug, "~> 1.8", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "69f204db9250afa94a0d559d9110139850f57de2b081719fbafa1e9a89e94466"}, "flake_id": {:hex, :flake_id, "0.1.0", "7716b086d2e405d09b647121a166498a0d93d1a623bead243e1f74216079ccb3", [:mix], [{:base62, "~> 1.2", [hex: :base62, repo: "hexpm", optional: false]}, {:ecto, ">= 2.0.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm", "31fc8090fde1acd267c07c36ea7365b8604055f897d3a53dd967658c691bd827"}, "floki": {:hex, :floki, "0.27.0", "6b29a14283f1e2e8fad824bc930eaa9477c462022075df6bea8f0ad811c13599", [:mix], [{:html_entities, "~> 0.5.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm", "583b8c13697c37179f1f82443bcc7ad2f76fbc0bf4c186606eebd658f7f2631b"}, From 8bd2b6eb138ace3408a03c78ecc339fc35b19f10 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov <alex.strizhakov@gmail.com> Date: Fri, 4 Sep 2020 14:24:15 +0300 Subject: [PATCH 031/128] temp hackney fix --- lib/pleroma/http/adapter_helper/hackney.ex | 4 ++++ mix.exs | 5 ++++- mix.lock | 14 +++++++------- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/http/adapter_helper/hackney.ex b/lib/pleroma/http/adapter_helper/hackney.ex index f47a671ad..42e3acfec 100644 --- a/lib/pleroma/http/adapter_helper/hackney.ex +++ b/lib/pleroma/http/adapter_helper/hackney.ex @@ -22,5 +22,9 @@ def options(connection_opts \\ [], %URI{} = uri) do |> Pleroma.HTTP.AdapterHelper.maybe_add_proxy(proxy) end + defp add_scheme_opts(opts, %URI{scheme: "https"}) do + Keyword.put(opts, :ssl_options, versions: [:"tlsv1.2", :"tlsv1.1", :tlsv1]) + end + defp add_scheme_opts(opts, _), do: opts end diff --git a/mix.exs b/mix.exs index 4de0c78db..f734e4f61 100644 --- a/mix.exs +++ b/mix.exs @@ -195,7 +195,10 @@ defp deps do {:ex_machina, "~> 2.4", only: :test}, {:credo, "~> 1.4", only: [:dev, :test], runtime: false}, {:mock, "~> 0.3.5", only: :test}, - {:excoveralls, "~> 0.13.1", only: :test}, + # temporary downgrade for excoveralls, hackney and httpoison - until hackney max_connections bug will be fixed + {:excoveralls, "0.12.3", only: :test}, + {:hackney, "1.15.2"}, + {:httpoison, "1.6.2"}, {:mox, "~> 0.5", only: :test}, {:websocket_client, git: "https://github.com/jeremyong/websocket_client.git", only: :test} ] ++ oauth_deps() diff --git a/mix.lock b/mix.lock index 874ea9215..b97dd6342 100644 --- a/mix.lock +++ b/mix.lock @@ -11,7 +11,7 @@ "calendar": {:hex, :calendar, "1.0.0", "f52073a708528482ec33d0a171954ca610fe2bd28f1e871f247dc7f1565fa807", [:mix], [{:tzdata, "~> 0.5.20 or ~> 0.1.201603 or ~> 1.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "990e9581920c82912a5ee50e62ff5ef96da6b15949a2ee4734f935fdef0f0a6f"}, "captcha": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/elixir-captcha.git", "e0f16822d578866e186a0974d65ad58cddc1e2ab", [ref: "e0f16822d578866e186a0974d65ad58cddc1e2ab"]}, "castore": {:hex, :castore, "0.1.7", "1ca19eee705cde48c9e809e37fdd0730510752cc397745e550f6065a56a701e9", [:mix], [], "hexpm", "a2ae2c13d40e9c308387f1aceb14786dca019ebc2a11484fb2a9f797ea0aa0d8"}, - "certifi": {:hex, :certifi, "2.5.2", "b7cfeae9d2ed395695dd8201c57a2d019c0c43ecaf8b8bcb9320b40d6662f340", [:rebar3], [{:parse_trans, "~>3.3", [hex: :parse_trans, repo: "hexpm", optional: false]}], "hexpm", "3b3b5f36493004ac3455966991eaf6e768ce9884693d9968055aeeeb1e575040"}, + "certifi": {:hex, :certifi, "2.5.1", "867ce347f7c7d78563450a18a6a28a8090331e77fa02380b4a21962a65d36ee5", [:rebar3], [{:parse_trans, "~>3.3", [hex: :parse_trans, repo: "hexpm", optional: false]}], "hexpm", "805abd97539caf89ec6d4732c91e62ba9da0cda51ac462380bbd28ee697a8c42"}, "combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"}, "comeonin": {:hex, :comeonin, "5.3.1", "7fe612b739c78c9c1a75186ef2d322ce4d25032d119823269d0aa1e2f1e20025", [:mix], [], "hexpm", "d6222483060c17f0977fad1b7401ef0c5863c985a64352755f366aee3799c245"}, "concurrent_limiter": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/concurrent_limiter.git", "55e92f84b4ed531bd487952a71040a9c69dc2807", [ref: "55e92f84b4ed531bd487952a71040a9c69dc2807"]}, @@ -41,7 +41,7 @@ "ex_doc": {:hex, :ex_doc, "0.22.2", "03a2a58bdd2ba0d83d004507c4ee113b9c521956938298eba16e55cc4aba4a6c", [:mix], [{:earmark_parser, "~> 1.4.0", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm", "cf60e1b3e2efe317095b6bb79651f83a2c1b3edcb4d319c421d7fcda8b3aff26"}, "ex_machina": {:hex, :ex_machina, "2.4.0", "09a34c5d371bfb5f78399029194a8ff67aff340ebe8ba19040181af35315eabb", [:mix], [{:ecto, "~> 2.2 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}], "hexpm", "a20bc9ddc721b33ea913b93666c5d0bdca5cbad7a67540784ae277228832d72c"}, "ex_syslogger": {:hex, :ex_syslogger, "1.5.2", "72b6aa2d47a236e999171f2e1ec18698740f40af0bd02c8c650bf5f1fd1bac79", [:mix], [{:poison, ">= 1.5.0", [hex: :poison, repo: "hexpm", optional: true]}, {:syslog, "~> 1.1.0", [hex: :syslog, repo: "hexpm", optional: false]}], "hexpm", "ab9fab4136dbc62651ec6f16fa4842f10cf02ab4433fa3d0976c01be99398399"}, - "excoveralls": {:hex, :excoveralls, "0.13.1", "b9f1697f7c9e0cfe15d1a1d737fb169c398803ffcbc57e672aa007e9fd42864c", [:mix], [{:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "b4bb550e045def1b4d531a37fb766cbbe1307f7628bf8f0414168b3f52021cce"}, + "excoveralls": {:hex, :excoveralls, "0.12.3", "2142be7cb978a3ae78385487edda6d1aff0e482ffc6123877bb7270a8ffbcfe0", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "568a3e616c264283f5dea5b020783ae40eef3f7ee2163f7a67cbd7b35bcadada"}, "fast_html": {:hex, :fast_html, "2.0.4", "4910ee49f2f6b19692e3bf30bf97f1b6b7dac489cd6b0f34cd0fe3042c56ba30", [:make, :mix], [{:elixir_make, "~> 0.4", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.1.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}], "hexpm", "3bb49d541dfc02ad5e425904f53376d758c09f89e521afc7d2b174b3227761ea"}, "fast_sanitize": {:hex, :fast_sanitize, "0.2.2", "3cbbaebaea6043865dfb5b4ecb0f1af066ad410a51470e353714b10c42007b81", [:mix], [{:fast_html, "~> 2.0", [hex: :fast_html, repo: "hexpm", optional: false]}, {:plug, "~> 1.8", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "69f204db9250afa94a0d559d9110139850f57de2b081719fbafa1e9a89e94466"}, "flake_id": {:hex, :flake_id, "0.1.0", "7716b086d2e405d09b647121a166498a0d93d1a623bead243e1f74216079ccb3", [:mix], [{:base62, "~> 1.2", [hex: :base62, repo: "hexpm", optional: false]}, {:ecto, ">= 2.0.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm", "31fc8090fde1acd267c07c36ea7365b8604055f897d3a53dd967658c691bd827"}, @@ -51,12 +51,12 @@ "gen_state_machine": {:hex, :gen_state_machine, "2.0.5", "9ac15ec6e66acac994cc442dcc2c6f9796cf380ec4b08267223014be1c728a95", [:mix], [], "hexpm"}, "gettext": {:hex, :gettext, "0.18.0", "406d6b9e0e3278162c2ae1de0a60270452c553536772167e2d701f028116f870", [:mix], [], "hexpm", "c3f850be6367ebe1a08616c2158affe4a23231c70391050bf359d5f92f66a571"}, "gun": {:git, "https://github.com/ninenines/gun.git", "921c47146b2d9567eac7e9a4d2ccc60fffd4f327", [ref: "921c47146b2d9567eac7e9a4d2ccc60fffd4f327"]}, - "hackney": {:hex, :hackney, "1.16.0", "5096ac8e823e3a441477b2d187e30dd3fff1a82991a806b2003845ce72ce2d84", [:rebar3], [{:certifi, "2.5.2", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.1", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.3.0", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.6", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "3bf0bebbd5d3092a3543b783bf065165fa5d3ad4b899b836810e513064134e18"}, + "hackney": {:hex, :hackney, "1.15.2", "07e33c794f8f8964ee86cebec1a8ed88db5070e52e904b8f12209773c1036085", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.5", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "e0100f8ef7d1124222c11ad362c857d3df7cb5f4204054f9f0f4a728666591fc"}, "html_entities": {:hex, :html_entities, "0.5.1", "1c9715058b42c35a2ab65edc5b36d0ea66dd083767bef6e3edb57870ef556549", [:mix], [], "hexpm", "30efab070904eb897ff05cd52fa61c1025d7f8ef3a9ca250bc4e6513d16c32de"}, "html_sanitize_ex": {:hex, :html_sanitize_ex, "1.3.0", "f005ad692b717691203f940c686208aa3d8ffd9dd4bb3699240096a51fa9564e", [:mix], [{:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm"}, "http_signatures": {:hex, :http_signatures, "0.1.0", "4e4b501a936dbf4cb5222597038a89ea10781776770d2e185849fa829686b34c", [:mix], [], "hexpm", "f8a7b3731e3fd17d38fa6e343fcad7b03d6874a3b0a108c8568a71ed9c2cf824"}, - "httpoison": {:hex, :httpoison, "1.7.0", "abba7d086233c2d8574726227b6c2c4f6e53c4deae7fe5f6de531162ce9929a0", [:mix], [{:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "975cc87c845a103d3d1ea1ccfd68a2700c211a434d8428b10c323dc95dc5b980"}, - "idna": {:hex, :idna, "6.0.1", "1d038fb2e7668ce41fbf681d2c45902e52b3cb9e9c77b55334353b222c2ee50c", [:rebar3], [{:unicode_util_compat, "0.5.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "a02c8a1c4fd601215bb0b0324c8a6986749f807ce35f25449ec9e69758708122"}, + "httpoison": {:hex, :httpoison, "1.6.2", "ace7c8d3a361cebccbed19c283c349b3d26991eff73a1eaaa8abae2e3c8089b6", [:mix], [{:hackney, "~> 1.15 and >= 1.15.2", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "aa2c74bd271af34239a3948779612f87df2422c2fdcfdbcec28d9c105f0773fe"}, + "idna": {:hex, :idna, "6.0.0", "689c46cbcdf3524c44d5f3dde8001f364cd7608a99556d8fbd8239a5798d4c10", [:rebar3], [{:unicode_util_compat, "0.4.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "4bdd305eb64e18b0273864920695cb18d7a2021f31a11b9c5fbcd9a253f936e2"}, "inet_cidr": {:hex, :inet_cidr, "1.0.4", "a05744ab7c221ca8e395c926c3919a821eb512e8f36547c062f62c4ca0cf3d6e", [:mix], [], "hexpm", "64a2d30189704ae41ca7dbdd587f5291db5d1dda1414e0774c29ffc81088c1bc"}, "jason": {:hex, :jason, "1.2.1", "12b22825e22f468c02eb3e4b9985f3d0cb8dc40b9bd704730efa11abd2708c44", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "b659b8571deedf60f79c5a608e15414085fa141344e2716fbd6988a084b5f993"}, "joken": {:hex, :joken, "2.2.0", "2daa1b12be05184aff7b5ace1d43ca1f81345962285fff3f88db74927c954d3a", [:mix], [{:jose, "~> 1.9", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "b4f92e30388206f869dd25d1af628a1d99d7586e5cf0672f64d4df84c4d2f5e9"}, @@ -105,7 +105,7 @@ "recon": {:hex, :recon, "2.5.1", "430ffa60685ac1efdfb1fe4c97b8767c92d0d92e6e7c3e8621559ba77598678a", [:mix, :rebar3], [], "hexpm", "5721c6b6d50122d8f68cccac712caa1231f97894bab779eff5ff0f886cb44648"}, "remote_ip": {:git, "https://git.pleroma.social/pleroma/remote_ip.git", "b647d0deecaa3acb140854fe4bda5b7e1dc6d1c8", [ref: "b647d0deecaa3acb140854fe4bda5b7e1dc6d1c8"]}, "sleeplocks": {:hex, :sleeplocks, "1.1.1", "3d462a0639a6ef36cc75d6038b7393ae537ab394641beb59830a1b8271faeed3", [:rebar3], [], "hexpm", "84ee37aeff4d0d92b290fff986d6a95ac5eedf9b383fadfd1d88e9b84a1c02e1"}, - "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.6", "cf344f5692c82d2cd7554f5ec8fd961548d4fd09e7d22f5b62482e5aeaebd4b0", [:make, :mix, :rebar3], [], "hexpm", "bdb0d2471f453c88ff3908e7686f86f9be327d065cc1ec16fa4540197ea04680"}, + "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.5", "6eaf7ad16cb568bb01753dbbd7a95ff8b91c7979482b95f38443fe2c8852a79b", [:make, :mix, :rebar3], [], "hexpm", "13104d7897e38ed7f044c4de953a6c28597d1c952075eb2e328bc6d6f2bfc496"}, "sweet_xml": {:hex, :sweet_xml, "0.6.6", "fc3e91ec5dd7c787b6195757fbcf0abc670cee1e4172687b45183032221b66b8", [:mix], [], "hexpm", "2e1ec458f892ffa81f9f8386e3f35a1af6db7a7a37748a64478f13163a1f3573"}, "swoosh": {:hex, :swoosh, "1.0.0", "c547cfc83f30e12d5d1fdcb623d7de2c2e29a5becfc68bf8f42ba4d23d2c2756", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm", "b3b08e463f876cb6167f7168e9ad99a069a724e124bcee61847e0e1ed13f4a0d"}, "syslog": {:hex, :syslog, "1.1.0", "6419a232bea84f07b56dc575225007ffe34d9fdc91abe6f1b2f254fd71d8efc2", [:rebar3], [], "hexpm", "4c6a41373c7e20587be33ef841d3de6f3beba08519809329ecc4d27b15b659e1"}, @@ -115,7 +115,7 @@ "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "bd4fde4c15f3e993a999e019d64347489b91b7a9096af68b2bdadd192afa693f"}, "tzdata": {:hex, :tzdata, "1.0.3", "73470ad29dde46e350c60a66e6b360d3b99d2d18b74c4c349dbebbc27a09a3eb", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "a6e1ee7003c4d04ecbd21dd3ec690d4c6662db5d3bbdd7262d53cdf5e7c746c1"}, "ueberauth": {:hex, :ueberauth, "0.6.3", "d42ace28b870e8072cf30e32e385579c57b9cc96ec74fa1f30f30da9c14f3cc0", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "afc293d8a1140d6591b53e3eaf415ca92842cb1d32fad3c450c6f045f7f91b60"}, - "unicode_util_compat": {:hex, :unicode_util_compat, "0.5.0", "8516502659002cec19e244ebd90d312183064be95025a319a6c7e89f4bccd65b", [:rebar3], [], "hexpm", "d48d002e15f5cc105a696cf2f1bbb3fc72b4b770a184d8420c8db20da2674b38"}, + "unicode_util_compat": {:hex, :unicode_util_compat, "0.4.1", "d869e4c68901dd9531385bb0c8c40444ebf624e60b6962d95952775cac5e90cd", [:rebar3], [], "hexpm", "1d1848c40487cdb0b30e8ed975e34e025860c02e419cb615d255849f3427439d"}, "unsafe": {:hex, :unsafe, "1.0.1", "a27e1874f72ee49312e0a9ec2e0b27924214a05e3ddac90e91727bc76f8613d8", [:mix], [], "hexpm", "6c7729a2d214806450d29766abc2afaa7a2cbecf415be64f36a6691afebb50e5"}, "web_push_encryption": {:hex, :web_push_encryption, "0.3.0", "598b5135e696fd1404dc8d0d7c0fa2c027244a4e5d5e5a98ba267f14fdeaabc8", [:mix], [{:httpoison, "~> 1.0", [hex: :httpoison, repo: "hexpm", optional: false]}, {:jose, "~> 1.8", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "f10bdd1afe527ede694749fb77a2f22f146a51b054c7fa541c9fd920fba7c875"}, "websocket_client": {:git, "https://github.com/jeremyong/websocket_client.git", "9a6f65d05ebf2725d62fb19262b21f1805a59fbf", []}, From 473458b0fbecb05121b235f525aefcef34f0409e Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov <alex.strizhakov@gmail.com> Date: Fri, 4 Sep 2020 14:45:30 +0300 Subject: [PATCH 032/128] fix for ReverseProxy --- lib/pleroma/reverse_proxy/client/hackney.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/reverse_proxy/client/hackney.ex b/lib/pleroma/reverse_proxy/client/hackney.ex index e84118a90..ad988fac3 100644 --- a/lib/pleroma/reverse_proxy/client/hackney.ex +++ b/lib/pleroma/reverse_proxy/client/hackney.ex @@ -7,6 +7,7 @@ defmodule Pleroma.ReverseProxy.Client.Hackney do @impl true def request(method, url, headers, body, opts \\ []) do + opts = Keyword.put(opts, :ssl_options, versions: [:"tlsv1.2", :"tlsv1.1", :tlsv1]) :hackney.request(method, url, headers, body, opts) end From 173b04df48abf1b40aa0aa4ba5de50c370687f56 Mon Sep 17 00:00:00 2001 From: Farhan Khan <farhan@farhan.codes> Date: Fri, 4 Sep 2020 18:03:58 +0000 Subject: [PATCH 033/128] Added cmake --- docs/installation/freebsd_en.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation/freebsd_en.md b/docs/installation/freebsd_en.md index 130d68766..ca2575d9b 100644 --- a/docs/installation/freebsd_en.md +++ b/docs/installation/freebsd_en.md @@ -7,7 +7,7 @@ This document was written for FreeBSD 12.1, but should be work on future release This assumes the target system has `pkg(8)`. ``` -# pkg install elixir postgresql12-server postgresql12-client postgresql12-contrib git-lite sudo nginx gmake acme.sh +# pkg install elixir postgresql12-server postgresql12-client postgresql12-contrib git-lite sudo nginx gmake acme.sh cmake ``` Copy the rc.d scripts to the right directory: From 10da13c71343623a5e52beebdc6abc1f400bc40d Mon Sep 17 00:00:00 2001 From: rinpatch <rinpatch@sdf.org> Date: Fri, 4 Sep 2020 22:10:40 +0300 Subject: [PATCH 034/128] ConnectionPool middleware: Fix connection leak on ReverseProxy redirects Requires a patched Tesla due to upstream not saving opts between redirects, patch submitted at https://github.com/teamon/tesla/pull/414 --- .../tesla/middleware/connection_pool.ex | 27 +++++++++++++------ mix.exs | 4 ++- mix.lock | 2 +- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/lib/pleroma/tesla/middleware/connection_pool.ex b/lib/pleroma/tesla/middleware/connection_pool.ex index a435ab4cc..5909e98d6 100644 --- a/lib/pleroma/tesla/middleware/connection_pool.ex +++ b/lib/pleroma/tesla/middleware/connection_pool.ex @@ -15,21 +15,32 @@ defmodule Pleroma.Tesla.Middleware.ConnectionPool do def call(%Tesla.Env{url: url, opts: opts} = env, next, _) do uri = URI.parse(url) + # Avoid leaking connections when the middleware is called twice + # with body_as: :chunks. We assume only the middleware can set + # opts[:adapter][:conn] + if opts[:adapter][:conn] do + ConnectionPool.release_conn(opts[:adapter][:conn]) + end + case ConnectionPool.get_conn(uri, opts[:adapter]) do {:ok, conn_pid} -> adapter_opts = Keyword.merge(opts[:adapter], conn: conn_pid, close_conn: false) opts = Keyword.put(opts, :adapter, adapter_opts) env = %{env | opts: opts} - res = Tesla.run(env, next) - unless opts[:adapter][:body_as] == :chunks do - ConnectionPool.release_conn(conn_pid) + case Tesla.run(env, next) do + {:ok, env} -> + unless opts[:adapter][:body_as] == :chunks do + ConnectionPool.release_conn(conn_pid) + {:ok, pop_in(env[:opts][:adapter][:conn])} + else + {:ok, env} + end + + err -> + ConnectionPool.release_conn(conn_pid) + err end - - res - - err -> - err end end end diff --git a/mix.exs b/mix.exs index 4de0c78db..c324960c5 100644 --- a/mix.exs +++ b/mix.exs @@ -134,7 +134,9 @@ defp deps do {:cachex, "~> 3.2"}, {:poison, "~> 3.0", override: true}, {:tesla, - github: "teamon/tesla", ref: "af3707078b10793f6a534938e56b963aff82fe3c", override: true}, + git: "https://git.pleroma.social/pleroma/elixir-libraries/tesla.git", + ref: "3a2789d8535f7b520ebbadc4494227e5ba0e5365", + override: true}, {:castore, "~> 0.1"}, {:cowlib, "~> 2.9", override: true}, {:gun, diff --git a/mix.lock b/mix.lock index 874ea9215..deb07eb68 100644 --- a/mix.lock +++ b/mix.lock @@ -110,7 +110,7 @@ "swoosh": {:hex, :swoosh, "1.0.0", "c547cfc83f30e12d5d1fdcb623d7de2c2e29a5becfc68bf8f42ba4d23d2c2756", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm", "b3b08e463f876cb6167f7168e9ad99a069a724e124bcee61847e0e1ed13f4a0d"}, "syslog": {:hex, :syslog, "1.1.0", "6419a232bea84f07b56dc575225007ffe34d9fdc91abe6f1b2f254fd71d8efc2", [:rebar3], [], "hexpm", "4c6a41373c7e20587be33ef841d3de6f3beba08519809329ecc4d27b15b659e1"}, "telemetry": {:hex, :telemetry, "0.4.2", "2808c992455e08d6177322f14d3bdb6b625fbcfd233a73505870d8738a2f4599", [:rebar3], [], "hexpm", "2d1419bd9dda6a206d7b5852179511722e2b18812310d304620c7bd92a13fcef"}, - "tesla": {:git, "https://github.com/teamon/tesla.git", "af3707078b10793f6a534938e56b963aff82fe3c", [ref: "af3707078b10793f6a534938e56b963aff82fe3c"]}, + "tesla": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/tesla.git", "3a2789d8535f7b520ebbadc4494227e5ba0e5365", [ref: "3a2789d8535f7b520ebbadc4494227e5ba0e5365"]}, "timex": {:hex, :timex, "3.6.2", "845cdeb6119e2fef10751c0b247b6c59d86d78554c83f78db612e3290f819bc2", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "26030b46199d02a590be61c2394b37ea25a3664c02fafbeca0b24c972025d47a"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "bd4fde4c15f3e993a999e019d64347489b91b7a9096af68b2bdadd192afa693f"}, "tzdata": {:hex, :tzdata, "1.0.3", "73470ad29dde46e350c60a66e6b360d3b99d2d18b74c4c349dbebbc27a09a3eb", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "a6e1ee7003c4d04ecbd21dd3ec690d4c6662db5d3bbdd7262d53cdf5e7c746c1"}, From 9fd0e5e0dd5e4c2b1748ef604e780bb31e4ada89 Mon Sep 17 00:00:00 2001 From: James Alseth <james@jalseth.me> Date: Fri, 4 Sep 2020 19:19:56 -0700 Subject: [PATCH 035/128] Use TLS when adding Alpine community repository in Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index aa50e27ec..c210cf79c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,7 +31,7 @@ LABEL maintainer="ops@pleroma.social" \ ARG HOME=/opt/pleroma ARG DATA=/var/lib/pleroma -RUN echo "http://nl.alpinelinux.org/alpine/latest-stable/community" >> /etc/apk/repositories &&\ +RUN echo "https://nl.alpinelinux.org/alpine/latest-stable/community" >> /etc/apk/repositories &&\ apk update &&\ apk add exiftool imagemagick ncurses postgresql-client &&\ adduser --system --shell /bin/false --home ${HOME} pleroma &&\ From 0d91f65284f99bded89c0400e976e0ffa5bc202f Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me> Date: Fri, 4 Sep 2020 07:52:22 +0200 Subject: [PATCH 036/128] Prevent AccountView and instance.get_or_update_favicon fails --- lib/pleroma/instances/instance.ex | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index 711c42158..ef5d17de4 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -145,6 +145,8 @@ def get_or_update_favicon(%URI{host: host} = instance_uri) do favicon end + rescue + _ -> nil end defp scrape_favicon(%URI{} = instance_uri) do @@ -159,7 +161,8 @@ defp scrape_favicon(%URI{} = instance_uri) do |> Floki.attribute("link[rel=icon]", "href") |> List.first(), favicon <- URI.merge(instance_uri, favicon_rel) |> to_string(), - true <- is_binary(favicon) do + true <- is_binary(favicon), + true <- String.length(favicon) <= 255 do favicon else _ -> nil From de7e2ae0b5c296d0896f5d3738cba0fcddfe54f1 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov <alex.strizhakov@gmail.com> Date: Sat, 5 Sep 2020 11:15:27 +0300 Subject: [PATCH 037/128] use override flag for hackney dependency --- mix.exs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mix.exs b/mix.exs index f734e4f61..fc1a83d91 100644 --- a/mix.exs +++ b/mix.exs @@ -195,10 +195,9 @@ defp deps do {:ex_machina, "~> 2.4", only: :test}, {:credo, "~> 1.4", only: [:dev, :test], runtime: false}, {:mock, "~> 0.3.5", only: :test}, - # temporary downgrade for excoveralls, hackney and httpoison - until hackney max_connections bug will be fixed + # temporary downgrade for excoveralls, hackney until hackney max_connections bug will be fixed {:excoveralls, "0.12.3", only: :test}, - {:hackney, "1.15.2"}, - {:httpoison, "1.6.2"}, + {:hackney, "1.15.2", override: true}, {:mox, "~> 0.5", only: :test}, {:websocket_client, git: "https://github.com/jeremyong/websocket_client.git", only: :test} ] ++ oauth_deps() From e198ba492e5cb1b6ff81775db08298bfcdf1454a Mon Sep 17 00:00:00 2001 From: rinpatch <rinpatch@sdf.org> Date: Sat, 5 Sep 2020 12:37:27 +0300 Subject: [PATCH 038/128] Rich Media: Do not cache URLs for preview statuses Closes #1987 --- CHANGELOG.md | 1 + lib/pleroma/html.ex | 32 +++++++++------- lib/pleroma/web/rich_media/helpers.ex | 2 +- test/html_test.exs | 14 +++---- .../controllers/status_controller_test.exs | 38 ++++++++++++++++++- 5 files changed, 65 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cdd5b94a8..4662045ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Search parameter `following` now correctly returns the followings rather than the followers - Mastodon API: Timelines hanging for (`number of posts with links * rich media timeout`) in the worst case. Reduced to just rich media timeout. +- Mastodon API: Cards being wrong for preview statuses due to cache key collision - Password resets no longer processed for deactivated accounts ## [2.1.0] - 2020-08-28 diff --git a/lib/pleroma/html.ex b/lib/pleroma/html.ex index 20b02f091..43e9145be 100644 --- a/lib/pleroma/html.ex +++ b/lib/pleroma/html.ex @@ -100,21 +100,27 @@ defp generate_scrubber_signature(scrubbers) do end) end - def extract_first_external_url(_, nil), do: {:error, "No content"} + def extract_first_external_url_from_object(%{data: %{"content" => content}} = object) + when is_binary(content) do + unless object.data["fake"] do + key = "URL|#{object.id}" - def extract_first_external_url(object, content) do - key = "URL|#{object.id}" + Cachex.fetch!(:scrubber_cache, key, fn _key -> + {:commit, {:ok, extract_first_external_url(content)}} + end) + else + {:ok, extract_first_external_url(content)} + end + end - Cachex.fetch!(:scrubber_cache, key, fn _key -> - result = - content - |> Floki.parse_fragment!() - |> Floki.find("a:not(.mention,.hashtag,.attachment,[rel~=\"tag\"])") - |> Enum.take(1) - |> Floki.attribute("href") - |> Enum.at(0) + def extract_first_external_url_from_object(_), do: {:error, :no_content} - {:commit, {:ok, result}} - end) + def extract_first_external_url(content) do + content + |> Floki.parse_fragment!() + |> Floki.find("a:not(.mention,.hashtag,.attachment,[rel~=\"tag\"])") + |> Enum.take(1) + |> Floki.attribute("href") + |> Enum.at(0) end end diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index 2fb482b51..752ca9f81 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -58,7 +58,7 @@ def fetch_data_for_object(object) do with true <- Config.get([:rich_media, :enabled]), false <- object.data["sensitive"] || false, {:ok, page_url} <- - HTML.extract_first_external_url(object, object.data["content"]), + HTML.extract_first_external_url_from_object(object), :ok <- validate_page_url(page_url), {:ok, rich_media} <- Parser.parse(page_url) do %{page_url: page_url, rich_media: rich_media} diff --git a/test/html_test.exs b/test/html_test.exs index f8907c8b4..7d3756884 100644 --- a/test/html_test.exs +++ b/test/html_test.exs @@ -165,7 +165,7 @@ test "filters invalid microformats markup" do end end - describe "extract_first_external_url" do + describe "extract_first_external_url_from_object" do test "extracts the url" do user = insert(:user) @@ -176,7 +176,7 @@ test "extracts the url" do }) object = Object.normalize(activity) - {:ok, url} = HTML.extract_first_external_url(object, object.data["content"]) + {:ok, url} = HTML.extract_first_external_url_from_object(object) assert url == "https://github.com/komeiji-satori/Dress" end @@ -191,7 +191,7 @@ test "skips mentions" do }) object = Object.normalize(activity) - {:ok, url} = HTML.extract_first_external_url(object, object.data["content"]) + {:ok, url} = HTML.extract_first_external_url_from_object(object) assert url == "https://github.com/syuilo/misskey/blob/develop/docs/setup.en.md" @@ -207,7 +207,7 @@ test "skips hashtags" do }) object = Object.normalize(activity) - {:ok, url} = HTML.extract_first_external_url(object, object.data["content"]) + {:ok, url} = HTML.extract_first_external_url_from_object(object) assert url == "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=72255140" end @@ -223,7 +223,7 @@ test "skips microformats hashtags" do }) object = Object.normalize(activity) - {:ok, url} = HTML.extract_first_external_url(object, object.data["content"]) + {:ok, url} = HTML.extract_first_external_url_from_object(object) assert url == "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=72255140" end @@ -235,7 +235,7 @@ test "does not crash when there is an HTML entity in a link" do object = Object.normalize(activity) - assert {:ok, nil} = HTML.extract_first_external_url(object, object.data["content"]) + assert {:ok, nil} = HTML.extract_first_external_url_from_object(object) end test "skips attachment links" do @@ -249,7 +249,7 @@ test "skips attachment links" do object = Object.normalize(activity) - assert {:ok, nil} = HTML.extract_first_external_url(object, object.data["content"]) + assert {:ok, nil} = HTML.extract_first_external_url_from_object(object) end end end diff --git a/test/web/mastodon_api/controllers/status_controller_test.exs b/test/web/mastodon_api/controllers/status_controller_test.exs index 5955d8334..f221884e7 100644 --- a/test/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/web/mastodon_api/controllers/status_controller_test.exs @@ -296,9 +296,45 @@ test "posting a fake status", %{conn: conn} do assert real_status == fake_status end + test "fake statuses' preview card is not cached", %{conn: conn} do + clear_config([:rich_media, :enabled], true) + + Tesla.Mock.mock(fn + %{ + method: :get, + url: "https://example.com/twitter-card" + } -> + %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/twitter_card.html")} + + env -> + apply(HttpRequestMock, :request, [env]) + end) + + conn1 = + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/statuses", %{ + "status" => "https://example.com/ogp", + "preview" => true + }) + + conn2 = + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/statuses", %{ + "status" => "https://example.com/twitter-card", + "preview" => true + }) + + assert %{"card" => %{"title" => "The Rock"}} = json_response_and_validate_schema(conn1, 200) + + assert %{"card" => %{"title" => "Small Island Developing States Photo Submission"}} = + json_response_and_validate_schema(conn2, 200) + end + test "posting a status with OGP link preview", %{conn: conn} do Tesla.Mock.mock(fn env -> apply(HttpRequestMock, :request, [env]) end) - Config.put([:rich_media, :enabled], true) + clear_config([:rich_media, :enabled], true) conn = conn From 5298de3be6683022fa53cc011dd567e8b2a706b9 Mon Sep 17 00:00:00 2001 From: rinpatch <rinpatch@sdf.org> Date: Sat, 5 Sep 2020 21:17:03 +0300 Subject: [PATCH 039/128] ConnectionPool middleware: fix a crash due to unimplemented behaviour Structs don't implement Access behaviour, so this crashed. Tests didn't catch it and I didn't test that part of the codepath. Very sorry --- lib/pleroma/tesla/middleware/connection_pool.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/tesla/middleware/connection_pool.ex b/lib/pleroma/tesla/middleware/connection_pool.ex index 5909e98d6..049db6eb3 100644 --- a/lib/pleroma/tesla/middleware/connection_pool.ex +++ b/lib/pleroma/tesla/middleware/connection_pool.ex @@ -32,7 +32,7 @@ def call(%Tesla.Env{url: url, opts: opts} = env, next, _) do {:ok, env} -> unless opts[:adapter][:body_as] == :chunks do ConnectionPool.release_conn(conn_pid) - {:ok, pop_in(env[:opts][:adapter][:conn])} + {:ok, pop_in(env.opts[:adapter][:conn])} else {:ok, env} end From 9d6aca5bee6f90f3c0af5a5353f052108c9def62 Mon Sep 17 00:00:00 2001 From: rinpatch <rinpatch@sdf.org> Date: Sat, 5 Sep 2020 21:27:06 +0300 Subject: [PATCH 040/128] ConnectionPool: fix the previous hotfix I rushed the hotfix and forgot how `pop_in` actually works, I want to die. We need some integration tests for the HTTP client --- lib/pleroma/tesla/middleware/connection_pool.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/tesla/middleware/connection_pool.ex b/lib/pleroma/tesla/middleware/connection_pool.ex index 049db6eb3..2c5a2b53b 100644 --- a/lib/pleroma/tesla/middleware/connection_pool.ex +++ b/lib/pleroma/tesla/middleware/connection_pool.ex @@ -32,7 +32,8 @@ def call(%Tesla.Env{url: url, opts: opts} = env, next, _) do {:ok, env} -> unless opts[:adapter][:body_as] == :chunks do ConnectionPool.release_conn(conn_pid) - {:ok, pop_in(env.opts[:adapter][:conn])} + {_, res} = pop_in(env.opts[:adapter][:conn]) + {:ok, res} else {:ok, env} end From 129a2f48df95ddd85fceee741a9991a6e092ed3d Mon Sep 17 00:00:00 2001 From: rinpatch <rinpatch@sdf.org> Date: Sat, 5 Sep 2020 21:36:17 +0300 Subject: [PATCH 041/128] ConnectionPool middleware: handle connection opening errors --- lib/pleroma/tesla/middleware/connection_pool.ex | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/pleroma/tesla/middleware/connection_pool.ex b/lib/pleroma/tesla/middleware/connection_pool.ex index 2c5a2b53b..056e736ce 100644 --- a/lib/pleroma/tesla/middleware/connection_pool.ex +++ b/lib/pleroma/tesla/middleware/connection_pool.ex @@ -42,6 +42,9 @@ def call(%Tesla.Env{url: url, opts: opts} = env, next, _) do ConnectionPool.release_conn(conn_pid) err end + + err -> + err end end end From 170599c390e7c82bdff0d4180d04b2f0f3906f35 Mon Sep 17 00:00:00 2001 From: rinpatch <rinpatch@sdf.org> Date: Sat, 5 Sep 2020 22:00:51 +0300 Subject: [PATCH 042/128] RichMedia: do not log webpages missing metadata as errors Also fixes the return value of Parser.parse on errors, previously was just `:ok` due to the logger call in the end --- lib/pleroma/web/rich_media/parser.ex | 11 ++++++++--- test/web/rich_media/parser_test.exs | 4 +--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex index e98c743ca..5727fda18 100644 --- a/lib/pleroma/web/rich_media/parser.ex +++ b/lib/pleroma/web/rich_media/parser.ex @@ -21,8 +21,13 @@ def parse(url) do {:ok, _} <- set_ttl_based_on_image(data, url) do {:ok, data} else + {:error, {:invalid_metadata, data}} = e -> + Logger.debug(fn -> "Incomplete or invalid metadata for #{url}: #{inspect(data)}" end) + e + error -> - Logger.error(fn -> "Rich media error: #{inspect(error)}" end) + Logger.error(fn -> "Rich media error for #{url}: #{inspect(error)}" end) + error end end @@ -90,7 +95,7 @@ defp get_ttl_from_image(data, url) do end) end - defp parse_url(url) do + def parse_url(url) do with {:ok, %Tesla.Env{body: html}} <- Pleroma.Web.RichMedia.Helpers.rich_media_get(url), {:ok, html} <- Floki.parse_document(html) do html @@ -116,7 +121,7 @@ defp check_parsed_data(%{"title" => title} = data) end defp check_parsed_data(data) do - {:error, "Found metadata was invalid or incomplete: #{inspect(data)}"} + {:error, {:invalid_metadata, data}} end defp clean_parsed_data(data) do diff --git a/test/web/rich_media/parser_test.exs b/test/web/rich_media/parser_test.exs index 1e09cbf84..21ae35f8b 100644 --- a/test/web/rich_media/parser_test.exs +++ b/test/web/rich_media/parser_test.exs @@ -66,9 +66,7 @@ test "returns error when no metadata present" do end test "doesn't just add a title" do - assert Parser.parse("http://example.com/non-ogp") == - {:error, - "Found metadata was invalid or incomplete: %{\"url\" => \"http://example.com/non-ogp\"}"} + assert {:error, {:invalid_metadata, _}} = Parser.parse("http://example.com/non-ogp") end test "parses ogp" do From 0b4fa769f4332e50cdc5f643bada18f2e50430de Mon Sep 17 00:00:00 2001 From: rinpatch <rinpatch@sdf.org> Date: Sun, 6 Sep 2020 11:38:38 +0300 Subject: [PATCH 043/128] Add a copy of CC-BY-4.0 to the repo We mentined it in COPYING, but didn't actually have a copy in the repo. --- CC-BY-4.0 | 395 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 395 insertions(+) create mode 100644 CC-BY-4.0 diff --git a/CC-BY-4.0 b/CC-BY-4.0 new file mode 100644 index 000000000..4ea99c213 --- /dev/null +++ b/CC-BY-4.0 @@ -0,0 +1,395 @@ +Attribution 4.0 International + +======================================================================= + +Creative Commons Corporation ("Creative Commons") is not a law firm and +does not provide legal services or legal advice. Distribution of +Creative Commons public licenses does not create a lawyer-client or +other relationship. Creative Commons makes its licenses and related +information available on an "as-is" basis. Creative Commons gives no +warranties regarding its licenses, any material licensed under their +terms and conditions, or any related information. Creative Commons +disclaims all liability for damages resulting from their use to the +fullest extent possible. + +Using Creative Commons Public Licenses + +Creative Commons public licenses provide a standard set of terms and +conditions that creators and other rights holders may use to share +original works of authorship and other material subject to copyright +and certain other rights specified in the public license below. The +following considerations are for informational purposes only, are not +exhaustive, and do not form part of our licenses. + + Considerations for licensors: Our public licenses are + intended for use by those authorized to give the public + permission to use material in ways otherwise restricted by + copyright and certain other rights. Our licenses are + irrevocable. Licensors should read and understand the terms + and conditions of the license they choose before applying it. + Licensors should also secure all rights necessary before + applying our licenses so that the public can reuse the + material as expected. Licensors should clearly mark any + material not subject to the license. This includes other CC- + licensed material, or material used under an exception or + limitation to copyright. More considerations for licensors: + wiki.creativecommons.org/Considerations_for_licensors + + Considerations for the public: By using one of our public + licenses, a licensor grants the public permission to use the + licensed material under specified terms and conditions. If + the licensor's permission is not necessary for any reason--for + example, because of any applicable exception or limitation to + copyright--then that use is not regulated by the license. Our + licenses grant only permissions under copyright and certain + other rights that a licensor has authority to grant. Use of + the licensed material may still be restricted for other + reasons, including because others have copyright or other + rights in the material. A licensor may make special requests, + such as asking that all changes be marked or described. + Although not required by our licenses, you are encouraged to + respect those requests where reasonable. More considerations + for the public: + wiki.creativecommons.org/Considerations_for_licensees + +======================================================================= + +Creative Commons Attribution 4.0 International Public License + +By exercising the Licensed Rights (defined below), You accept and agree +to be bound by the terms and conditions of this Creative Commons +Attribution 4.0 International Public License ("Public License"). To the +extent this Public License may be interpreted as a contract, You are +granted the Licensed Rights in consideration of Your acceptance of +these terms and conditions, and the Licensor grants You such rights in +consideration of benefits the Licensor receives from making the +Licensed Material available under these terms and conditions. + + +Section 1 -- Definitions. + + a. Adapted Material means material subject to Copyright and Similar + Rights that is derived from or based upon the Licensed Material + and in which the Licensed Material is translated, altered, + arranged, transformed, or otherwise modified in a manner requiring + permission under the Copyright and Similar Rights held by the + Licensor. For purposes of this Public License, where the Licensed + Material is a musical work, performance, or sound recording, + Adapted Material is always produced where the Licensed Material is + synched in timed relation with a moving image. + + b. Adapter's License means the license You apply to Your Copyright + and Similar Rights in Your contributions to Adapted Material in + accordance with the terms and conditions of this Public License. + + c. Copyright and Similar Rights means copyright and/or similar rights + closely related to copyright including, without limitation, + performance, broadcast, sound recording, and Sui Generis Database + Rights, without regard to how the rights are labeled or + categorized. For purposes of this Public License, the rights + specified in Section 2(b)(1)-(2) are not Copyright and Similar + Rights. + + d. Effective Technological Measures means those measures that, in the + absence of proper authority, may not be circumvented under laws + fulfilling obligations under Article 11 of the WIPO Copyright + Treaty adopted on December 20, 1996, and/or similar international + agreements. + + e. Exceptions and Limitations means fair use, fair dealing, and/or + any other exception or limitation to Copyright and Similar Rights + that applies to Your use of the Licensed Material. + + f. Licensed Material means the artistic or literary work, database, + or other material to which the Licensor applied this Public + License. + + g. Licensed Rights means the rights granted to You subject to the + terms and conditions of this Public License, which are limited to + all Copyright and Similar Rights that apply to Your use of the + Licensed Material and that the Licensor has authority to license. + + h. Licensor means the individual(s) or entity(ies) granting rights + under this Public License. + + i. Share means to provide material to the public by any means or + process that requires permission under the Licensed Rights, such + as reproduction, public display, public performance, distribution, + dissemination, communication, or importation, and to make material + available to the public including in ways that members of the + public may access the material from a place and at a time + individually chosen by them. + + j. Sui Generis Database Rights means rights other than copyright + resulting from Directive 96/9/EC of the European Parliament and of + the Council of 11 March 1996 on the legal protection of databases, + as amended and/or succeeded, as well as other essentially + equivalent rights anywhere in the world. + + k. You means the individual or entity exercising the Licensed Rights + under this Public License. Your has a corresponding meaning. + + +Section 2 -- Scope. + + a. License grant. + + 1. Subject to the terms and conditions of this Public License, + the Licensor hereby grants You a worldwide, royalty-free, + non-sublicensable, non-exclusive, irrevocable license to + exercise the Licensed Rights in the Licensed Material to: + + a. reproduce and Share the Licensed Material, in whole or + in part; and + + b. produce, reproduce, and Share Adapted Material. + + 2. Exceptions and Limitations. For the avoidance of doubt, where + Exceptions and Limitations apply to Your use, this Public + License does not apply, and You do not need to comply with + its terms and conditions. + + 3. Term. The term of this Public License is specified in Section + 6(a). + + 4. Media and formats; technical modifications allowed. The + Licensor authorizes You to exercise the Licensed Rights in + all media and formats whether now known or hereafter created, + and to make technical modifications necessary to do so. The + Licensor waives and/or agrees not to assert any right or + authority to forbid You from making technical modifications + necessary to exercise the Licensed Rights, including + technical modifications necessary to circumvent Effective + Technological Measures. For purposes of this Public License, + simply making modifications authorized by this Section 2(a) + (4) never produces Adapted Material. + + 5. Downstream recipients. + + a. Offer from the Licensor -- Licensed Material. Every + recipient of the Licensed Material automatically + receives an offer from the Licensor to exercise the + Licensed Rights under the terms and conditions of this + Public License. + + b. No downstream restrictions. You may not offer or impose + any additional or different terms or conditions on, or + apply any Effective Technological Measures to, the + Licensed Material if doing so restricts exercise of the + Licensed Rights by any recipient of the Licensed + Material. + + 6. No endorsement. Nothing in this Public License constitutes or + may be construed as permission to assert or imply that You + are, or that Your use of the Licensed Material is, connected + with, or sponsored, endorsed, or granted official status by, + the Licensor or others designated to receive attribution as + provided in Section 3(a)(1)(A)(i). + + b. Other rights. + + 1. Moral rights, such as the right of integrity, are not + licensed under this Public License, nor are publicity, + privacy, and/or other similar personality rights; however, to + the extent possible, the Licensor waives and/or agrees not to + assert any such rights held by the Licensor to the limited + extent necessary to allow You to exercise the Licensed + Rights, but not otherwise. + + 2. Patent and trademark rights are not licensed under this + Public License. + + 3. To the extent possible, the Licensor waives any right to + collect royalties from You for the exercise of the Licensed + Rights, whether directly or through a collecting society + under any voluntary or waivable statutory or compulsory + licensing scheme. In all other cases the Licensor expressly + reserves any right to collect such royalties. + + +Section 3 -- License Conditions. + +Your exercise of the Licensed Rights is expressly made subject to the +following conditions. + + a. Attribution. + + 1. If You Share the Licensed Material (including in modified + form), You must: + + a. retain the following if it is supplied by the Licensor + with the Licensed Material: + + i. identification of the creator(s) of the Licensed + Material and any others designated to receive + attribution, in any reasonable manner requested by + the Licensor (including by pseudonym if + designated); + + ii. a copyright notice; + + iii. a notice that refers to this Public License; + + iv. a notice that refers to the disclaimer of + warranties; + + v. a URI or hyperlink to the Licensed Material to the + extent reasonably practicable; + + b. indicate if You modified the Licensed Material and + retain an indication of any previous modifications; and + + c. indicate the Licensed Material is licensed under this + Public License, and include the text of, or the URI or + hyperlink to, this Public License. + + 2. You may satisfy the conditions in Section 3(a)(1) in any + reasonable manner based on the medium, means, and context in + which You Share the Licensed Material. For example, it may be + reasonable to satisfy the conditions by providing a URI or + hyperlink to a resource that includes the required + information. + + 3. If requested by the Licensor, You must remove any of the + information required by Section 3(a)(1)(A) to the extent + reasonably practicable. + + 4. If You Share Adapted Material You produce, the Adapter's + License You apply must not prevent recipients of the Adapted + Material from complying with this Public License. + + +Section 4 -- Sui Generis Database Rights. + +Where the Licensed Rights include Sui Generis Database Rights that +apply to Your use of the Licensed Material: + + a. for the avoidance of doubt, Section 2(a)(1) grants You the right + to extract, reuse, reproduce, and Share all or a substantial + portion of the contents of the database; + + b. if You include all or a substantial portion of the database + contents in a database in which You have Sui Generis Database + Rights, then the database in which You have Sui Generis Database + Rights (but not its individual contents) is Adapted Material; and + + c. You must comply with the conditions in Section 3(a) if You Share + all or a substantial portion of the contents of the database. + +For the avoidance of doubt, this Section 4 supplements and does not +replace Your obligations under this Public License where the Licensed +Rights include other Copyright and Similar Rights. + + +Section 5 -- Disclaimer of Warranties and Limitation of Liability. + + a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE + EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS + AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF + ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS, + IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION, + WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR + PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS, + ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT + KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT + ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU. + + b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE + TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION, + NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT, + INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES, + COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR + USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN + ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR + DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR + IN PART, THIS LIMITATION MAY NOT APPLY TO YOU. + + c. The disclaimer of warranties and limitation of liability provided + above shall be interpreted in a manner that, to the extent + possible, most closely approximates an absolute disclaimer and + waiver of all liability. + + +Section 6 -- Term and Termination. + + a. This Public License applies for the term of the Copyright and + Similar Rights licensed here. However, if You fail to comply with + this Public License, then Your rights under this Public License + terminate automatically. + + b. Where Your right to use the Licensed Material has terminated under + Section 6(a), it reinstates: + + 1. automatically as of the date the violation is cured, provided + it is cured within 30 days of Your discovery of the + violation; or + + 2. upon express reinstatement by the Licensor. + + For the avoidance of doubt, this Section 6(b) does not affect any + right the Licensor may have to seek remedies for Your violations + of this Public License. + + c. For the avoidance of doubt, the Licensor may also offer the + Licensed Material under separate terms or conditions or stop + distributing the Licensed Material at any time; however, doing so + will not terminate this Public License. + + d. Sections 1, 5, 6, 7, and 8 survive termination of this Public + License. + + +Section 7 -- Other Terms and Conditions. + + a. The Licensor shall not be bound by any additional or different + terms or conditions communicated by You unless expressly agreed. + + b. Any arrangements, understandings, or agreements regarding the + Licensed Material not stated herein are separate from and + independent of the terms and conditions of this Public License. + + +Section 8 -- Interpretation. + + a. For the avoidance of doubt, this Public License does not, and + shall not be interpreted to, reduce, limit, restrict, or impose + conditions on any use of the Licensed Material that could lawfully + be made without permission under this Public License. + + b. To the extent possible, if any provision of this Public License is + deemed unenforceable, it shall be automatically reformed to the + minimum extent necessary to make it enforceable. If the provision + cannot be reformed, it shall be severed from this Public License + without affecting the enforceability of the remaining terms and + conditions. + + c. No term or condition of this Public License will be waived and no + failure to comply consented to unless expressly agreed to by the + Licensor. + + d. Nothing in this Public License constitutes or may be interpreted + as a limitation upon, or waiver of, any privileges and immunities + that apply to the Licensor or You, including from the legal + processes of any jurisdiction or authority. + + +======================================================================= + +Creative Commons is not a party to its public +licenses. Notwithstanding, Creative Commons may elect to apply one of +its public licenses to material it publishes and in those instances +will be considered the “Licensor.” The text of the Creative Commons +public licenses is dedicated to the public domain under the CC0 Public +Domain Dedication. Except for the limited purpose of indicating that +material is shared under a Creative Commons public license or as +otherwise permitted by the Creative Commons policies published at +creativecommons.org/policies, Creative Commons does not authorize the +use of the trademark "Creative Commons" or any other trademark or logo +of Creative Commons without its prior written consent including, +without limitation, in connection with any unauthorized modifications +to any of its public licenses or any other arrangements, +understandings, or agreements concerning use of licensed material. For +the avoidance of doubt, this paragraph does not form part of the +public licenses. + +Creative Commons may be contacted at creativecommons.org. From 08aef7dd4e054c5ed02e359b61fe57daad97fbde Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me> Date: Sat, 5 Sep 2020 06:38:07 +0200 Subject: [PATCH 044/128] instance: Log catch favicon errors as warnings --- lib/pleroma/instances/instance.ex | 16 ++++++++++---- test/web/instances/instance_test.exs | 33 ++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index ef5d17de4..8bf53c090 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -14,6 +14,8 @@ defmodule Pleroma.Instances.Instance do import Ecto.Query import Ecto.Changeset + require Logger + schema "instances" do field(:host, :string) field(:unreachable_since, :naive_datetime_usec) @@ -146,7 +148,9 @@ def get_or_update_favicon(%URI{host: host} = instance_uri) do favicon end rescue - _ -> nil + e -> + Logger.warn("Instance.get_or_update_favicon(\"#{host}\") error: #{inspect(e)}") + nil end defp scrape_favicon(%URI{} = instance_uri) do @@ -161,14 +165,18 @@ defp scrape_favicon(%URI{} = instance_uri) do |> Floki.attribute("link[rel=icon]", "href") |> List.first(), favicon <- URI.merge(instance_uri, favicon_rel) |> to_string(), - true <- is_binary(favicon), - true <- String.length(favicon) <= 255 do + true <- is_binary(favicon) do favicon else _ -> nil end rescue - _ -> nil + e -> + Logger.warn( + "Instance.scrape_favicon(\"#{to_string(instance_uri)}\") error: #{inspect(e)}" + ) + + nil end end end diff --git a/test/web/instances/instance_test.exs b/test/web/instances/instance_test.exs index e463200ca..dc6ace843 100644 --- a/test/web/instances/instance_test.exs +++ b/test/web/instances/instance_test.exs @@ -8,6 +8,7 @@ defmodule Pleroma.Instances.InstanceTest do use Pleroma.DataCase + import ExUnit.CaptureLog import Pleroma.Factory setup_all do: clear_config([:instance, :federation_reachability_timeout_days], 1) @@ -97,4 +98,36 @@ test "does NOT modify `unreachable_since` value of existing record in case it's assert initial_value == instance.unreachable_since end end + + test "Scrapes favicon URLs" do + Tesla.Mock.mock(fn %{url: "https://favicon.example.org/"} -> + %Tesla.Env{ + status: 200, + body: ~s[<html><head><link rel="icon" href="/favicon.png"></head></html>] + } + end) + + assert "https://favicon.example.org/favicon.png" == + Instance.get_or_update_favicon(URI.parse("https://favicon.example.org/")) + end + + test "Returns nil on too long favicon URLs" do + long_favicon_url = + "https://Lorem.ipsum.dolor.sit.amet/consecteturadipiscingelit/Praesentpharetrapurusutaliquamtempus/Mauriseulaoreetarcu/atfacilisisorci/Nullamporttitor/nequesedfeugiatmollis/dolormagnaefficiturlorem/nonpretiumsapienorcieurisus/Nullamveleratsem/Maecenassedaccumsanexnam/favicon.png" + + Tesla.Mock.mock(fn %{url: "https://long-favicon.example.org/"} -> + %Tesla.Env{ + status: 200, + body: ~s[<html><head><link rel="icon" href="] <> long_favicon_url <> ~s["></head></html>] + } + end) + + assert capture_log(fn -> + assert nil == + Instance.get_or_update_favicon( + URI.parse("https://long-favicon.example.org/") + ) + end) =~ + "Instance.get_or_update_favicon(\"long-favicon.example.org\") error: %Postgrex.Error{" + end end From 1984ff310341766adfc224342c9d29272f80e9cb Mon Sep 17 00:00:00 2001 From: rinpatch <rinpatch@sdf.org> Date: Mon, 7 Sep 2020 15:16:04 +0300 Subject: [PATCH 045/128] Add a changelog entry for hackney downgrade --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cdd5b94a8..25645c185 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Rich media failure tracking (along with `:failure_backoff` option) ### Fixed +- Possible OOM errors with the default HTTP adapter - Mastodon API: Search parameter `following` now correctly returns the followings rather than the followers - Mastodon API: Timelines hanging for (`number of posts with links * rich media timeout`) in the worst case. Reduced to just rich media timeout. From 8628e1b216339eaf0c294aa8f01911a9399d912d Mon Sep 17 00:00:00 2001 From: rinpatch <rinpatch@sdf.org> Date: Mon, 7 Sep 2020 15:21:20 +0300 Subject: [PATCH 046/128] switch back to upstream tesla The patch we required got merged upstream: https://github.com/teamon/tesla/commit/9f7261ca49f9f901ceb73b60219ad6f8a9f6aa30 --- mix.exs | 4 ++-- mix.lock | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mix.exs b/mix.exs index c324960c5..8f69a15f2 100644 --- a/mix.exs +++ b/mix.exs @@ -134,8 +134,8 @@ defp deps do {:cachex, "~> 3.2"}, {:poison, "~> 3.0", override: true}, {:tesla, - git: "https://git.pleroma.social/pleroma/elixir-libraries/tesla.git", - ref: "3a2789d8535f7b520ebbadc4494227e5ba0e5365", + git: "https://github.com/teamon/tesla/", + ref: "9f7261ca49f9f901ceb73b60219ad6f8a9f6aa30", override: true}, {:castore, "~> 0.1"}, {:cowlib, "~> 2.9", override: true}, diff --git a/mix.lock b/mix.lock index deb07eb68..058428e66 100644 --- a/mix.lock +++ b/mix.lock @@ -110,7 +110,7 @@ "swoosh": {:hex, :swoosh, "1.0.0", "c547cfc83f30e12d5d1fdcb623d7de2c2e29a5becfc68bf8f42ba4d23d2c2756", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm", "b3b08e463f876cb6167f7168e9ad99a069a724e124bcee61847e0e1ed13f4a0d"}, "syslog": {:hex, :syslog, "1.1.0", "6419a232bea84f07b56dc575225007ffe34d9fdc91abe6f1b2f254fd71d8efc2", [:rebar3], [], "hexpm", "4c6a41373c7e20587be33ef841d3de6f3beba08519809329ecc4d27b15b659e1"}, "telemetry": {:hex, :telemetry, "0.4.2", "2808c992455e08d6177322f14d3bdb6b625fbcfd233a73505870d8738a2f4599", [:rebar3], [], "hexpm", "2d1419bd9dda6a206d7b5852179511722e2b18812310d304620c7bd92a13fcef"}, - "tesla": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/tesla.git", "3a2789d8535f7b520ebbadc4494227e5ba0e5365", [ref: "3a2789d8535f7b520ebbadc4494227e5ba0e5365"]}, + "tesla": {:git, "https://github.com/teamon/tesla/", "9f7261ca49f9f901ceb73b60219ad6f8a9f6aa30", [ref: "9f7261ca49f9f901ceb73b60219ad6f8a9f6aa30"]}, "timex": {:hex, :timex, "3.6.2", "845cdeb6119e2fef10751c0b247b6c59d86d78554c83f78db612e3290f819bc2", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "26030b46199d02a590be61c2394b37ea25a3664c02fafbeca0b24c972025d47a"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "bd4fde4c15f3e993a999e019d64347489b91b7a9096af68b2bdadd192afa693f"}, "tzdata": {:hex, :tzdata, "1.0.3", "73470ad29dde46e350c60a66e6b360d3b99d2d18b74c4c349dbebbc27a09a3eb", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "a6e1ee7003c4d04ecbd21dd3ec690d4c6662db5d3bbdd7262d53cdf5e7c746c1"}, From ee67c98e550310813cfdb9242e5fab2e566e1e2a Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov <alex.strizhakov@gmail.com> Date: Sun, 6 Sep 2020 12:13:26 +0300 Subject: [PATCH 047/128] removing Stats worker from Oban cron jobs --- CHANGELOG.md | 8 ++ config/config.exs | 3 +- config/description.exs | 1 - lib/mix/pleroma.ex | 1 + lib/pleroma/application.ex | 1 + lib/pleroma/config/oban.ex | 30 ++++++++ lib/pleroma/stats.ex | 76 ++++++++++++++----- lib/pleroma/workers/cron/stats_worker.ex | 17 ----- ...ove_cron_stats_worker_from_oban_config.exs | 19 +++++ test/stats_test.exs | 21 ++--- 10 files changed, 127 insertions(+), 50 deletions(-) create mode 100644 lib/pleroma/config/oban.ex delete mode 100644 lib/pleroma/workers/cron/stats_worker.ex create mode 100644 priv/repo/migrations/20200906072147_remove_cron_stats_worker_from_oban_config.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e6353a4e..78aae6f07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,9 +6,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## unreleased-patch - ??? ### Added + - Rich media failure tracking (along with `:failure_backoff` option) ### Fixed + - Possible OOM errors with the default HTTP adapter - Mastodon API: Search parameter `following` now correctly returns the followings rather than the followers - Mastodon API: Timelines hanging for (`number of posts with links * rich media timeout`) in the worst case. @@ -16,6 +18,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Cards being wrong for preview statuses due to cache key collision - Password resets no longer processed for deactivated accounts +## Unreleased + +### Removed + +- **Breaking:** Removed `Pleroma.Workers.Cron.StatsWorker` setting from Oban `:crontab`. + ## [2.1.0] - 2020-08-28 ### Changed diff --git a/config/config.exs b/config/config.exs index ed37b93c0..d631c3962 100644 --- a/config/config.exs +++ b/config/config.exs @@ -546,7 +546,6 @@ plugins: [Oban.Plugins.Pruner], crontab: [ {"0 0 * * *", Pleroma.Workers.Cron.ClearOauthTokenWorker}, - {"0 * * * *", Pleroma.Workers.Cron.StatsWorker}, {"* * * * *", Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker}, {"0 0 * * 0", Pleroma.Workers.Cron.DigestEmailsWorker}, {"0 0 * * *", Pleroma.Workers.Cron.NewUsersDigestWorker} @@ -672,7 +671,7 @@ # With no frontend configuration, the bundled files from the `static` directory will # be used. # -# config :pleroma, :frontends, +# config :pleroma, :frontends, # primary: %{"name" => "pleroma-fe", "ref" => "develop"}, # admin: %{"name" => "admin-fe", "ref" => "stable"}, # available: %{...} diff --git a/config/description.exs b/config/description.exs index 5e08ba109..18c133f02 100644 --- a/config/description.exs +++ b/config/description.exs @@ -2291,7 +2291,6 @@ description: "Settings for cron background jobs", suggestions: [ {"0 0 * * *", Pleroma.Workers.Cron.ClearOauthTokenWorker}, - {"0 * * * *", Pleroma.Workers.Cron.StatsWorker}, {"* * * * *", Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker}, {"0 0 * * 0", Pleroma.Workers.Cron.DigestEmailsWorker}, {"0 0 * * *", Pleroma.Workers.Cron.NewUsersDigestWorker} diff --git a/lib/mix/pleroma.ex b/lib/mix/pleroma.ex index fe9b0d16c..49ba2aae4 100644 --- a/lib/mix/pleroma.ex +++ b/lib/mix/pleroma.ex @@ -18,6 +18,7 @@ defmodule Mix.Pleroma do @doc "Common functions to be reused in mix tasks" def start_pleroma do Pleroma.Config.Holder.save_default() + Pleroma.Config.Oban.warn() Application.put_env(:phoenix, :serve_endpoints, false, persistent: true) if Pleroma.Config.get(:env) != :test do diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 33b1e3872..c39e24919 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -50,6 +50,7 @@ def start(_type, _args) do Pleroma.Telemetry.Logger.attach() Config.Holder.save_default() Pleroma.HTML.compile_scrubbers() + Pleroma.Config.Oban.warn() Config.DeprecationWarnings.warn() Pleroma.Plugs.HTTPSecurityPlug.warn_if_disabled() Pleroma.ApplicationRequirements.verify!() diff --git a/lib/pleroma/config/oban.ex b/lib/pleroma/config/oban.ex new file mode 100644 index 000000000..c2d56ebab --- /dev/null +++ b/lib/pleroma/config/oban.ex @@ -0,0 +1,30 @@ +defmodule Pleroma.Config.Oban do + require Logger + + def warn do + oban_config = Pleroma.Config.get(Oban) + + crontab = + [Pleroma.Workers.Cron.StatsWorker] + |> Enum.reduce(oban_config[:crontab], fn removed_worker, acc -> + with acc when is_list(acc) <- acc, + setting when is_tuple(setting) <- + Enum.find(acc, fn {_, worker} -> worker == removed_worker end) do + """ + !!!OBAN CONFIG WARNING!!! + You are using old workers in Oban crontab settings, which were removed. + Please, remove setting from crontab in your config file (prod.secret.exs): #{ + inspect(setting) + } + """ + |> Logger.warn() + + List.delete(acc, setting) + else + _ -> acc + end + end) + + Pleroma.Config.put(Oban, Keyword.put(oban_config, :crontab, crontab)) + end +end diff --git a/lib/pleroma/stats.ex b/lib/pleroma/stats.ex index 9a03f01db..e7f8d272c 100644 --- a/lib/pleroma/stats.ex +++ b/lib/pleroma/stats.ex @@ -3,12 +3,15 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Stats do + use GenServer + import Ecto.Query + alias Pleroma.CounterCache alias Pleroma.Repo alias Pleroma.User - use GenServer + @interval :timer.seconds(60) def start_link(_) do GenServer.start_link( @@ -18,6 +21,11 @@ def start_link(_) do ) end + @impl true + def init(_args) do + {:ok, nil, {:continue, :calculate_stats}} + end + @doc "Performs update stats" def force_update do GenServer.call(__MODULE__, :force_update) @@ -29,7 +37,11 @@ def do_collect do end @doc "Returns stats data" - @spec get_stats() :: %{domain_count: integer(), status_count: integer(), user_count: integer()} + @spec get_stats() :: %{ + domain_count: non_neg_integer(), + status_count: non_neg_integer(), + user_count: non_neg_integer() + } def get_stats do %{stats: stats} = GenServer.call(__MODULE__, :get_state) @@ -44,25 +56,14 @@ def get_peers do peers end - def init(_args) do - {:ok, calculate_stat_data()} - end - - def handle_call(:force_update, _from, _state) do - new_stats = calculate_stat_data() - {:reply, new_stats, new_stats} - end - - def handle_call(:get_state, _from, state) do - {:reply, state, state} - end - - def handle_cast(:run_update, _state) do - new_stats = calculate_stat_data() - - {:noreply, new_stats} - end - + @spec calculate_stat_data() :: %{ + peers: list(), + stats: %{ + domain_count: non_neg_integer(), + status_count: non_neg_integer(), + user_count: non_neg_integer() + } + } def calculate_stat_data do peers = from( @@ -97,6 +98,7 @@ def calculate_stat_data do } end + @spec get_status_visibility_count(String.t() | nil) :: map() def get_status_visibility_count(instance \\ nil) do if is_nil(instance) do CounterCache.get_sum() @@ -104,4 +106,36 @@ def get_status_visibility_count(instance \\ nil) do CounterCache.get_by_instance(instance) end end + + @impl true + def handle_continue(:calculate_stats, _) do + stats = calculate_stat_data() + Process.send_after(self(), :run_update, @interval) + {:noreply, stats} + end + + @impl true + def handle_call(:force_update, _from, _state) do + new_stats = calculate_stat_data() + {:reply, new_stats, new_stats} + end + + @impl true + def handle_call(:get_state, _from, state) do + {:reply, state, state} + end + + @impl true + def handle_cast(:run_update, _state) do + new_stats = calculate_stat_data() + + {:noreply, new_stats} + end + + @impl true + def handle_info(:run_update, _) do + new_stats = calculate_stat_data() + Process.send_after(self(), :run_update, @interval) + {:noreply, new_stats} + end end diff --git a/lib/pleroma/workers/cron/stats_worker.ex b/lib/pleroma/workers/cron/stats_worker.ex deleted file mode 100644 index 6a79540bc..000000000 --- a/lib/pleroma/workers/cron/stats_worker.ex +++ /dev/null @@ -1,17 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Workers.Cron.StatsWorker do - @moduledoc """ - The worker to update peers statistics. - """ - - use Oban.Worker, queue: "background" - - @impl Oban.Worker - def perform(_job) do - Pleroma.Stats.do_collect() - :ok - end -end diff --git a/priv/repo/migrations/20200906072147_remove_cron_stats_worker_from_oban_config.exs b/priv/repo/migrations/20200906072147_remove_cron_stats_worker_from_oban_config.exs new file mode 100644 index 000000000..022f21dc7 --- /dev/null +++ b/priv/repo/migrations/20200906072147_remove_cron_stats_worker_from_oban_config.exs @@ -0,0 +1,19 @@ +defmodule Pleroma.Repo.Migrations.RemoveCronStatsWorkerFromObanConfig do + use Ecto.Migration + + def change do + with %Pleroma.ConfigDB{} = config <- + Pleroma.ConfigDB.get_by_params(%{group: :pleroma, key: Oban}), + crontab when is_list(crontab) <- config.value[:crontab], + index when is_integer(index) <- + Enum.find_index(crontab, fn {_, worker} -> + worker == Pleroma.Workers.Cron.StatsWorker + end) do + updated_value = Keyword.put(config.value, :crontab, List.delete_at(crontab, index)) + + config + |> Ecto.Changeset.change(value: updated_value) + |> Pleroma.Repo.update() + end + end +end diff --git a/test/stats_test.exs b/test/stats_test.exs index f09d8d31a..74bf785b0 100644 --- a/test/stats_test.exs +++ b/test/stats_test.exs @@ -4,7 +4,10 @@ defmodule Pleroma.StatsTest do use Pleroma.DataCase + import Pleroma.Factory + + alias Pleroma.Stats alias Pleroma.Web.CommonAPI describe "user count" do @@ -13,7 +16,7 @@ test "it ignores internal users" do _internal = insert(:user, local: true, nickname: nil) _internal = Pleroma.Web.ActivityPub.Relay.get_actor() - assert match?(%{stats: %{user_count: 1}}, Pleroma.Stats.calculate_stat_data()) + assert match?(%{stats: %{user_count: 1}}, Stats.calculate_stat_data()) end end @@ -47,23 +50,23 @@ test "on new status" do end) assert %{"direct" => 3, "private" => 4, "public" => 1, "unlisted" => 2} = - Pleroma.Stats.get_status_visibility_count() + Stats.get_status_visibility_count() end test "on status delete" do user = insert(:user) {:ok, activity} = CommonAPI.post(user, %{visibility: "public", status: "hey"}) - assert %{"public" => 1} = Pleroma.Stats.get_status_visibility_count() + assert %{"public" => 1} = Stats.get_status_visibility_count() CommonAPI.delete(activity.id, user) - assert %{"public" => 0} = Pleroma.Stats.get_status_visibility_count() + assert %{"public" => 0} = Stats.get_status_visibility_count() end test "on status visibility update" do user = insert(:user) {:ok, activity} = CommonAPI.post(user, %{visibility: "public", status: "hey"}) - assert %{"public" => 1, "private" => 0} = Pleroma.Stats.get_status_visibility_count() + assert %{"public" => 1, "private" => 0} = Stats.get_status_visibility_count() {:ok, _} = CommonAPI.update_activity_scope(activity.id, %{visibility: "private"}) - assert %{"public" => 0, "private" => 1} = Pleroma.Stats.get_status_visibility_count() + assert %{"public" => 0, "private" => 1} = Stats.get_status_visibility_count() end test "doesn't count unrelated activities" do @@ -75,7 +78,7 @@ test "doesn't count unrelated activities" do CommonAPI.repeat(activity.id, other_user) assert %{"direct" => 0, "private" => 0, "public" => 1, "unlisted" => 0} = - Pleroma.Stats.get_status_visibility_count() + Stats.get_status_visibility_count() end end @@ -110,10 +113,10 @@ test "single instance" do end) assert %{"direct" => 10, "private" => 0, "public" => 1, "unlisted" => 5} = - Pleroma.Stats.get_status_visibility_count(local_instance) + Stats.get_status_visibility_count(local_instance) assert %{"direct" => 0, "private" => 20, "public" => 0, "unlisted" => 0} = - Pleroma.Stats.get_status_visibility_count(instance2) + Stats.get_status_visibility_count(instance2) end end end From 8c6485c470c1a7cd6ec4b63867d94e9724c4502b Mon Sep 17 00:00:00 2001 From: rinpatch <rinpatch@sdf.org> Date: Mon, 7 Sep 2020 19:22:56 +0300 Subject: [PATCH 048/128] CHANGELOG.md: move Unreleased section ahead of unreleased-patch --- CHANGELOG.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 78aae6f07..46f1e859b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,13 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## Unreleased + +### Removed + +- **Breaking:** Removed `Pleroma.Workers.Cron.StatsWorker` setting from Oban `:crontab`. + + ## unreleased-patch - ??? ### Added @@ -18,11 +25,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Cards being wrong for preview statuses due to cache key collision - Password resets no longer processed for deactivated accounts -## Unreleased - -### Removed - -- **Breaking:** Removed `Pleroma.Workers.Cron.StatsWorker` setting from Oban `:crontab`. ## [2.1.0] - 2020-08-28 From a83916fdacac7b11ca478ef9a61b32dd269c8fd2 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov <alex.strizhakov@gmail.com> Date: Fri, 4 Sep 2020 19:05:08 +0300 Subject: [PATCH 049/128] adapter options unification not needed options deletion --- config/config.exs | 10 +++++----- config/description.exs | 8 +++++++- docs/configuration/cheatsheet.md | 4 ++-- lib/mix/tasks/pleroma/frontend.ex | 4 +--- lib/pleroma/gun/conn.ex | 8 ++++---- lib/pleroma/http/adapter_helper.ex | 2 +- lib/pleroma/http/adapter_helper/gun.ex | 14 ++++++-------- lib/pleroma/http/adapter_helper/hackney.ex | 14 ++++++++++---- .../mrf/media_proxy_warming_policy.ex | 12 +++--------- lib/pleroma/web/rel_me.ex | 15 +++------------ lib/pleroma/web/rich_media/helpers.ex | 19 +++++-------------- 11 files changed, 47 insertions(+), 63 deletions(-) diff --git a/config/config.exs b/config/config.exs index d631c3962..2426fbd52 100644 --- a/config/config.exs +++ b/config/config.exs @@ -735,28 +735,28 @@ max_connections: 250, max_idle_time: 30_000, retry: 0, - await_up_timeout: 5_000 + connect_timeout: 5_000 config :pleroma, :pools, federation: [ size: 50, max_waiting: 10, - timeout: 10_000 + recv_timeout: 10_000 ], media: [ size: 50, max_waiting: 10, - timeout: 10_000 + recv_timeout: 10_000 ], upload: [ size: 25, max_waiting: 5, - timeout: 15_000 + recv_timeout: 15_000 ], default: [ size: 10, max_waiting: 2, - timeout: 5_000 + recv_timeout: 5_000 ] config :pleroma, :hackney_pools, diff --git a/config/description.exs b/config/description.exs index 18c133f02..eac97ad64 100644 --- a/config/description.exs +++ b/config/description.exs @@ -3377,7 +3377,7 @@ suggestions: [250] }, %{ - key: :await_up_timeout, + key: :connect_timeout, type: :integer, description: "Timeout while `gun` will wait until connection is up. Default: 5000ms.", suggestions: [5000] @@ -3415,6 +3415,12 @@ description: "Maximum number of requests waiting for other requests to finish. After this number is reached, the pool will start returning errrors when a new request is made", suggestions: [10] + }, + %{ + key: :recv_timeout, + type: :integer, + description: "Timeout for the pool while gun will wait for response", + suggestions: [10_000] } ] } diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index a9a650fab..7f0725b48 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -498,7 +498,7 @@ Settings for HTTP connection pool. * `:connection_acquisition_wait` - Timeout to acquire a connection from pool.The total max time is this value multiplied by the number of retries. * `connection_acquisition_retries` - Number of attempts to acquire the connection from the pool if it is overloaded. Each attempt is timed `:connection_acquisition_wait` apart. * `:max_connections` - Maximum number of connections in the pool. -* `:await_up_timeout` - Timeout to connect to the host. +* `:connect_timeout` - Timeout to connect to the host. * `:reclaim_multiplier` - Multiplied by `:max_connections` this will be the maximum number of idle connections that will be reclaimed in case the pool is overloaded. ### :pools @@ -517,7 +517,7 @@ There are four pools used: For each pool, the options are: * `:size` - limit to how much requests can be concurrently executed. -* `:timeout` - timeout while `gun` will wait for response +* `:recv_timeout` - timeout while `gun` will wait for response * `:max_waiting` - limit to how much requests can be waiting for others to finish, after this is reached, subsequent requests will be dropped. ## Captcha diff --git a/lib/mix/tasks/pleroma/frontend.ex b/lib/mix/tasks/pleroma/frontend.ex index 1957b1d84..73df67439 100644 --- a/lib/mix/tasks/pleroma/frontend.ex +++ b/lib/mix/tasks/pleroma/frontend.ex @@ -124,9 +124,7 @@ defp download_build(frontend_info, dest) do url = String.replace(frontend_info["build_url"], "${ref}", frontend_info["ref"]) with {:ok, %{status: 200, body: zip_body}} <- - Pleroma.HTTP.get(url, [], - adapter: [pool: :media, timeout: 120_000, recv_timeout: 120_000] - ) do + Pleroma.HTTP.get(url, [], adapter: [pool: :media, recv_timeout: 120_000]) do unzip(zip_body, dest) else e -> {:error, e} diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex index a3f75a4bb..75b1ffc0a 100644 --- a/lib/pleroma/gun/conn.ex +++ b/lib/pleroma/gun/conn.ex @@ -13,7 +13,7 @@ def open(%URI{} = uri, opts) do opts = opts |> Enum.into(%{}) - |> Map.put_new(:await_up_timeout, pool_opts[:await_up_timeout] || 5_000) + |> Map.put_new(:connect_timeout, pool_opts[:connect_timeout] || 5_000) |> Map.put_new(:supervise, false) |> maybe_add_tls_opts(uri) @@ -50,7 +50,7 @@ defp do_open(uri, %{proxy: {proxy_host, proxy_port}} = opts) do with open_opts <- Map.delete(opts, :tls_opts), {:ok, conn} <- Gun.open(proxy_host, proxy_port, open_opts), - {:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]), + {:ok, _} <- Gun.await_up(conn, opts[:connect_timeout]), stream <- Gun.connect(conn, connect_opts), {:response, :fin, 200, _} <- Gun.await(conn, stream) do {:ok, conn} @@ -88,7 +88,7 @@ defp do_open(uri, %{proxy: {proxy_type, proxy_host, proxy_port}} = opts) do |> Map.put(:socks_opts, socks_opts) with {:ok, conn} <- Gun.open(proxy_host, proxy_port, opts), - {:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do + {:ok, _} <- Gun.await_up(conn, opts[:connect_timeout]) do {:ok, conn} else error -> @@ -106,7 +106,7 @@ defp do_open(%URI{host: host, port: port} = uri, opts) do host = Pleroma.HTTP.AdapterHelper.parse_host(host) with {:ok, conn} <- Gun.open(host, port, opts), - {:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do + {:ok, _} <- Gun.await_up(conn, opts[:connect_timeout]) do {:ok, conn} else error -> diff --git a/lib/pleroma/http/adapter_helper.ex b/lib/pleroma/http/adapter_helper.ex index d72297323..08b51578a 100644 --- a/lib/pleroma/http/adapter_helper.ex +++ b/lib/pleroma/http/adapter_helper.ex @@ -6,7 +6,7 @@ defmodule Pleroma.HTTP.AdapterHelper do @moduledoc """ Configure Tesla.Client with default and customized adapter options. """ - @defaults [pool: :federation] + @defaults [pool: :federation, connect_timeout: 5_000, recv_timeout: 5_000] @type proxy_type() :: :socks4 | :socks5 @type host() :: charlist() | :inet.ip_address() diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex index 4a967d8f2..1dbb71362 100644 --- a/lib/pleroma/http/adapter_helper/gun.ex +++ b/lib/pleroma/http/adapter_helper/gun.ex @@ -11,12 +11,8 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do require Logger @defaults [ - connect_timeout: 5_000, - domain_lookup_timeout: 5_000, - tls_handshake_timeout: 5_000, retry: 1, - retry_timeout: 1000, - await_up_timeout: 5_000 + retry_timeout: 1_000 ] @type pool() :: :federation | :upload | :media | :default @@ -45,15 +41,17 @@ defp add_scheme_opts(opts, %{scheme: "https"}) do end defp put_timeout(opts) do + {recv_timeout, opts} = Keyword.pop(opts, :recv_timeout, pool_timeout(opts[:pool])) # this is the timeout to receive a message from Gun - Keyword.put_new(opts, :timeout, pool_timeout(opts[:pool])) + # `:timeout` key is used in Tesla + Keyword.put(opts, :timeout, recv_timeout) end @spec pool_timeout(pool()) :: non_neg_integer() def pool_timeout(pool) do - default = Config.get([:pools, :default, :timeout], 5_000) + default = Config.get([:pools, :default, :recv_timeout], 5_000) - Config.get([:pools, pool, :timeout], default) + Config.get([:pools, pool, :recv_timeout], default) end @prefix Pleroma.Gun.ConnectionPool diff --git a/lib/pleroma/http/adapter_helper/hackney.ex b/lib/pleroma/http/adapter_helper/hackney.ex index 42e3acfec..ef84553c1 100644 --- a/lib/pleroma/http/adapter_helper/hackney.ex +++ b/lib/pleroma/http/adapter_helper/hackney.ex @@ -2,11 +2,8 @@ defmodule Pleroma.HTTP.AdapterHelper.Hackney do @behaviour Pleroma.HTTP.AdapterHelper @defaults [ - connect_timeout: 10_000, - recv_timeout: 20_000, follow_redirect: true, - force_redirect: true, - pool: :federation + force_redirect: true ] @spec options(keyword(), URI.t()) :: keyword() @@ -19,6 +16,7 @@ def options(connection_opts \\ [], %URI{} = uri) do |> Keyword.merge(config_opts) |> Keyword.merge(connection_opts) |> add_scheme_opts(uri) + |> maybe_add_with_body() |> Pleroma.HTTP.AdapterHelper.maybe_add_proxy(proxy) end @@ -27,4 +25,12 @@ defp add_scheme_opts(opts, %URI{scheme: "https"}) do end defp add_scheme_opts(opts, _), do: opts + + defp maybe_add_with_body(opts) do + if opts[:max_body] do + Keyword.put(opts, :with_body, true) + else + opts + end + end end diff --git a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex index dfab105a3..a203405a0 100644 --- a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex @@ -13,22 +13,16 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do require Logger @options [ - pool: :media + pool: :media, + recv_timeout: 10_000 ] def perform(:prefetch, url) do Logger.debug("Prefetching #{inspect(url)}") - opts = - if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do - Keyword.put(@options, :recv_timeout, 10_000) - else - @options - end - url |> MediaProxy.url() - |> HTTP.get([], adapter: opts) + |> HTTP.get([], adapter: @options) end def perform(:preload, %{"object" => %{"attachment" => attachments}} = _message) do diff --git a/lib/pleroma/web/rel_me.ex b/lib/pleroma/web/rel_me.ex index 8e2b51508..32bce3c1b 100644 --- a/lib/pleroma/web/rel_me.ex +++ b/lib/pleroma/web/rel_me.ex @@ -5,7 +5,8 @@ defmodule Pleroma.Web.RelMe do @options [ pool: :media, - max_body: 2_000_000 + max_body: 2_000_000, + recv_timeout: 2_000 ] if Pleroma.Config.get(:env) == :test do @@ -23,18 +24,8 @@ def parse(url) when is_binary(url) do def parse(_), do: {:error, "No URL provided"} defp parse_url(url) do - opts = - if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do - Keyword.merge(@options, - recv_timeout: 2_000, - with_body: true - ) - else - @options - end - with {:ok, %Tesla.Env{body: html, status: status}} when status in 200..299 <- - Pleroma.HTTP.get(url, [], adapter: opts), + Pleroma.HTTP.get(url, [], adapter: @options), {:ok, html_tree} <- Floki.parse_document(html), data <- Floki.attribute(html_tree, "link[rel~=me]", "href") ++ diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index 752ca9f81..084a66466 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -9,14 +9,15 @@ defmodule Pleroma.Web.RichMedia.Helpers do alias Pleroma.Object alias Pleroma.Web.RichMedia.Parser - @rich_media_options [ + @options [ pool: :media, - max_body: 2_000_000 + max_body: 2_000_000, + recv_timeout: 2_000 ] @spec validate_page_url(URI.t() | binary()) :: :ok | :error defp validate_page_url(page_url) when is_binary(page_url) do - validate_tld = Pleroma.Config.get([Pleroma.Formatter, :validate_tld]) + validate_tld = Config.get([Pleroma.Formatter, :validate_tld]) page_url |> Linkify.Parser.url?(validate_tld: validate_tld) @@ -86,16 +87,6 @@ def perform(:fetch, %Activity{} = activity) do def rich_media_get(url) do headers = [{"user-agent", Pleroma.Application.user_agent() <> "; Bot"}] - options = - if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do - Keyword.merge(@rich_media_options, - recv_timeout: 2_000, - with_body: true - ) - else - @rich_media_options - end - - Pleroma.HTTP.get(url, headers, adapter: options) + Pleroma.HTTP.get(url, headers, adapter: @options) end end From 8a3d43044a06488545490a89e29c8349d914f164 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov <alex.strizhakov@gmail.com> Date: Sat, 5 Sep 2020 12:41:01 +0300 Subject: [PATCH 050/128] migrations for renaming gun timeout options --- ...e_await_up_timeout_in_connections_pool.exs | 13 +++++++++++++ ...20200905091427_rename_timeout_in_pools.exs | 19 +++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 priv/repo/migrations/20200905082737_rename_await_up_timeout_in_connections_pool.exs create mode 100644 priv/repo/migrations/20200905091427_rename_timeout_in_pools.exs diff --git a/priv/repo/migrations/20200905082737_rename_await_up_timeout_in_connections_pool.exs b/priv/repo/migrations/20200905082737_rename_await_up_timeout_in_connections_pool.exs new file mode 100644 index 000000000..22c40663c --- /dev/null +++ b/priv/repo/migrations/20200905082737_rename_await_up_timeout_in_connections_pool.exs @@ -0,0 +1,13 @@ +defmodule Pleroma.Repo.Migrations.RenameAwaitUpTimeoutInConnectionsPool do + use Ecto.Migration + + def change do + with %Pleroma.ConfigDB{} = config <- + Pleroma.ConfigDB.get_by_params(%{group: :pleroma, key: :connections_pool}), + {timeout, value} when is_integer(timeout) <- Keyword.pop(config.value, :await_up_timeout) do + config + |> Ecto.Changeset.change(value: Keyword.put(value, :connect_timeout, timeout)) + |> Pleroma.Repo.update() + end + end +end diff --git a/priv/repo/migrations/20200905091427_rename_timeout_in_pools.exs b/priv/repo/migrations/20200905091427_rename_timeout_in_pools.exs new file mode 100644 index 000000000..bb2f50ecc --- /dev/null +++ b/priv/repo/migrations/20200905091427_rename_timeout_in_pools.exs @@ -0,0 +1,19 @@ +defmodule Pleroma.Repo.Migrations.RenameTimeoutInPools do + use Ecto.Migration + + def change do + with %Pleroma.ConfigDB{} = config <- + Pleroma.ConfigDB.get_by_params(%{group: :pleroma, key: :pools}) do + updated_value = + Enum.map(config.value, fn {pool, pool_value} -> + with {timeout, value} when is_integer(timeout) <- Keyword.pop(pool_value, :timeout) do + {pool, Keyword.put(value, :recv_timeout, timeout)} + end + end) + + config + |> Ecto.Changeset.change(value: updated_value) + |> Pleroma.Repo.update() + end + end +end From 696bf09433aa7f33cf580c71cb7f1f3367d4c124 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov <alex.strizhakov@gmail.com> Date: Mon, 7 Sep 2020 16:57:42 +0300 Subject: [PATCH 051/128] passing adapter options directly without adapter key --- config/test.exs | 2 +- lib/mix/tasks/pleroma/benchmark.ex | 11 ++--- lib/mix/tasks/pleroma/frontend.ex | 2 +- lib/pleroma/http/ex_aws.ex | 2 +- lib/pleroma/http/http.ex | 2 +- lib/pleroma/http/tzdata.ex | 4 +- .../mrf/media_proxy_warming_policy.ex | 2 +- lib/pleroma/web/rel_me.ex | 2 +- lib/pleroma/web/rich_media/helpers.ex | 2 +- test/web/instances/instance_test.exs | 2 + .../mastodon_api/views/account_view_test.exs | 40 ++++++++++--------- 11 files changed, 36 insertions(+), 35 deletions(-) diff --git a/config/test.exs b/config/test.exs index f0358e384..e9c2273e8 100644 --- a/config/test.exs +++ b/config/test.exs @@ -114,7 +114,7 @@ config :pleroma, Pleroma.Web.ApiSpec.CastAndValidate, strict: true -config :pleroma, :instances_favicons, enabled: true +config :pleroma, :instances_favicons, enabled: false config :pleroma, Pleroma.Uploaders.S3, bucket: nil, diff --git a/lib/mix/tasks/pleroma/benchmark.ex b/lib/mix/tasks/pleroma/benchmark.ex index dd2b9c8f2..a607d5d4f 100644 --- a/lib/mix/tasks/pleroma/benchmark.ex +++ b/lib/mix/tasks/pleroma/benchmark.ex @@ -91,20 +91,17 @@ def run(["adapters"]) do "Without conn and without pool" => fn -> {:ok, %Tesla.Env{}} = Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], - adapter: [pool: :no_pool, receive_conn: false] + pool: :no_pool, + receive_conn: false ) end, "Without conn and with pool" => fn -> {:ok, %Tesla.Env{}} = - Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], - adapter: [receive_conn: false] - ) + Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], receive_conn: false) end, "With reused conn and without pool" => fn -> {:ok, %Tesla.Env{}} = - Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], - adapter: [pool: :no_pool] - ) + Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], pool: :no_pool) end, "With reused conn and with pool" => fn -> {:ok, %Tesla.Env{}} = Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500") diff --git a/lib/mix/tasks/pleroma/frontend.ex b/lib/mix/tasks/pleroma/frontend.ex index 73df67439..cbce81ab9 100644 --- a/lib/mix/tasks/pleroma/frontend.ex +++ b/lib/mix/tasks/pleroma/frontend.ex @@ -124,7 +124,7 @@ defp download_build(frontend_info, dest) do url = String.replace(frontend_info["build_url"], "${ref}", frontend_info["ref"]) with {:ok, %{status: 200, body: zip_body}} <- - Pleroma.HTTP.get(url, [], adapter: [pool: :media, recv_timeout: 120_000]) do + Pleroma.HTTP.get(url, [], pool: :media, recv_timeout: 120_000) do unzip(zip_body, dest) else e -> {:error, e} diff --git a/lib/pleroma/http/ex_aws.ex b/lib/pleroma/http/ex_aws.ex index c3f335c73..5cac3532f 100644 --- a/lib/pleroma/http/ex_aws.ex +++ b/lib/pleroma/http/ex_aws.ex @@ -11,7 +11,7 @@ defmodule Pleroma.HTTP.ExAws do @impl true def request(method, url, body \\ "", headers \\ [], http_opts \\ []) do - http_opts = Keyword.put_new(http_opts, :adapter, pool: :upload) + http_opts = Keyword.put_new(http_opts, :pool, :upload) case HTTP.request(method, url, body, headers, http_opts) do {:ok, env} -> diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 7bc73f4a0..052597191 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -60,7 +60,7 @@ def post(url, body, headers \\ [], options \\ []), {:ok, Env.t()} | {:error, any()} def request(method, url, body, headers, options) when is_binary(url) do uri = URI.parse(url) - adapter_opts = AdapterHelper.options(uri, options[:adapter] || []) + adapter_opts = AdapterHelper.options(uri, options || []) options = put_in(options[:adapter], adapter_opts) params = options[:params] || [] diff --git a/lib/pleroma/http/tzdata.ex b/lib/pleroma/http/tzdata.ex index 4539ac359..09cfdadf7 100644 --- a/lib/pleroma/http/tzdata.ex +++ b/lib/pleroma/http/tzdata.ex @@ -11,7 +11,7 @@ defmodule Pleroma.HTTP.Tzdata do @impl true def get(url, headers, options) do - options = Keyword.put_new(options, :adapter, pool: :default) + options = Keyword.put_new(options, :pool, :default) with {:ok, %Tesla.Env{} = env} <- HTTP.get(url, headers, options) do {:ok, {env.status, env.headers, env.body}} @@ -20,7 +20,7 @@ def get(url, headers, options) do @impl true def head(url, headers, options) do - options = Keyword.put_new(options, :adapter, pool: :default) + options = Keyword.put_new(options, :pool, :default) with {:ok, %Tesla.Env{} = env} <- HTTP.head(url, headers, options) do {:ok, {env.status, env.headers}} diff --git a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex index a203405a0..98d595469 100644 --- a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex @@ -22,7 +22,7 @@ def perform(:prefetch, url) do url |> MediaProxy.url() - |> HTTP.get([], adapter: @options) + |> HTTP.get([], @options) end def perform(:preload, %{"object" => %{"attachment" => attachments}} = _message) do diff --git a/lib/pleroma/web/rel_me.ex b/lib/pleroma/web/rel_me.ex index 32bce3c1b..28f75b18d 100644 --- a/lib/pleroma/web/rel_me.ex +++ b/lib/pleroma/web/rel_me.ex @@ -25,7 +25,7 @@ def parse(_), do: {:error, "No URL provided"} defp parse_url(url) do with {:ok, %Tesla.Env{body: html, status: status}} when status in 200..299 <- - Pleroma.HTTP.get(url, [], adapter: @options), + Pleroma.HTTP.get(url, [], @options), {:ok, html_tree} <- Floki.parse_document(html), data <- Floki.attribute(html_tree, "link[rel~=me]", "href") ++ diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index 084a66466..bd7f03cbe 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -87,6 +87,6 @@ def perform(:fetch, %Activity{} = activity) do def rich_media_get(url) do headers = [{"user-agent", Pleroma.Application.user_agent() <> "; Bot"}] - Pleroma.HTTP.get(url, headers, adapter: @options) + Pleroma.HTTP.get(url, headers, @options) end end diff --git a/test/web/instances/instance_test.exs b/test/web/instances/instance_test.exs index dc6ace843..5d4efcebe 100644 --- a/test/web/instances/instance_test.exs +++ b/test/web/instances/instance_test.exs @@ -112,6 +112,8 @@ test "Scrapes favicon URLs" do end test "Returns nil on too long favicon URLs" do + clear_config([:instances_favicons, :enabled], true) + long_favicon_url = "https://Lorem.ipsum.dolor.sit.amet/consecteturadipiscingelit/Praesentpharetrapurusutaliquamtempus/Mauriseulaoreetarcu/atfacilisisorci/Nullamporttitor/nequesedfeugiatmollis/dolormagnaefficiturlorem/nonpretiumsapienorcieurisus/Nullamveleratsem/Maecenassedaccumsanexnam/favicon.png" diff --git a/test/web/mastodon_api/views/account_view_test.exs b/test/web/mastodon_api/views/account_view_test.exs index 8f37efa3c..68a5d0091 100644 --- a/test/web/mastodon_api/views/account_view_test.exs +++ b/test/web/mastodon_api/views/account_view_test.exs @@ -5,7 +5,6 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do use Pleroma.DataCase - alias Pleroma.Config alias Pleroma.User alias Pleroma.UserRelationship alias Pleroma.Web.CommonAPI @@ -19,8 +18,6 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do :ok end - setup do: clear_config([:instances_favicons, :enabled]) - test "Represent a user account" do background_image = %{ "url" => [%{"href" => "https://example.com/images/asuka_hospital.png"}] @@ -78,8 +75,7 @@ test "Represent a user account" do pleroma: %{ ap_id: user.ap_id, background_image: "https://example.com/images/asuka_hospital.png", - favicon: - "https://shitposter.club/plugins/Qvitter/img/gnusocial-favicons/favicon-16x16.png", + favicon: nil, confirmation_pending: false, tags: [], is_admin: false, @@ -98,22 +94,29 @@ test "Represent a user account" do assert expected == AccountView.render("show.json", %{user: user, skip_visibility_check: true}) end - test "Favicon is nil when :instances_favicons is disabled" do - user = insert(:user) + describe "favicon" do + setup do + [user: insert(:user)] + end - Config.put([:instances_favicons, :enabled], true) + test "is parsed when :instance_favicons is enabled", %{user: user} do + clear_config([:instances_favicons, :enabled], true) - assert %{ - pleroma: %{ - favicon: - "https://shitposter.club/plugins/Qvitter/img/gnusocial-favicons/favicon-16x16.png" - } - } = AccountView.render("show.json", %{user: user, skip_visibility_check: true}) + assert %{ + pleroma: %{ + favicon: + "https://shitposter.club/plugins/Qvitter/img/gnusocial-favicons/favicon-16x16.png" + } + } = AccountView.render("show.json", %{user: user, skip_visibility_check: true}) + end - Config.put([:instances_favicons, :enabled], false) + test "is nil when :instances_favicons is disabled", %{user: user} do + assert %{pleroma: %{favicon: nil}} = + AccountView.render("show.json", %{user: user, skip_visibility_check: true}) + end + end - assert %{pleroma: %{favicon: nil}} = - AccountView.render("show.json", %{user: user, skip_visibility_check: true}) + test "Favicon when :instance_favicons is enabled" do end test "Represent the user account for the account owner" do @@ -173,8 +176,7 @@ test "Represent a Service(bot) account" do pleroma: %{ ap_id: user.ap_id, background_image: nil, - favicon: - "https://shitposter.club/plugins/Qvitter/img/gnusocial-favicons/favicon-16x16.png", + favicon: nil, confirmation_pending: false, tags: [], is_admin: false, From 18d21aed00dcbdaabd7db25b8b7d0c88141ec98a Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov <alex.strizhakov@gmail.com> Date: Mon, 7 Sep 2020 19:04:16 +0300 Subject: [PATCH 052/128] deprecation warnings --- lib/pleroma/config/deprecation_warnings.ex | 43 ++++++++++++++++ test/config/deprecation_warnings_test.exs | 59 +++++++++++++++++++--- 2 files changed, 96 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/config/deprecation_warnings.ex b/lib/pleroma/config/deprecation_warnings.ex index 0f52eb210..2bfe4ddba 100644 --- a/lib/pleroma/config/deprecation_warnings.ex +++ b/lib/pleroma/config/deprecation_warnings.ex @@ -56,6 +56,7 @@ def warn do check_old_mrf_config() check_media_proxy_whitelist_config() check_welcome_message_config() + check_gun_pool_options() end def check_welcome_message_config do @@ -115,4 +116,46 @@ def check_media_proxy_whitelist_config do """) end end + + def check_gun_pool_options do + pool_config = Config.get(:connections_pool) + + if timeout = pool_config[:await_up_timeout] do + Logger.warn(""" + !!!DEPRECATION WARNING!!! + Your config is using old setting name `await_up_timeout` instead of `connect_timeout`. Setting should work for now, but you are advised to change format to scheme with port to prevent possible issues later. + """) + + Config.put(:connections_pool, Keyword.put_new(pool_config, :connect_timeout, timeout)) + end + + pools_configs = Config.get(:pools) + + warning_preface = """ + !!!DEPRECATION WARNING!!! + Your config is using old setting name `timeout` instead of `recv_timeout` in pool settings. Setting should work for now, but you are advised to change format to scheme with port to prevent possible issues later. + """ + + updated_config = + Enum.reduce(pools_configs, [], fn {pool_name, config}, acc -> + if timeout = config[:timeout] do + Keyword.put(acc, pool_name, Keyword.put_new(config, :recv_timeout, timeout)) + else + acc + end + end) + + if updated_config != [] do + pool_warnings = + updated_config + |> Keyword.keys() + |> Enum.map(fn pool_name -> + "\n* `:timeout` options in #{pool_name} pool is now `:recv_timeout`" + end) + + Logger.warn(Enum.join([warning_preface | pool_warnings])) + + Config.put(:pools, updated_config) + end + end end diff --git a/test/config/deprecation_warnings_test.exs b/test/config/deprecation_warnings_test.exs index 555661a71..e22052404 100644 --- a/test/config/deprecation_warnings_test.exs +++ b/test/config/deprecation_warnings_test.exs @@ -4,12 +4,15 @@ defmodule Pleroma.Config.DeprecationWarningsTest do import ExUnit.CaptureLog + alias Pleroma.Config + alias Pleroma.Config.DeprecationWarnings + test "check_old_mrf_config/0" do clear_config([:instance, :rewrite_policy], Pleroma.Web.ActivityPub.MRF.NoOpPolicy) clear_config([:instance, :mrf_transparency], true) clear_config([:instance, :mrf_transparency_exclusions], []) - assert capture_log(fn -> Pleroma.Config.DeprecationWarnings.check_old_mrf_config() end) =~ + assert capture_log(fn -> DeprecationWarnings.check_old_mrf_config() end) =~ """ !!!DEPRECATION WARNING!!! Your config is using old namespaces for MRF configuration. They should work for now, but you are advised to change to new namespaces to prevent possible issues later: @@ -44,22 +47,66 @@ test "move_namespace_and_warn/2" do ] assert capture_log(fn -> - Pleroma.Config.DeprecationWarnings.move_namespace_and_warn( + DeprecationWarnings.move_namespace_and_warn( config_map, "Warning preface" ) end) =~ "Warning preface\n error :key\n error :key2\n error :key3" - assert Pleroma.Config.get(new_group1) == 1 - assert Pleroma.Config.get(new_group2) == 2 - assert Pleroma.Config.get(new_group3) == 3 + assert Config.get(new_group1) == 1 + assert Config.get(new_group2) == 2 + assert Config.get(new_group3) == 3 end test "check_media_proxy_whitelist_config/0" do clear_config([:media_proxy, :whitelist], ["https://example.com", "example2.com"]) assert capture_log(fn -> - Pleroma.Config.DeprecationWarnings.check_media_proxy_whitelist_config() + DeprecationWarnings.check_media_proxy_whitelist_config() end) =~ "Your config is using old format (only domain) for MediaProxy whitelist option" end + + describe "check_gun_pool_options/0" do + test "await_up_timeout" do + config = Config.get(:connections_pool) + clear_config(:connections_pool, Keyword.put(config, :await_up_timeout, 5_000)) + + assert capture_log(fn -> + DeprecationWarnings.check_gun_pool_options() + end) =~ + "Your config is using old setting name `await_up_timeout` instead of `connect_timeout`" + end + + test "pool timeout" do + old_config = [ + federation: [ + size: 50, + max_waiting: 10, + timeout: 10_000 + ], + media: [ + size: 50, + max_waiting: 10, + timeout: 10_000 + ], + upload: [ + size: 25, + max_waiting: 5, + timeout: 15_000 + ], + default: [ + size: 10, + max_waiting: 2, + timeout: 5_000 + ] + ] + + clear_config(:pools, old_config) + + assert capture_log(fn -> + DeprecationWarnings.check_gun_pool_options() + end) =~ + "Your config is using old setting name `timeout` instead of `recv_timeout` in pool settings" + end + end end From 7ad1732ed2444d4d7bb1a89e66d46e5d52536e0f Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov <alex.strizhakov@gmail.com> Date: Mon, 7 Sep 2020 19:55:14 +0300 Subject: [PATCH 053/128] changelog entry --- CHANGELOG.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 46f1e859b..f303e22a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,11 +5,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Unreleased +### Changed + +- Settings renaming: `:await_up_timeout` in `:connections_pool` namespace to `connect_timeout`, `timeout` in `pools` namespace to `recv_timeout`. + ### Removed - **Breaking:** Removed `Pleroma.Workers.Cron.StatsWorker` setting from Oban `:crontab`. - ## unreleased-patch - ??? ### Added @@ -25,7 +28,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Cards being wrong for preview statuses due to cache key collision - Password resets no longer processed for deactivated accounts - ## [2.1.0] - 2020-08-28 ### Changed From 699224a900d54b6d32e0bd3f2abd9eccc523df11 Mon Sep 17 00:00:00 2001 From: Alibek Omarov <a1ba.omarov@gmail.com> Date: Mon, 7 Sep 2020 22:14:40 +0200 Subject: [PATCH 054/128] ForceBotUnlistedPolicy: initial add, tiny clean up from my previous version --- .../mrf/force_bot_unlisted_policy.ex | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex diff --git a/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex b/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex new file mode 100644 index 000000000..31fd90586 --- /dev/null +++ b/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex @@ -0,0 +1,61 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.ForceBotUnlistedPolicy do + alias Pleroma.User + @behaviour Pleroma.Web.ActivityPub.MRF + @moduledoc "Remove bot posts from federated timeline" + + require Pleroma.Constants + + defp check_by_actor_type(user) do + if user.actor_type in ["Application", "Service"], do: 1.0, else: 0.0 + end + + defp check_by_nickname(user) do + if Regex.match?(~r/bot@|ebooks@/i, user.nickname), do: 1.0, else: 0.0 + end + + defp botness_score(user), do: check_by_actor_type(user) + check_by_nickname(user) + + @impl true + def filter( + %{ + "type" => "Create", + "to" => to, + "cc" => cc, + "actor" => actor, + "object" => object + } = message + ) do + user = User.get_cached_by_ap_id(actor) + isbot = 0.8 < botness_score(user) + + if isbot and Enum.member?(to, Pleroma.Constants.as_public()) do + to = List.delete(to, Pleroma.Constants.as_public()) ++ [user.follower_address] + cc = List.delete(cc, user.follower_address) ++ [Pleroma.Constants.as_public()] + + object = + object + |> Map.put("to", to) + |> Map.put("cc", cc) + + message = + message + |> Map.put("to", to) + |> Map.put("cc", cc) + |> Map.put("object", object) + + {:ok, message} + else + {:ok, message} + end + end + + @impl true + def filter(message), do: {:ok, message} + + @impl true + def describe, do: {:ok, %{}} +end From 57cf0cc3b3029cb0ff017c53e2602ad945b8d9b3 Mon Sep 17 00:00:00 2001 From: Alibek Omarov <a1ba.omarov@gmail.com> Date: Mon, 7 Sep 2020 22:50:37 +0200 Subject: [PATCH 055/128] ForceBotUnlistedPolicy: add test --- .../mrf/force_bot_unlisted_policy_test.ex | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 test/web/activity_pub/mrf/force_bot_unlisted_policy_test.ex diff --git a/test/web/activity_pub/mrf/force_bot_unlisted_policy_test.ex b/test/web/activity_pub/mrf/force_bot_unlisted_policy_test.ex new file mode 100644 index 000000000..84e2a9024 --- /dev/null +++ b/test/web/activity_pub/mrf/force_bot_unlisted_policy_test.ex @@ -0,0 +1,58 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.ForceBotUnlistedPolicyTest do + use Pleroma.DataCase + import Pleroma.Factory + + import Pleroma.Web.ActivityPub.MRF.ForceBotUnlistedPolicy + + defp generate_messages(actor) do + {%{ + "actor" => actor.ap_id, + "type" => "Create", + "object" => %{}, + "to" => [@public, "f"], + "cc" => [actor.follower_address, "d"] + }, %{ + "actor" => actor.ap_id, + "type" => "Create", + "object" => %{"to" => ["f", actor.follower_address], "cc" => ["d", @public]}, + "to" => ["f", actor.follower_address], + "cc" => ["d", @public] + }} + end + + test "removes from the federated timeline by nickname heuristics 1" do + actor = insert(:user, %{nickname: "annoying_ebooks@example.com"}) + + {message, except_message} = generate_messages(actor) + + assert ForceBotUnlistedPolicy.filter(message) == {:ok, except_message} + end + + test "removes from the federated timeline by nickname heuristics 2" do + actor = insert(:user, %{nickname: "cirnonewsnetworkbot@meow.cat"}) + + {message, except_message} = generate_messages(actor) + + assert ForceBotUnlistedPolicy.filter(message) == {:ok, except_message} + end + + test "removes from the federated timeline by actor type Application" do + actor = insert(:user, %{actor_type: "Application"}) + + {message, except_message} = generate_messages(actor) + + assert ForceBotUnlistedPolicy.filter(message) == {:ok, except_message} + end + + test "removes from the federated timeline by actor type Service" do + actor = insert(:user, %{actor_type: "Service"}) + + {message, except_message} = generate_messages(actor) + + assert ForceBotUnlistedPolicy.filter(message) == {:ok, except_message} + end +end From 8b695c3eeb6ee7a91fc5a8a4293fb3cb53212818 Mon Sep 17 00:00:00 2001 From: Alibek Omarov <a1ba.omarov@gmail.com> Date: Mon, 7 Sep 2020 22:53:45 +0200 Subject: [PATCH 056/128] ForceBotUnlistedPolicy: format --- .../mrf/force_bot_unlisted_policy.ex | 16 ++++++------ .../mrf/force_bot_unlisted_policy_test.ex | 25 ++++++++++--------- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex b/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex index 31fd90586..7290f444b 100644 --- a/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex @@ -21,14 +21,14 @@ defp botness_score(user), do: check_by_actor_type(user) + check_by_nickname(user @impl true def filter( - %{ - "type" => "Create", - "to" => to, - "cc" => cc, - "actor" => actor, - "object" => object - } = message - ) do + %{ + "type" => "Create", + "to" => to, + "cc" => cc, + "actor" => actor, + "object" => object + } = message + ) do user = User.get_cached_by_ap_id(actor) isbot = 0.8 < botness_score(user) diff --git a/test/web/activity_pub/mrf/force_bot_unlisted_policy_test.ex b/test/web/activity_pub/mrf/force_bot_unlisted_policy_test.ex index 84e2a9024..6f001c233 100644 --- a/test/web/activity_pub/mrf/force_bot_unlisted_policy_test.ex +++ b/test/web/activity_pub/mrf/force_bot_unlisted_policy_test.ex @@ -10,18 +10,19 @@ defmodule Pleroma.Web.ActivityPub.MRF.ForceBotUnlistedPolicyTest do defp generate_messages(actor) do {%{ - "actor" => actor.ap_id, - "type" => "Create", - "object" => %{}, - "to" => [@public, "f"], - "cc" => [actor.follower_address, "d"] - }, %{ - "actor" => actor.ap_id, - "type" => "Create", - "object" => %{"to" => ["f", actor.follower_address], "cc" => ["d", @public]}, - "to" => ["f", actor.follower_address], - "cc" => ["d", @public] - }} + "actor" => actor.ap_id, + "type" => "Create", + "object" => %{}, + "to" => [@public, "f"], + "cc" => [actor.follower_address, "d"] + }, + %{ + "actor" => actor.ap_id, + "type" => "Create", + "object" => %{"to" => ["f", actor.follower_address], "cc" => ["d", @public]}, + "to" => ["f", actor.follower_address], + "cc" => ["d", @public] + }} end test "removes from the federated timeline by nickname heuristics 1" do From d2fd1d348122d8b0c3f4b0262cfc980afed83448 Mon Sep 17 00:00:00 2001 From: Alibek Omarov <a1ba.omarov@gmail.com> Date: Mon, 7 Sep 2020 23:04:07 +0200 Subject: [PATCH 057/128] ForceBotUnlistedPolicy: fix test extension --- ...unlisted_policy_test.ex => force_bot_unlisted_policy_test.exs} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/web/activity_pub/mrf/{force_bot_unlisted_policy_test.ex => force_bot_unlisted_policy_test.exs} (100%) diff --git a/test/web/activity_pub/mrf/force_bot_unlisted_policy_test.ex b/test/web/activity_pub/mrf/force_bot_unlisted_policy_test.exs similarity index 100% rename from test/web/activity_pub/mrf/force_bot_unlisted_policy_test.ex rename to test/web/activity_pub/mrf/force_bot_unlisted_policy_test.exs From 0a25c92cfafce6af9ff9a40ac278dafac69e0c08 Mon Sep 17 00:00:00 2001 From: Alibek Omarov <a1ba.omarov@gmail.com> Date: Mon, 7 Sep 2020 23:18:36 +0200 Subject: [PATCH 058/128] ForceBotUnlistedPolicy: try to fix test --- test/web/activity_pub/mrf/force_bot_unlisted_policy_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/web/activity_pub/mrf/force_bot_unlisted_policy_test.exs b/test/web/activity_pub/mrf/force_bot_unlisted_policy_test.exs index 6f001c233..85ca95b0a 100644 --- a/test/web/activity_pub/mrf/force_bot_unlisted_policy_test.exs +++ b/test/web/activity_pub/mrf/force_bot_unlisted_policy_test.exs @@ -6,7 +6,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.ForceBotUnlistedPolicyTest do use Pleroma.DataCase import Pleroma.Factory - import Pleroma.Web.ActivityPub.MRF.ForceBotUnlistedPolicy + alias Pleroma.Web.ActivityPub.MRF.ForceBotUnlistedPolicy defp generate_messages(actor) do {%{ From d074e54013cb38d308693891e0353c0dccc54b85 Mon Sep 17 00:00:00 2001 From: Alibek Omarov <a1ba.omarov@gmail.com> Date: Mon, 7 Sep 2020 23:28:29 +0200 Subject: [PATCH 059/128] ForceBotUnlistedPolicy: try to fix test 2 --- test/web/activity_pub/mrf/force_bot_unlisted_policy_test.exs | 1 + 1 file changed, 1 insertion(+) diff --git a/test/web/activity_pub/mrf/force_bot_unlisted_policy_test.exs b/test/web/activity_pub/mrf/force_bot_unlisted_policy_test.exs index 85ca95b0a..86dd9ddae 100644 --- a/test/web/activity_pub/mrf/force_bot_unlisted_policy_test.exs +++ b/test/web/activity_pub/mrf/force_bot_unlisted_policy_test.exs @@ -7,6 +7,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.ForceBotUnlistedPolicyTest do import Pleroma.Factory alias Pleroma.Web.ActivityPub.MRF.ForceBotUnlistedPolicy + @public "https://www.w3.org/ns/activitystreams#Public" defp generate_messages(actor) do {%{ From 95688c90ad9cd6438a764b4ea6e0f2e3b594b5c8 Mon Sep 17 00:00:00 2001 From: Alibek Omarov <a1ba.omarov@gmail.com> Date: Tue, 8 Sep 2020 01:13:49 +0200 Subject: [PATCH 060/128] ForceBotUnlistedPolicy: simplify code --- .../activity_pub/mrf/force_bot_unlisted_policy.ex | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex b/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex index 7290f444b..ea9c3d3f5 100644 --- a/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex @@ -9,15 +9,10 @@ defmodule Pleroma.Web.ActivityPub.MRF.ForceBotUnlistedPolicy do require Pleroma.Constants - defp check_by_actor_type(user) do - if user.actor_type in ["Application", "Service"], do: 1.0, else: 0.0 - end + defp check_by_actor_type(user), do: user.actor_type in ["Application", "Service"] + defp check_by_nickname(user), do: Regex.match?(~r/bot@|ebooks@/i, user.nickname) - defp check_by_nickname(user) do - if Regex.match?(~r/bot@|ebooks@/i, user.nickname), do: 1.0, else: 0.0 - end - - defp botness_score(user), do: check_by_actor_type(user) + check_by_nickname(user) + defp check_if_bot(user), do: check_by_actor_type(user) or check_by_nickname(user) @impl true def filter( @@ -30,7 +25,7 @@ def filter( } = message ) do user = User.get_cached_by_ap_id(actor) - isbot = 0.8 < botness_score(user) + isbot = check_if_bot(user) if isbot and Enum.member?(to, Pleroma.Constants.as_public()) do to = List.delete(to, Pleroma.Constants.as_public()) ++ [user.follower_address] From d8a48f838697ce6f1d37f5504c1e51b316aed037 Mon Sep 17 00:00:00 2001 From: rinpatch <rinpatch@sdf.org> Date: Tue, 8 Sep 2020 12:23:08 +0300 Subject: [PATCH 061/128] CHANGELOG.md: Split settings renaming to 2 separate entries --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f303e22a7..e6180a6da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Changed -- Settings renaming: `:await_up_timeout` in `:connections_pool` namespace to `connect_timeout`, `timeout` in `pools` namespace to `recv_timeout`. +- Renamed `:await_up_timeout` in `:connections_pool` namespace to `:connect_timeout`, old name is deprecated. +- Renamed `:timeout` in `pools` namespace to `:recv_timeout`, old name is deprecated. ### Removed From fa347b9c2f416cd2c402e3e6eebb561dfc0ee8a8 Mon Sep 17 00:00:00 2001 From: Mark Felder <feld@FreeBSD.org> Date: Wed, 26 Aug 2020 13:32:03 -0500 Subject: [PATCH 062/128] Fix uploading webp image files when Exiftool Upload Filter is enabled --- CHANGELOG.md | 1 + lib/pleroma/upload/filter/exiftool.ex | 17 +++++++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 46f1e859b..cfe0651c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -138,6 +138,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Fix whole_word always returning false on filter get requests - Migrations not working on OTP releases if the database was connected over ssl - Fix relay following +- Fixed uploading webp images when the Exiftool Upload Filter is enabled ## [2.0.7] - 2020-06-13 diff --git a/lib/pleroma/upload/filter/exiftool.ex b/lib/pleroma/upload/filter/exiftool.ex index ea8798fe3..a91bd5e24 100644 --- a/lib/pleroma/upload/filter/exiftool.ex +++ b/lib/pleroma/upload/filter/exiftool.ex @@ -10,9 +10,20 @@ defmodule Pleroma.Upload.Filter.Exiftool do @behaviour Pleroma.Upload.Filter @spec filter(Pleroma.Upload.t()) :: :ok | {:error, String.t()} - def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do + def filter(%Pleroma.Upload{name: file, tempfile: path, content_type: "image" <> _}) do + # webp is not compatible with exiftool at this time + if Regex.match?(~r/\.(webp)$/i, file) do + :ok + else + strip_exif(path) + end + end + + def filter(_), do: :ok + + defp strip_exif(path) do try do - case System.cmd("exiftool", ["-overwrite_original", "-gps:all=", file], parallelism: true) do + case System.cmd("exiftool", ["-overwrite_original", "-gps:all=", path], parallelism: true) do {_response, 0} -> :ok {error, 1} -> {:error, error} end @@ -21,6 +32,4 @@ def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do {:error, "exiftool command not found"} end end - - def filter(_), do: :ok end From 2165a249744a1ad4386a9d237871abe88e298942 Mon Sep 17 00:00:00 2001 From: Mark Felder <feld@FreeBSD.org> Date: Fri, 4 Sep 2020 17:40:59 -0500 Subject: [PATCH 063/128] Improve upload filter return values so we can identify when filters make no changes to the input --- lib/pleroma/upload/filter.ex | 13 ++++++++++--- lib/pleroma/upload/filter/anonymize_filename.ex | 4 +++- lib/pleroma/upload/filter/dedupe.ex | 4 ++-- lib/pleroma/upload/filter/exiftool.ex | 8 ++++---- lib/pleroma/upload/filter/mogrifun.ex | 6 +++--- lib/pleroma/upload/filter/mogrify.ex | 6 +++--- test/upload/filter/anonymize_filename_test.exs | 6 +++--- test/upload/filter/dedupe_test.exs | 1 + test/upload/filter/exiftool_test.exs | 2 +- test/upload/filter/mogrifun_test.exs | 2 +- test/upload/filter/mogrify_test.exs | 2 +- 11 files changed, 32 insertions(+), 22 deletions(-) diff --git a/lib/pleroma/upload/filter.ex b/lib/pleroma/upload/filter.ex index dbdadc97f..661135634 100644 --- a/lib/pleroma/upload/filter.ex +++ b/lib/pleroma/upload/filter.ex @@ -15,7 +15,11 @@ defmodule Pleroma.Upload.Filter do require Logger - @callback filter(Pleroma.Upload.t()) :: :ok | {:ok, Pleroma.Upload.t()} | {:error, any()} + @callback filter(Pleroma.Upload.t()) :: + {:ok, :filtered} + | {:ok, :noop} + | {:ok, :filtered, Pleroma.Upload.t()} + | {:error, any()} @spec filter([module()], Pleroma.Upload.t()) :: {:ok, Pleroma.Upload.t()} | {:error, any()} @@ -25,10 +29,13 @@ def filter([], upload) do def filter([filter | rest], upload) do case filter.filter(upload) do - :ok -> + {:ok, :filtered} -> filter(rest, upload) - {:ok, upload} -> + {:ok, :filtered, upload} -> + filter(rest, upload) + + {:ok, :noop} -> filter(rest, upload) error -> diff --git a/lib/pleroma/upload/filter/anonymize_filename.ex b/lib/pleroma/upload/filter/anonymize_filename.ex index 07ead8203..fc456e4f4 100644 --- a/lib/pleroma/upload/filter/anonymize_filename.ex +++ b/lib/pleroma/upload/filter/anonymize_filename.ex @@ -16,9 +16,11 @@ defmodule Pleroma.Upload.Filter.AnonymizeFilename do def filter(%Upload{name: name} = upload) do extension = List.last(String.split(name, ".")) name = predefined_name(extension) || random(extension) - {:ok, %Upload{upload | name: name}} + {:ok, :filtered, %Upload{upload | name: name}} end + def filter(_), do: {:ok, :noop} + @spec predefined_name(String.t()) :: String.t() | nil defp predefined_name(extension) do with name when not is_nil(name) <- Config.get([__MODULE__, :text]), diff --git a/lib/pleroma/upload/filter/dedupe.ex b/lib/pleroma/upload/filter/dedupe.ex index 41218a918..86cbc8996 100644 --- a/lib/pleroma/upload/filter/dedupe.ex +++ b/lib/pleroma/upload/filter/dedupe.ex @@ -17,8 +17,8 @@ def filter(%Upload{name: name, tempfile: tempfile} = upload) do |> Base.encode16(case: :lower) filename = shasum <> "." <> extension - {:ok, %Upload{upload | id: shasum, path: filename}} + {:ok, :filtered, %Upload{upload | id: shasum, path: filename}} end - def filter(_), do: :ok + def filter(_), do: {:ok, :noop} end diff --git a/lib/pleroma/upload/filter/exiftool.ex b/lib/pleroma/upload/filter/exiftool.ex index a91bd5e24..94d12c01b 100644 --- a/lib/pleroma/upload/filter/exiftool.ex +++ b/lib/pleroma/upload/filter/exiftool.ex @@ -9,22 +9,22 @@ defmodule Pleroma.Upload.Filter.Exiftool do """ @behaviour Pleroma.Upload.Filter - @spec filter(Pleroma.Upload.t()) :: :ok | {:error, String.t()} + @spec filter(Pleroma.Upload.t()) :: {:ok, any()} | {:error, String.t()} def filter(%Pleroma.Upload{name: file, tempfile: path, content_type: "image" <> _}) do # webp is not compatible with exiftool at this time if Regex.match?(~r/\.(webp)$/i, file) do - :ok + {:ok, :noop} else strip_exif(path) end end - def filter(_), do: :ok + def filter(_), do: {:ok, :noop} defp strip_exif(path) do try do case System.cmd("exiftool", ["-overwrite_original", "-gps:all=", path], parallelism: true) do - {_response, 0} -> :ok + {_response, 0} -> {:ok, :filtered} {error, 1} -> {:error, error} end rescue diff --git a/lib/pleroma/upload/filter/mogrifun.ex b/lib/pleroma/upload/filter/mogrifun.ex index c8fa7b190..363e5cf0f 100644 --- a/lib/pleroma/upload/filter/mogrifun.ex +++ b/lib/pleroma/upload/filter/mogrifun.ex @@ -38,16 +38,16 @@ defmodule Pleroma.Upload.Filter.Mogrifun do [{"fill", "yellow"}, {"tint", "40"}] ] - @spec filter(Pleroma.Upload.t()) :: :ok | {:error, String.t()} + @spec filter(Pleroma.Upload.t()) :: {:ok, atom()} | {:error, String.t()} def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do try do Filter.Mogrify.do_filter(file, [Enum.random(@filters)]) - :ok + {:ok, :filtered} rescue _e in ErlangError -> {:error, "mogrify command not found"} end end - def filter(_), do: :ok + def filter(_), do: {:ok, :noop} end diff --git a/lib/pleroma/upload/filter/mogrify.ex b/lib/pleroma/upload/filter/mogrify.ex index 7a45add5a..71968fd9c 100644 --- a/lib/pleroma/upload/filter/mogrify.ex +++ b/lib/pleroma/upload/filter/mogrify.ex @@ -8,18 +8,18 @@ defmodule Pleroma.Upload.Filter.Mogrify do @type conversion :: action :: String.t() | {action :: String.t(), opts :: String.t()} @type conversions :: conversion() | [conversion()] - @spec filter(Pleroma.Upload.t()) :: :ok | {:error, String.t()} + @spec filter(Pleroma.Upload.t()) :: {:ok, :atom} | {:error, String.t()} def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do try do do_filter(file, Pleroma.Config.get!([__MODULE__, :args])) - :ok + {:ok, :filtered} rescue _e in ErlangError -> {:error, "mogrify command not found"} end end - def filter(_), do: :ok + def filter(_), do: {:ok, :noop} def do_filter(file, filters) do file diff --git a/test/upload/filter/anonymize_filename_test.exs b/test/upload/filter/anonymize_filename_test.exs index adff70f57..19b915cc8 100644 --- a/test/upload/filter/anonymize_filename_test.exs +++ b/test/upload/filter/anonymize_filename_test.exs @@ -24,18 +24,18 @@ defmodule Pleroma.Upload.Filter.AnonymizeFilenameTest do test "it replaces filename on pre-defined text", %{upload_file: upload_file} do Config.put([Upload.Filter.AnonymizeFilename, :text], "custom-file.png") - {:ok, %Upload{name: name}} = Upload.Filter.AnonymizeFilename.filter(upload_file) + {:ok, :filtered, %Upload{name: name}} = Upload.Filter.AnonymizeFilename.filter(upload_file) assert name == "custom-file.png" end test "it replaces filename on pre-defined text expression", %{upload_file: upload_file} do Config.put([Upload.Filter.AnonymizeFilename, :text], "custom-file.{extension}") - {:ok, %Upload{name: name}} = Upload.Filter.AnonymizeFilename.filter(upload_file) + {:ok, :filtered, %Upload{name: name}} = Upload.Filter.AnonymizeFilename.filter(upload_file) assert name == "custom-file.jpg" end test "it replaces filename on random text", %{upload_file: upload_file} do - {:ok, %Upload{name: name}} = Upload.Filter.AnonymizeFilename.filter(upload_file) + {:ok, :filtered, %Upload{name: name}} = Upload.Filter.AnonymizeFilename.filter(upload_file) assert <<_::bytes-size(14)>> <> ".jpg" = name refute name == "an… image.jpg" end diff --git a/test/upload/filter/dedupe_test.exs b/test/upload/filter/dedupe_test.exs index 966c353f7..75c7198e1 100644 --- a/test/upload/filter/dedupe_test.exs +++ b/test/upload/filter/dedupe_test.exs @@ -25,6 +25,7 @@ test "adds shasum" do assert { :ok, + :filtered, %Pleroma.Upload{id: @shasum, path: @shasum <> ".jpg"} } = Dedupe.filter(upload) end diff --git a/test/upload/filter/exiftool_test.exs b/test/upload/filter/exiftool_test.exs index 8ed7d650b..094253a25 100644 --- a/test/upload/filter/exiftool_test.exs +++ b/test/upload/filter/exiftool_test.exs @@ -21,7 +21,7 @@ test "apply exiftool filter" do tempfile: Path.absname("test/fixtures/DSCN0010_tmp.jpg") } - assert Filter.Exiftool.filter(upload) == :ok + assert Filter.Exiftool.filter(upload) == {:ok, :filtered} {exif_original, 0} = System.cmd("exiftool", ["test/fixtures/DSCN0010.jpg"]) {exif_filtered, 0} = System.cmd("exiftool", ["test/fixtures/DSCN0010_tmp.jpg"]) diff --git a/test/upload/filter/mogrifun_test.exs b/test/upload/filter/mogrifun_test.exs index 2426a8496..dc1e9e78f 100644 --- a/test/upload/filter/mogrifun_test.exs +++ b/test/upload/filter/mogrifun_test.exs @@ -36,7 +36,7 @@ test "apply mogrify filter" do save: fn _f, _o -> :ok end ]} ]) do - assert Filter.Mogrifun.filter(upload) == :ok + assert Filter.Mogrifun.filter(upload) == {:ok, :filtered} end Task.await(task) diff --git a/test/upload/filter/mogrify_test.exs b/test/upload/filter/mogrify_test.exs index 62ca30487..bf64b96b3 100644 --- a/test/upload/filter/mogrify_test.exs +++ b/test/upload/filter/mogrify_test.exs @@ -33,7 +33,7 @@ test "apply mogrify filter" do custom: fn _m, _a -> :ok end, custom: fn m, a, o -> send(task.pid, {:apply_filter, {m, a, o}}) end, save: fn _f, _o -> :ok end do - assert Filter.Mogrify.filter(upload) == :ok + assert Filter.Mogrify.filter(upload) == {:ok, :filtered} end Task.await(task) From 3a98960c2684229435c5d04e4f3e0611098ceea2 Mon Sep 17 00:00:00 2001 From: Mark Felder <feld@FreeBSD.org> Date: Fri, 4 Sep 2020 17:50:16 -0500 Subject: [PATCH 064/128] Verify webp files are not processed with exiftool --- test/upload/filter/exiftool_test.exs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/upload/filter/exiftool_test.exs b/test/upload/filter/exiftool_test.exs index 094253a25..fe24036d9 100644 --- a/test/upload/filter/exiftool_test.exs +++ b/test/upload/filter/exiftool_test.exs @@ -30,4 +30,15 @@ test "apply exiftool filter" do assert String.match?(exif_original, ~r/GPS/) refute String.match?(exif_filtered, ~r/GPS/) end + + test "verify webp files are skipped" do + upload = %Pleroma.Upload{ + name: "sample.webp", + content_type: "image/webp", + path: Path.absname("/dev/null"), + tempfile: Path.absname("/dev/null") + } + + assert Filter.Exiftool.filter(upload) == {:ok, :noop} + end end From 216c84a8f4d82649110ffaa2bc9d02b879805c5f Mon Sep 17 00:00:00 2001 From: Mark Felder <feld@FreeBSD.org> Date: Fri, 4 Sep 2020 17:56:05 -0500 Subject: [PATCH 065/128] Bypass the filter based on content-type as well in case a webp image is uploaded with the wrong file extension. --- lib/pleroma/upload/filter/exiftool.ex | 5 ++++- test/upload/filter/exiftool_test.exs | 10 +++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/upload/filter/exiftool.ex b/lib/pleroma/upload/filter/exiftool.ex index 94d12c01b..b07a671ac 100644 --- a/lib/pleroma/upload/filter/exiftool.ex +++ b/lib/pleroma/upload/filter/exiftool.ex @@ -10,8 +10,11 @@ defmodule Pleroma.Upload.Filter.Exiftool do @behaviour Pleroma.Upload.Filter @spec filter(Pleroma.Upload.t()) :: {:ok, any()} | {:error, String.t()} + + # webp is not compatible with exiftool at this time + def filter(%Pleroma.Upload{content_type: "image/webp"}), do: {:ok, :noop} + def filter(%Pleroma.Upload{name: file, tempfile: path, content_type: "image" <> _}) do - # webp is not compatible with exiftool at this time if Regex.match?(~r/\.(webp)$/i, file) do {:ok, :noop} else diff --git a/test/upload/filter/exiftool_test.exs b/test/upload/filter/exiftool_test.exs index fe24036d9..84a3f8b30 100644 --- a/test/upload/filter/exiftool_test.exs +++ b/test/upload/filter/exiftool_test.exs @@ -34,11 +34,15 @@ test "apply exiftool filter" do test "verify webp files are skipped" do upload = %Pleroma.Upload{ name: "sample.webp", - content_type: "image/webp", - path: Path.absname("/dev/null"), - tempfile: Path.absname("/dev/null") + content_type: "image/webp" + } + + bad_type = %Pleroma.Upload{ + name: "sample.webp", + content_type: "image/jpeg" } assert Filter.Exiftool.filter(upload) == {:ok, :noop} + assert Filter.Exiftool.filter(bad_type) == {:ok, :noop} end end From 4ea07f74e9416da8f97a12cfdc24da82e1c00d91 Mon Sep 17 00:00:00 2001 From: Mark Felder <feld@FreeBSD.org> Date: Fri, 4 Sep 2020 22:18:01 -0500 Subject: [PATCH 066/128] Revert/simplify. We only need to check the content-type. There's no chance a webp file will get mismatched as another image type. --- lib/pleroma/upload/filter/exiftool.ex | 16 ++++------------ test/upload/filter/exiftool_test.exs | 6 ------ 2 files changed, 4 insertions(+), 18 deletions(-) diff --git a/lib/pleroma/upload/filter/exiftool.ex b/lib/pleroma/upload/filter/exiftool.ex index b07a671ac..1fd0cfdaa 100644 --- a/lib/pleroma/upload/filter/exiftool.ex +++ b/lib/pleroma/upload/filter/exiftool.ex @@ -14,19 +14,9 @@ defmodule Pleroma.Upload.Filter.Exiftool do # webp is not compatible with exiftool at this time def filter(%Pleroma.Upload{content_type: "image/webp"}), do: {:ok, :noop} - def filter(%Pleroma.Upload{name: file, tempfile: path, content_type: "image" <> _}) do - if Regex.match?(~r/\.(webp)$/i, file) do - {:ok, :noop} - else - strip_exif(path) - end - end - - def filter(_), do: {:ok, :noop} - - defp strip_exif(path) do + def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do try do - case System.cmd("exiftool", ["-overwrite_original", "-gps:all=", path], parallelism: true) do + case System.cmd("exiftool", ["-overwrite_original", "-gps:all=", file], parallelism: true) do {_response, 0} -> {:ok, :filtered} {error, 1} -> {:error, error} end @@ -35,4 +25,6 @@ defp strip_exif(path) do {:error, "exiftool command not found"} end end + + def filter(_), do: {:ok, :noop} end diff --git a/test/upload/filter/exiftool_test.exs b/test/upload/filter/exiftool_test.exs index 84a3f8b30..d4cd4ba11 100644 --- a/test/upload/filter/exiftool_test.exs +++ b/test/upload/filter/exiftool_test.exs @@ -37,12 +37,6 @@ test "verify webp files are skipped" do content_type: "image/webp" } - bad_type = %Pleroma.Upload{ - name: "sample.webp", - content_type: "image/jpeg" - } - assert Filter.Exiftool.filter(upload) == {:ok, :noop} - assert Filter.Exiftool.filter(bad_type) == {:ok, :noop} end end From 0f27211dd0b9177abf0eb8b10665c6351f07756a Mon Sep 17 00:00:00 2001 From: rinpatch <rinpatch@sdf.org> Date: Tue, 8 Sep 2020 12:28:38 +0300 Subject: [PATCH 067/128] CHANGELOG.md: move the exiftool webp entry to a proper section Also clarify how it was fixed --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cfe0651c8..527df8438 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Fixed - Possible OOM errors with the default HTTP adapter +- Fixed uploading webp images when the Exiftool Upload Filter is enabled by skipping them - Mastodon API: Search parameter `following` now correctly returns the followings rather than the followers - Mastodon API: Timelines hanging for (`number of posts with links * rich media timeout`) in the worst case. Reduced to just rich media timeout. @@ -138,7 +139,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Fix whole_word always returning false on filter get requests - Migrations not working on OTP releases if the database was connected over ssl - Fix relay following -- Fixed uploading webp images when the Exiftool Upload Filter is enabled ## [2.0.7] - 2020-06-13 From 7215563641e2a5096293128519d6a454aabc46f2 Mon Sep 17 00:00:00 2001 From: Alibek Omarov <a1ba.omarov@gmail.com> Date: Tue, 8 Sep 2020 11:32:46 +0000 Subject: [PATCH 068/128] description.exs: add ForceBotUnlistedPolicy --- config/description.exs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/config/description.exs b/config/description.exs index 18c133f02..2191e8d6d 100644 --- a/config/description.exs +++ b/config/description.exs @@ -1669,6 +1669,15 @@ } ] }, + %{ + group: :pleroma, + key: :mrf_force_bot_unlisted, + tab: :mrf, + related_policy: "Pleroma.Web.ActivityPub.MRF.ForceBotUnlistedPolicy", + label: "MRF Force Bot Unlisted Policy", + type: :boolean, + description: "Makes bot posts to disappear from public timelines" + }, %{ group: :pleroma, key: :mrf_subchain, From efff2caccccae76dd749df67322c61a70957268b Mon Sep 17 00:00:00 2001 From: Alibek Omarov <a1ba.omarov@gmail.com> Date: Tue, 8 Sep 2020 11:34:04 +0000 Subject: [PATCH 069/128] docs: cheatsheet: add ForceBotUnlistedPolicy --- docs/configuration/cheatsheet.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index a9a650fab..2ca4d7351 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -115,6 +115,7 @@ To add configuration to your config file, you can copy it from the base config. * `Pleroma.Web.ActivityPub.MRF.VocabularyPolicy`: Restricts activities to a configured set of vocabulary. (See [`:mrf_vocabulary`](#mrf_vocabulary)). * `Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy`: Rejects or delists posts based on their age when received. (See [`:mrf_object_age`](#mrf_object_age)). * `Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy`: Sets a default expiration on all posts made by users of the local instance. Requires `Pleroma.ActivityExpiration` to be enabled for processing the scheduled delections. + * `Pleroma.Web.ActivityPub.MRF.ForceBotUnlistedPolicy`: Makes all bot posts to disappear from public timelines. * `transparency`: Make the content of your Message Rewrite Facility settings public (via nodeinfo). * `transparency_exclusions`: Exclude specific instance names from MRF transparency. The use of the exclusions feature will be disclosed in nodeinfo as a boolean value. From 5d814f739cd2bb55822e0cfdb9ad2526d10482bb Mon Sep 17 00:00:00 2001 From: Alibek Omarov <a1ba.omarov@gmail.com> Date: Tue, 8 Sep 2020 11:35:34 +0000 Subject: [PATCH 070/128] changelog: add ForceBotUnlistedPolicy --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 46f1e859b..1b1ea02ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Unreleased +### Added + +- MRF policy to rewrite bot posts scope from public to unlisted + ### Removed - **Breaking:** Removed `Pleroma.Workers.Cron.StatsWorker` setting from Oban `:crontab`. From bb0d7b7aaad4f441a14c94c56081ec30c96ea799 Mon Sep 17 00:00:00 2001 From: Mark Felder <feld@FreeBSD.org> Date: Tue, 8 Sep 2020 09:31:47 -0500 Subject: [PATCH 071/128] Revert "description.exs: add ForceBotUnlistedPolicy" This reverts commit 7215563641e2a5096293128519d6a454aabc46f2. --- config/description.exs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/config/description.exs b/config/description.exs index d9f9b6b84..eac97ad64 100644 --- a/config/description.exs +++ b/config/description.exs @@ -1669,15 +1669,6 @@ } ] }, - %{ - group: :pleroma, - key: :mrf_force_bot_unlisted, - tab: :mrf, - related_policy: "Pleroma.Web.ActivityPub.MRF.ForceBotUnlistedPolicy", - label: "MRF Force Bot Unlisted Policy", - type: :boolean, - description: "Makes bot posts to disappear from public timelines" - }, %{ group: :pleroma, key: :mrf_subchain, From 788dececff5d1cea48cf37e59d9a46b48e0db6ed Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me> Date: Tue, 8 Sep 2020 16:08:01 +0200 Subject: [PATCH 072/128] test: remove extraneous :instances_favicons config bits --- config/test.exs | 2 -- test/web/instances/instance_test.exs | 2 -- 2 files changed, 4 deletions(-) diff --git a/config/test.exs b/config/test.exs index e9c2273e8..0ee6f1b7f 100644 --- a/config/test.exs +++ b/config/test.exs @@ -114,8 +114,6 @@ config :pleroma, Pleroma.Web.ApiSpec.CastAndValidate, strict: true -config :pleroma, :instances_favicons, enabled: false - config :pleroma, Pleroma.Uploaders.S3, bucket: nil, streaming_enabled: true, diff --git a/test/web/instances/instance_test.exs b/test/web/instances/instance_test.exs index 5d4efcebe..dc6ace843 100644 --- a/test/web/instances/instance_test.exs +++ b/test/web/instances/instance_test.exs @@ -112,8 +112,6 @@ test "Scrapes favicon URLs" do end test "Returns nil on too long favicon URLs" do - clear_config([:instances_favicons, :enabled], true) - long_favicon_url = "https://Lorem.ipsum.dolor.sit.amet/consecteturadipiscingelit/Praesentpharetrapurusutaliquamtempus/Mauriseulaoreetarcu/atfacilisisorci/Nullamporttitor/nequesedfeugiatmollis/dolormagnaefficiturlorem/nonpretiumsapienorcieurisus/Nullamveleratsem/Maecenassedaccumsanexnam/favicon.png" From f6723dc9bd4f05319835158be9937dda4400feb9 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me> Date: Tue, 8 Sep 2020 15:47:20 +0200 Subject: [PATCH 073/128] account_view_test: Remove empty test --- test/web/mastodon_api/views/account_view_test.exs | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/web/mastodon_api/views/account_view_test.exs b/test/web/mastodon_api/views/account_view_test.exs index 68a5d0091..9f22f9dcf 100644 --- a/test/web/mastodon_api/views/account_view_test.exs +++ b/test/web/mastodon_api/views/account_view_test.exs @@ -116,9 +116,6 @@ test "is nil when :instances_favicons is disabled", %{user: user} do end end - test "Favicon when :instance_favicons is enabled" do - end - test "Represent the user account for the account owner" do user = insert(:user) From fd7e9bdd25ad05eb6fca109be3b0fe5fa01a71bb Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov <alex.strizhakov@gmail.com> Date: Tue, 8 Sep 2020 17:12:38 +0300 Subject: [PATCH 074/128] don't run async tests, which use Mock --- test/plugs/admin_secret_authentication_plug_test.exs | 2 +- test/plugs/oauth_scopes_plug_test.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/plugs/admin_secret_authentication_plug_test.exs b/test/plugs/admin_secret_authentication_plug_test.exs index 89df03c4b..14094eda8 100644 --- a/test/plugs/admin_secret_authentication_plug_test.exs +++ b/test/plugs/admin_secret_authentication_plug_test.exs @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Plugs.AdminSecretAuthenticationPlugTest do - use Pleroma.Web.ConnCase, async: true + use Pleroma.Web.ConnCase import Mock import Pleroma.Factory diff --git a/test/plugs/oauth_scopes_plug_test.exs b/test/plugs/oauth_scopes_plug_test.exs index 884de7b4d..334316043 100644 --- a/test/plugs/oauth_scopes_plug_test.exs +++ b/test/plugs/oauth_scopes_plug_test.exs @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Plugs.OAuthScopesPlugTest do - use Pleroma.Web.ConnCase, async: true + use Pleroma.Web.ConnCase alias Pleroma.Plugs.OAuthScopesPlug alias Pleroma.Repo From 87d2805791e1dd6746009e8c1445719e8cbfd31d Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov <alex.strizhakov@gmail.com> Date: Tue, 8 Sep 2020 17:29:28 +0300 Subject: [PATCH 075/128] combo fixes --- lib/pleroma/stats.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/stats.ex b/lib/pleroma/stats.ex index e7f8d272c..e5c9c668b 100644 --- a/lib/pleroma/stats.ex +++ b/lib/pleroma/stats.ex @@ -23,6 +23,7 @@ def start_link(_) do @impl true def init(_args) do + if Pleroma.Config.get(:env) == :test, do: :ok = Ecto.Adapters.SQL.Sandbox.checkout(Repo) {:ok, nil, {:continue, :calculate_stats}} end From ee0e05f9301e149c769f36bfd0fc8527ec7b6326 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me> Date: Tue, 8 Sep 2020 10:43:57 +0200 Subject: [PATCH 076/128] Drop unused "inReplyToAtomUri" in objects --- lib/pleroma/web/activity_pub/transmogrifier.ex | 4 +--- test/web/activity_pub/transmogrifier_test.exs | 14 ++++---------- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 0831efadc..af4384213 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -168,7 +168,6 @@ def fix_in_reply_to(object, options \\ []) def fix_in_reply_to(%{"inReplyTo" => in_reply_to} = object, options) when not is_nil(in_reply_to) do in_reply_to_id = prepare_in_reply_to(in_reply_to) - object = Map.put(object, "inReplyToAtomUri", in_reply_to_id) depth = (options[:depth] || 0) + 1 if Federator.allowed_thread_distance?(depth) do @@ -176,9 +175,8 @@ def fix_in_reply_to(%{"inReplyTo" => in_reply_to} = object, options) %Activity{} <- Activity.get_create_by_object_ap_id(replied_object.data["id"]) do object |> Map.put("inReplyTo", replied_object.data["id"]) - |> Map.put("inReplyToAtomUri", object["inReplyToAtomUri"] || in_reply_to_id) |> Map.put("context", replied_object.data["context"] || object["conversation"]) - |> Map.drop(["conversation"]) + |> Map.drop(["conversation", "inReplyToAtomUri"]) else e -> Logger.warn("Couldn't fetch #{inspect(in_reply_to_id)}, error: #{inspect(e)}") diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 3fa41b0c7..6e096e9ec 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -116,7 +116,8 @@ test "it fetches reply-to activities if we don't have them" do "tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment" ) - assert returned_object.data["inReplyToAtomUri"] == "https://shitposter.club/notice/2827873" + assert returned_object.data["inReplyTo"] == + "tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment" end test "it does not fetch reply-to activities beyond max replies depth limit" do @@ -140,8 +141,7 @@ test "it does not fetch reply-to activities beyond max replies depth limit" do "tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment" ) - assert returned_object.data["inReplyToAtomUri"] == - "https://shitposter.club/notice/2827873" + assert returned_object.data["inReplyTo"] == "https://shitposter.club/notice/2827873" end end @@ -1072,7 +1072,7 @@ test "returns not modified object when hasn't containts inReplyTo field", %{data assert Transmogrifier.fix_in_reply_to(data) == data end - test "returns object with inReplyToAtomUri when denied incoming reply", %{data: data} do + test "returns object with inReplyTo when denied incoming reply", %{data: data} do Pleroma.Config.put([:instance, :federation_incoming_replies_max_depth], 0) object_with_reply = @@ -1080,26 +1080,22 @@ test "returns object with inReplyToAtomUri when denied incoming reply", %{data: modified_object = Transmogrifier.fix_in_reply_to(object_with_reply) assert modified_object["inReplyTo"] == "https://shitposter.club/notice/2827873" - assert modified_object["inReplyToAtomUri"] == "https://shitposter.club/notice/2827873" object_with_reply = Map.put(data["object"], "inReplyTo", %{"id" => "https://shitposter.club/notice/2827873"}) modified_object = Transmogrifier.fix_in_reply_to(object_with_reply) assert modified_object["inReplyTo"] == %{"id" => "https://shitposter.club/notice/2827873"} - assert modified_object["inReplyToAtomUri"] == "https://shitposter.club/notice/2827873" object_with_reply = Map.put(data["object"], "inReplyTo", ["https://shitposter.club/notice/2827873"]) modified_object = Transmogrifier.fix_in_reply_to(object_with_reply) assert modified_object["inReplyTo"] == ["https://shitposter.club/notice/2827873"] - assert modified_object["inReplyToAtomUri"] == "https://shitposter.club/notice/2827873" object_with_reply = Map.put(data["object"], "inReplyTo", []) modified_object = Transmogrifier.fix_in_reply_to(object_with_reply) assert modified_object["inReplyTo"] == [] - assert modified_object["inReplyToAtomUri"] == "" end @tag capture_log: true @@ -1117,8 +1113,6 @@ test "returns modified object when allowed incoming reply", %{data: data} do assert modified_object["inReplyTo"] == "tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment" - assert modified_object["inReplyToAtomUri"] == "https://shitposter.club/notice/2827873" - assert modified_object["context"] == "tag:shitposter.club,2017-05-05:objectType=thread:nonce=3c16e9c2681f6d26" end From 921f926e96fd07131d4b79f5a29caed17ae2cc56 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me> Date: Tue, 8 Sep 2020 09:13:11 +0200 Subject: [PATCH 077/128] Remove OStatus in testsuite --- lib/pleroma/object/containment.ex | 7 - test/fixtures/23211.atom | 508 ------------- test/fixtures/cw_retweet.xml | 68 -- test/fixtures/delete.xml | 39 - test/fixtures/dm.xml | 27 - test/fixtures/favorite.xml | 65 -- test/fixtures/favorite_with_local_note.xml | 64 -- test/fixtures/follow.xml | 68 -- test/fixtures/incoming_note_activity.xml | 42 - .../incoming_note_activity_answer.xml | 42 - test/fixtures/incoming_reply_mastodon.xml | 29 - .../incoming_websub_gnusocial_attachments.xml | 59 -- test/fixtures/lambadalambda.atom | 479 ------------ test/fixtures/mastodon-note-cw.xml | 39 - test/fixtures/mastodon-note-unlisted.xml | 38 - test/fixtures/mastodon-problematic.xml | 72 -- test/fixtures/mastodon_conversation.xml | 30 - test/fixtures/nil_mention_entry.xml | 52 -- test/fixtures/ostatus_incoming_post.xml | 57 -- test/fixtures/ostatus_incoming_post_tag.xml | 59 -- test/fixtures/ostatus_incoming_reply.xml | 60 -- test/fixtures/share-gs-local.xml | 99 --- test/fixtures/share-gs.xml | 99 --- test/fixtures/share.xml | 54 -- test/fixtures/tesla_mock/7369654.atom | 44 -- test/fixtures/tesla_mock/atarifrosch_feed.xml | 473 ------------ test/fixtures/tesla_mock/emelie.atom | 306 -------- ....php_api_statuses_user_timeline_1.atom.xml | 460 ----------- .../https___mamot.fr_users_Skruyb.atom | 342 --------- ...__mastodon.social_users_lambadalambda.atom | 464 ----------- .../https___pawoo.net_users_pekorino.atom | 231 ------ ...leroma.soykaf.com_users_lain_feed.atom.xml | 1 - ...er.club_api_statuses_show_2827873.atom.xml | 54 -- ...club_api_statuses_user_timeline_1.atom.xml | 454 ----------- ...ttps___shitposter.club_notice_2827873.json | 1 - ..._api_statuses_user_timeline_23211.atom.xml | 591 -------------- ..._api_statuses_user_timeline_29191.atom.xml | 719 ------------------ test/fixtures/tesla_mock/sakamoto.atom | 1 - .../tesla_mock/sakamoto_eal_feed.atom | 1 - .../tesla_mock/shp@pleroma.soykaf.com.feed | 1 - test/fixtures/tesla_mock/spc_5381.atom | 438 ----------- test/fixtures/unfollow.xml | 68 -- test/support/http_request_mock.ex | 174 ----- test/web/activity_pub/transmogrifier_test.exs | 16 +- .../controllers/search_controller_test.exs | 12 +- 45 files changed, 15 insertions(+), 6992 deletions(-) delete mode 100644 test/fixtures/23211.atom delete mode 100644 test/fixtures/cw_retweet.xml delete mode 100644 test/fixtures/delete.xml delete mode 100644 test/fixtures/dm.xml delete mode 100644 test/fixtures/favorite.xml delete mode 100644 test/fixtures/favorite_with_local_note.xml delete mode 100644 test/fixtures/follow.xml delete mode 100644 test/fixtures/incoming_note_activity.xml delete mode 100644 test/fixtures/incoming_note_activity_answer.xml delete mode 100644 test/fixtures/incoming_reply_mastodon.xml delete mode 100644 test/fixtures/incoming_websub_gnusocial_attachments.xml delete mode 100644 test/fixtures/lambadalambda.atom delete mode 100644 test/fixtures/mastodon-note-cw.xml delete mode 100644 test/fixtures/mastodon-note-unlisted.xml delete mode 100644 test/fixtures/mastodon-problematic.xml delete mode 100644 test/fixtures/mastodon_conversation.xml delete mode 100644 test/fixtures/nil_mention_entry.xml delete mode 100644 test/fixtures/ostatus_incoming_post.xml delete mode 100644 test/fixtures/ostatus_incoming_post_tag.xml delete mode 100644 test/fixtures/ostatus_incoming_reply.xml delete mode 100644 test/fixtures/share-gs-local.xml delete mode 100644 test/fixtures/share-gs.xml delete mode 100644 test/fixtures/share.xml delete mode 100644 test/fixtures/tesla_mock/7369654.atom delete mode 100644 test/fixtures/tesla_mock/atarifrosch_feed.xml delete mode 100644 test/fixtures/tesla_mock/emelie.atom delete mode 100644 test/fixtures/tesla_mock/http__gs.example.org_index.php_api_statuses_user_timeline_1.atom.xml delete mode 100644 test/fixtures/tesla_mock/https___mamot.fr_users_Skruyb.atom delete mode 100644 test/fixtures/tesla_mock/https___mastodon.social_users_lambadalambda.atom delete mode 100644 test/fixtures/tesla_mock/https___pawoo.net_users_pekorino.atom delete mode 100644 test/fixtures/tesla_mock/https___pleroma.soykaf.com_users_lain_feed.atom.xml delete mode 100644 test/fixtures/tesla_mock/https___shitposter.club_api_statuses_show_2827873.atom.xml delete mode 100644 test/fixtures/tesla_mock/https___shitposter.club_api_statuses_user_timeline_1.atom.xml delete mode 100644 test/fixtures/tesla_mock/https___shitposter.club_notice_2827873.json delete mode 100644 test/fixtures/tesla_mock/https___social.heldscal.la_api_statuses_user_timeline_23211.atom.xml delete mode 100644 test/fixtures/tesla_mock/https___social.heldscal.la_api_statuses_user_timeline_29191.atom.xml delete mode 100644 test/fixtures/tesla_mock/sakamoto.atom delete mode 100644 test/fixtures/tesla_mock/sakamoto_eal_feed.atom delete mode 100644 test/fixtures/tesla_mock/shp@pleroma.soykaf.com.feed delete mode 100644 test/fixtures/tesla_mock/spc_5381.atom delete mode 100644 test/fixtures/unfollow.xml diff --git a/lib/pleroma/object/containment.ex b/lib/pleroma/object/containment.ex index bc88e8a0c..29cb3d672 100644 --- a/lib/pleroma/object/containment.ex +++ b/lib/pleroma/object/containment.ex @@ -44,13 +44,6 @@ def get_object(_) do nil end - # TODO: We explicitly allow 'tag' URIs through, due to references to legacy OStatus - # objects being present in the test suite environment. Once these objects are - # removed, please also remove this. - if Mix.env() == :test do - defp compare_uris(_, %URI{scheme: "tag"}), do: :ok - end - defp compare_uris(%URI{host: host} = _id_uri, %URI{host: host} = _other_uri), do: :ok defp compare_uris(_id_uri, _other_uri), do: :error diff --git a/test/fixtures/23211.atom b/test/fixtures/23211.atom deleted file mode 100644 index d5d111baa..000000000 --- a/test/fixtures/23211.atom +++ /dev/null @@ -1,508 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<feed xml:lang="en-US" xmlns="http://www.w3.org/2005/Atom" xmlns:thr="http://purl.org/syndication/thread/1.0" xmlns:georss="http://www.georss.org/georss" xmlns:activity="http://activitystrea.ms/spec/1.0/" xmlns:media="http://purl.org/syndication/atommedia" xmlns:poco="http://portablecontacts.net/spec/1.0" xmlns:ostatus="http://ostatus.org/schema/1.0" xmlns:statusnet="http://status.net/schema/api/1/"> - <generator uri="https://gnu.io/social" version="1.0.2-dev">GNU social</generator> - <id>https://social.heldscal.la/api/statuses/user_timeline/23211.atom</id> - <title>lambadalambda timeline - Updates from lambadalambda on social.heldscal.la! - https://social.heldscal.la/avatar/23211-96-20170416114255.jpeg - 2017-05-02T14:59:30+00:00 - - http://activitystrea.ms/schema/1.0/person - https://social.heldscal.la/user/23211 - lambadalambda - Call me Deacon Blues. - - - - - - lambadalambda - Constance Variable - Call me Deacon Blues. - - Berlin - - - homepage - https://heldscal.la - true - - - - - - - - - - - - - - tag:social.heldscal.la,2017-05-02:fave:23211:comment:2015260:2017-05-02T14:45:47+00:00 - Favorite - lambadalambda favorited something by godemperorofdune: <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> It's because your instance decided to be trap! lol.</p> - - http://activitystrea.ms/schema/1.0/favorite - 2017-05-02T14:45:47+00:00 - 2017-05-02T14:45:47+00:00 - - http://activitystrea.ms/schema/1.0/comment - tag:pawoo.net,2017-05-02:objectId=7397439:objectType=Status - New comment by godemperorofdune - <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> It's because your instance decided to be trap! lol.</p> - - - - - - - tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=136e244b26cdf1e9 - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:social.heldscal.la,2017-05-02:noticeId=2015221:objectType=note - New note by lambadalambda - Some script thinks I'm a mastodon server.<br /> <br /> [info] GET /api/v1/timelines/public<br /> [debug] Processing with Fallback.RedirectController.redirector/2<br /> Parameters: %{&quot;limit&quot; =&gt; &quot;40&quot;, &quot;path&quot; =&gt; [&quot;api&quot;, &quot;v1&quot;, &quot;timelines&quot;, &quot;public&quot;]}<br /> Pipelines: [] - - - http://activitystrea.ms/schema/1.0/post - 2017-05-02T14:40:50+00:00 - 2017-05-02T14:40:50+00:00 - - tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=136e244b26cdf1e9 - - - - - - - http://activitystrea.ms/schema/1.0/comment - tag:social.heldscal.la,2017-05-02:noticeId=2014759:objectType=comment - New comment by lambadalambda - @<a href="https://mstdn.io/users/mattskala" class="h-card u-url p-nickname mention" title="Matthew Skala">mattskala</a> You and @<a href="https://mastodon.social/users/kevinmarks" class="h-card u-url p-nickname mention" title="Kevin Marks">kevinmarks</a> are not wrong, but my comment was a suggestion to users and admins: Don't use big instances, don't run big instances. Also, it's a secondary advice to devs: Don't add features that encourage big instances. - - - http://activitystrea.ms/schema/1.0/post - 2017-05-02T14:11:54+00:00 - 2017-05-02T14:11:54+00:00 - - - - tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=58e32e013ab6487d - - - - - - - - - http://activitystrea.ms/schema/1.0/comment - tag:social.heldscal.la,2017-05-02:noticeId=2014684:objectType=comment - New comment by lambadalambda - @<a href="https://mastodon.social/users/Ronkjeffries" class="h-card u-url p-nickname mention" title="Ron K Jeffries social">ronkjeffries</a> @<a href="https://xoxo.zone/users/KevinMarks" class="h-card u-url p-nickname mention" title="Kevin Marks ">kevinmarks</a> Usually people who run their own private instance just look at the timelines of other servers, follow a seed population and then go from there. This is of course hard on Mastodon, because it doesn't have a publicly visible timeline. - - - http://activitystrea.ms/schema/1.0/post - 2017-05-02T14:07:00+00:00 - 2017-05-02T14:07:00+00:00 - - - - tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=58e32e013ab6487d - - - - - - - - - tag:social.heldscal.la,2017-05-02:fave:23211:comment:2014584:2017-05-02T14:05:32+00:00 - Favorite - lambadalambda favorited something by mattskala: <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> It's reasonable to expect that instance sizes will obey a power-law distribution because that's what such things in nature nearly always do. If so, there'll necessarily be a few instances much larger than the others; even if most are small, the network both socially and technically has to be able to deal with the existence of the few large ones.</p> - - http://activitystrea.ms/schema/1.0/favorite - 2017-05-02T14:05:32+00:00 - 2017-05-02T14:05:32+00:00 - - http://activitystrea.ms/schema/1.0/comment - tag:mstdn.io,2017-05-02:objectId=1316931:objectType=Status - New comment by mattskala - <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> It's reasonable to expect that instance sizes will obey a power-law distribution because that's what such things in nature nearly always do. If so, there'll necessarily be a few instances much larger than the others; even if most are small, the network both socially and technically has to be able to deal with the existence of the few large ones.</p> - - - - - - - tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=58e32e013ab6487d - - - - - - - tag:social.heldscal.la,2017-05-02:fave:23211:comment:2013568:2017-05-02T14:05:29+00:00 - Favorite - lambadalambda favorited something by kevinmarks: <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> except instance populations will be power law distributed, and the problems for the tummlers are worse at scale</p> - - http://activitystrea.ms/schema/1.0/favorite - 2017-05-02T14:05:29+00:00 - 2017-05-02T14:05:29+00:00 - - http://activitystrea.ms/schema/1.0/comment - tag:xoxo.zone,2017-05-02:objectId=89478:objectType=Status - New comment by kevinmarks - <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> except instance populations will be power law distributed, and the problems for the tummlers are worse at scale</p> - - - - - - - tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=58e32e013ab6487d - - - - - - - tag:social.heldscal.la,2017-05-02:fave:23211:comment:2014060:2017-05-02T13:34:32+00:00 - Favorite - lambadalambda favorited something by gcarregues: <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> Oh purée ! Ma vie en images !</p> - - http://activitystrea.ms/schema/1.0/favorite - 2017-05-02T13:34:32+00:00 - 2017-05-02T13:34:32+00:00 - - http://activitystrea.ms/schema/1.0/comment - tag:mastodon.etalab.gouv.fr,2017-05-02:objectId=55287:objectType=Status - New comment by gcarregues - <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> Oh purée ! Ma vie en images !</p> - - - - - - - tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=2c27c27df8ec4dcc - - - - - - - tag:social.heldscal.la,2017-05-02:fave:23211:note:2013573:2017-05-02T13:03:33+00:00 - Favorite - lambadalambda favorited something by phildobangnz: also @<a href="https://sealion.club/user/579" class="h-card mention" title="Sim Bot">sim</a> reminder you are awesome; don't even trip- u kewler than Tutankhamen's cucumber, fam. Okay, good night. - - http://activitystrea.ms/schema/1.0/favorite - 2017-05-02T13:03:33+00:00 - 2017-05-02T13:03:33+00:00 - - http://activitystrea.ms/schema/1.0/note - tag:sealion.club,2017-05-02:noticeId=3060818:objectType=note - New note by phildobangnz - also @<a href="https://sealion.club/user/579" class="h-card mention" title="Sim Bot">sim</a> reminder you are awesome; don't even trip- u kewler than Tutankhamen's cucumber, fam. Okay, good night. - - - - - - - https://sealion.club/conversation/1633267 - - - - - - - http://activitystrea.ms/schema/1.0/comment - tag:social.heldscal.la,2017-05-02:noticeId=2013586:objectType=comment - New comment by lambadalambda - @<a href="https://xoxo.zone/users/KevinMarks" class="h-card u-url p-nickname mention" title="Kevin Marks ">kevinmarks</a> People can stay in their giant unmoderatable instances with meaningless public and federated timelines and experience constant federation drama if they want. I'll stay here with my 5 friends. - - - http://activitystrea.ms/schema/1.0/post - 2017-05-02T12:54:59+00:00 - 2017-05-02T12:54:59+00:00 - - - - tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=58e32e013ab6487d - - - - - - - - tag:social.heldscal.la,2017-05-02:fave:23211:note:2013486:2017-05-02T12:46:48+00:00 - Favorite - lambadalambda favorited something by fortune: There once was a dentist named Stone<br /> Who saw all his patients alone.<br /> In a fit of depravity<br /> He filled the wrong cavity,<br /> And my, how his practice has grown! - - http://activitystrea.ms/schema/1.0/favorite - 2017-05-02T12:46:48+00:00 - 2017-05-02T12:46:48+00:00 - - http://activitystrea.ms/schema/1.0/note - tag:gs.kawa-kun.com,2017-05-02:noticeId=1655658:objectType=note - New note by fortune - There once was a dentist named Stone<br /> Who saw all his patients alone.<br /> In a fit of depravity<br /> He filled the wrong cavity,<br /> And my, how his practice has grown! - - - - - - - https://gs.kawa-kun.com/conversation/714072 - - - - - - - tag:social.heldscal.la,2017-05-02:fave:23211:note:2013365:2017-05-02T12:37:55+00:00 - Favorite - lambadalambda favorited something by xj9: <p>&gt; rollerblading to work</p> - - http://activitystrea.ms/schema/1.0/favorite - 2017-05-02T12:37:55+00:00 - 2017-05-02T12:37:55+00:00 - - http://activitystrea.ms/schema/1.0/note - tag:sunshinegardens.org,2017-05-02:objectId=61020:objectType=Status - New note by xj9 - <p>&gt; rollerblading to work</p> - - - - - - - tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=5a0e98612f634218 - - - - - - - tag:social.heldscal.la,2017-05-02:fave:23211:comment:2013259:2017-05-02T12:29:03+00:00 - Favorite - lambadalambda favorited something by cereal: @<a href="https://gs.smuglo.li/user/28250" class="h-card mention" title="Bricky">thatbrickster</a> @<a href="https://social.heldscal.la/user/23211" class="h-card mention" title="Constance Variable">lambadalambda</a> But why? - - http://activitystrea.ms/schema/1.0/favorite - 2017-05-02T12:29:03+00:00 - 2017-05-02T12:29:03+00:00 - - http://activitystrea.ms/schema/1.0/comment - tag:sealion.club,2017-05-02:noticeId=3059985:objectType=comment - New comment by cereal - @<a href="https://gs.smuglo.li/user/28250" class="h-card mention" title="Bricky">thatbrickster</a> @<a href="https://social.heldscal.la/user/23211" class="h-card mention" title="Constance Variable">lambadalambda</a> But why? - - - - - - - tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=2c27c27df8ec4dcc - - - - - - - tag:social.heldscal.la,2017-05-02:fave:23211:comment:2013227:2017-05-02T12:24:27+00:00 - Favorite - lambadalambda favorited something by thatbrickster: @<a href="https://social.heldscal.la/user/23211" class="h-card u-url p-nickname mention" title="Constance Variable">lambadalambda</a> install gentoo - - http://activitystrea.ms/schema/1.0/favorite - 2017-05-02T12:24:27+00:00 - 2017-05-02T12:24:27+00:00 - - http://activitystrea.ms/schema/1.0/comment - tag:gs.smuglo.li,2017-05-02:noticeId=2144296:objectType=comment - New comment by thatbrickster - @<a href="https://social.heldscal.la/user/23211" class="h-card u-url p-nickname mention" title="Constance Variable">lambadalambda</a> install gentoo - - - - - - - tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=2c27c27df8ec4dcc - - - - - - - tag:social.heldscal.la,2017-05-02:fave:23211:comment:2013213:2017-05-02T12:22:53+00:00 - Favorite - lambadalambda favorited something by dwmatiz: @<a href="https://social.heldscal.la/user/23211" class="h-card mention">lambadalambda</a> *unzips dick* - - http://activitystrea.ms/schema/1.0/favorite - 2017-05-02T12:22:53+00:00 - 2017-05-02T12:22:53+00:00 - - http://activitystrea.ms/schema/1.0/comment - tag:sealion.club,2017-05-02:noticeId=3059800:objectType=comment - New comment by dwmatiz - @<a href="https://social.heldscal.la/user/23211" class="h-card mention">lambadalambda</a> *unzips dick* - - - - - - - tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=2c27c27df8ec4dcc - - - - - - - tag:social.heldscal.la,2017-05-02:fave:23211:comment:2013199:2017-05-02T12:22:03+00:00 - Favorite - lambadalambda favorited something by shpuld: @<a href="https://social.heldscal.la/user/23211" class="h-card mention" title="Constance Variable">lambadalambda</a> get #<span class="tag"><a href="https://shitposter.club/tag/cofe" rel="tag">cofe</a></span> - - http://activitystrea.ms/schema/1.0/favorite - 2017-05-02T12:22:03+00:00 - 2017-05-02T12:22:03+00:00 - - http://activitystrea.ms/schema/1.0/comment - tag:shitposter.club,2017-05-02:noticeId=2783524:objectType=comment - New comment by shpuld - @<a href="https://social.heldscal.la/user/23211" class="h-card mention" title="Constance Variable">lambadalambda</a> get #<span class="tag"><a href="https://shitposter.club/tag/cofe" rel="tag">cofe</a></span> - - - - - - - tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=2c27c27df8ec4dcc - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:social.heldscal.la,2017-05-02:noticeId=2013185:objectType=note - New note by lambadalambda - What now? <a href="https://social.heldscal.la/file/e4822d95de677757ff50d49672a4046c83218b76c04a0ad5e5f1f0a9a9eb1a74.gif" title="https://social.heldscal.la/file/e4822d95de677757ff50d49672a4046c83218b76c04a0ad5e5f1f0a9a9eb1a74.gif" rel="nofollow external noreferrer" class="attachment" id="attachment-422572">https://social.heldscal.la/attachment/422572</a> - - - http://activitystrea.ms/schema/1.0/post - 2017-05-02T12:21:04+00:00 - 2017-05-02T12:21:04+00:00 - - tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=2c27c27df8ec4dcc - - - - - - - - tag:social.heldscal.la,2017-05-02:fave:23211:note:2012929:2017-05-02T12:01:25+00:00 - Favorite - lambadalambda favorited something by drkmttr: <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> I checked out No Agenda because I saw you mention it several time. Sadly, I wasn't impressed. I'm all about varying perspectives but Adam and John basically just sound like resentful curmudgeons. It seems like their shtick is basically playing devil's advocate to everything to arouse some discontent. Just my two cents. 😉</p> - - http://activitystrea.ms/schema/1.0/favorite - 2017-05-02T12:01:25+00:00 - 2017-05-02T12:01:25+00:00 - - http://activitystrea.ms/schema/1.0/note - tag:mstdn.io,2017-05-02:objectId=1310093:objectType=Status - New note by drkmttr - <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> I checked out No Agenda because I saw you mention it several time. Sadly, I wasn't impressed. I'm all about varying perspectives but Adam and John basically just sound like resentful curmudgeons. It seems like their shtick is basically playing devil's advocate to everything to arouse some discontent. Just my two cents. 😉</p> - - - - - - - tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=2f329b4eb20e83e2 - - - - - - - tag:social.heldscal.la,2017-05-02:fave:23211:comment:2012336:2017-05-02T11:06:42+00:00 - Favorite - lambadalambda favorited something by clacke: @<a href="https://mastodon.org.uk/users/dick_turpin" class="h-card u-url p-nickname mention" title="dick_turpin">dickturpin</a> @<a href="http://quitter.se/user/113503" class="h-card u-url p-nickname mention" title="Luke">luke</a> Oh no, I miss being irritated by you, it helps me understand myself and others. Also it builds character. :-)<br /> <br /> So if this is not federation because you can't follow all of online mankind, what should we call it? Proto-federated? Pre-federated?<br /> <br /> The term has been used decades ago for just one Microsoft Active Directory domain cross-certifying the root of another, by mutual agreement. I don't see how it's any less relevant to opportunistic federation between open servers on an open internet.<br /> <br /> I'm not saying we should be satisfied, I'm just saying that "federate" is a useful word and to build a big system we need to start with a small one. And focus on the things we *can* change, like helping the OStatus network grow and making the tools more useful.<br /> <br /> Saying that the network's ideals have failed because other networks aren't joining is doing neither of that. - - http://activitystrea.ms/schema/1.0/favorite - 2017-05-02T11:06:42+00:00 - 2017-05-02T11:06:42+00:00 - - http://activitystrea.ms/schema/1.0/comment - tag:social.heldscal.la,2017-05-02:noticeId=2012336:objectType=comment - New comment by clacke - @<a href="https://mastodon.org.uk/users/dick_turpin" class="h-card u-url p-nickname mention" title="dick_turpin">dickturpin</a> @<a href="http://quitter.se/user/113503" class="h-card u-url p-nickname mention" title="Luke">luke</a> Oh no, I miss being irritated by you, it helps me understand myself and others. Also it builds character. :-)<br /> <br /> So if this is not federation because you can't follow all of online mankind, what should we call it? Proto-federated? Pre-federated?<br /> <br /> The term has been used decades ago for just one Microsoft Active Directory domain cross-certifying the root of another, by mutual agreement. I don't see how it's any less relevant to opportunistic federation between open servers on an open internet.<br /> <br /> I'm not saying we should be satisfied, I'm just saying that &quot;federate&quot; is a useful word and to build a big system we need to start with a small one. And focus on the things we *can* change, like helping the OStatus network grow and making the tools more useful.<br /> <br /> Saying that the network's ideals have failed because other networks aren't joining is doing neither of that. - - - - - - - https://s.wefamlee.be/conversation/16478 - - - - - - - tag:social.heldscal.la,2017-05-02:fave:23211:comment:2011332:2017-05-02T10:37:40+00:00 - Favorite - lambadalambda favorited something by moonman: @<a href="https://social.heldscal.la/user/23211" class="h-card mention" title="Constance Variable">lambadalambda</a> <a href="https://www.youtube.com/watch?v=mKLizztikRk" title="https://www.youtube.com/watch?v=mKLizztikRk" class="attachment" rel="nofollow">https://www.youtube.com/watch?v=mKLizztikRk</a> - - http://activitystrea.ms/schema/1.0/favorite - 2017-05-02T10:37:40+00:00 - 2017-05-02T10:37:40+00:00 - - http://activitystrea.ms/schema/1.0/comment - tag:shitposter.club,2017-05-02:noticeId=2781833:objectType=comment - New comment by moonman - @<a href="https://social.heldscal.la/user/23211" class="h-card mention" title="Constance Variable">lambadalambda</a> <a href="https://www.youtube.com/watch?v=mKLizztikRk" title="https://www.youtube.com/watch?v=mKLizztikRk" class="attachment" rel="nofollow">https://www.youtube.com/watch?v=mKLizztikRk</a> - - - - - - - tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=11d8b8c27d9513ec - - - - - - - http://activitystrea.ms/schema/1.0/comment - tag:social.heldscal.la,2017-05-02:noticeId=2012145:objectType=comment - New comment by lambadalambda - @<a href="https://sealion.club/user/186" class="h-card u-url p-nickname mention" title="I'M CEREAL U GUISE">cereal</a> ? No, you don't even need the identity servers for federation. - - - http://activitystrea.ms/schema/1.0/post - 2017-05-02T10:37:33+00:00 - 2017-05-02T10:37:33+00:00 - - - - https://sealion.club/conversation/1629037 - - - - - - - diff --git a/test/fixtures/cw_retweet.xml b/test/fixtures/cw_retweet.xml deleted file mode 100644 index c99a569d7..000000000 --- a/test/fixtures/cw_retweet.xml +++ /dev/null @@ -1,68 +0,0 @@ - - - https://mastodon.social/users/lambadalambda.atom - Critical Value - - 2017-04-16T21:47:25Z - https://files.mastodon.social/accounts/avatars/000/000/264/original/1429214160519.gif - - https://mastodon.social/users/lambadalambda - http://activitystrea.ms/schema/1.0/person - https://mastodon.social/users/lambadalambda - lambadalambda - lambadalambda@mastodon.social - - - - lambadalambda - Critical Value - public - - - - - - - tag:mastodon.social,2017-05-11:objectId=5647963:objectType=Status - 2017-05-11T10:23:15Z - 2017-05-11T10:23:15Z - lambadalambda shared a status by Skruyb@mamot.fr - http://activitystrea.ms/schema/1.0/activity - http://activitystrea.ms/schema/1.0/share - - tag:mamot.fr,2017-05-10:objectId=1294943:objectType=Status - 2017-05-10T17:31:44Z - 2017-05-10T17:31:45Z - New status by Skruyb@mamot.fr - - https://mamot.fr/users/Skruyb - http://activitystrea.ms/schema/1.0/person - https://mamot.fr/users/Skruyb - Skruyb - Skruyb@mamot.fr - <p>Fr and En.<br>Posts will disappear on a regular basis.</p> - - - - Skruyb - The 7th Son - Fr and En.Posts will disappear on a regular basis. - public - - http://activitystrea.ms/schema/1.0/comment - http://activitystrea.ms/schema/1.0/post - Hey. - <p><span class="h-card"><a href="https://mastodon.social/@lambadalambda">@<span>lambadalambda</span></a></span></p><p>Hey!!!</p> - - - public - - - - <p><span class="h-card"><a href="https://mastodon.social/@lambadalambda">@<span>lambadalambda</span></a></span></p><p>Hey!!!</p> - - public - - - - diff --git a/test/fixtures/delete.xml b/test/fixtures/delete.xml deleted file mode 100644 index 731e1c204..000000000 --- a/test/fixtures/delete.xml +++ /dev/null @@ -1,39 +0,0 @@ - - - https://mastodon.sdf.org/users/snowdusk.atom - snowdusk - Amateur live performance DJ/radio DJ on SDF's underground Internet radio http://aNONradio.net (LIVE Sat Sun Mon Tue 23:00-24:00 UTC) - http://snowdusk.sdf.org - 2017-06-17T04:14:34Z - https://mastodon.sdf.org/system/accounts/avatars/000/000/002/original/405a7652d5f60449.jpg?1497672873 - - https://mastodon.sdf.org/users/snowdusk - http://activitystrea.ms/schema/1.0/person - https://mastodon.sdf.org/users/snowdusk - snowdusk - snowdusk@mastodon.sdf.org - <p>Amateur live performance DJ/radio DJ on SDF&apos;s underground Internet radio <a href="http://anonradio.net/" rel="nofollow noopener" target="_blank"><span class="invisible">http://</span><span class="">anonradio.net/</span><span class="invisible"></span></a> (LIVE Sat Sun Mon Tue 23:00-24:00 UTC) - <a href="http://snowdusk.sdf.org/" rel="nofollow noopener" target="_blank"><span class="invisible">http://</span><span class="">snowdusk.sdf.org/</span><span class="invisible"></span></a></p> - - - - snowdusk - snowdusk - Amateur live performance DJ/radio DJ on SDF's underground Internet radio http://aNONradio.net (LIVE Sat Sun Mon Tue 23:00-24:00 UTC) - http://snowdusk.sdf.org - public - - - - - - - - tag:mastodon.sdf.org,2017-06-10:objectId=310513:objectType=Status - 2017-06-10T22:02:31Z - 2017-06-10T22:02:31Z - snowdusk deleted status - http://activitystrea.ms/schema/1.0/activity - http://activitystrea.ms/schema/1.0/delete - Deleted status - - - - diff --git a/test/fixtures/dm.xml b/test/fixtures/dm.xml deleted file mode 100644 index d0b8aa811..000000000 --- a/test/fixtures/dm.xml +++ /dev/null @@ -1,27 +0,0 @@ - - - tag:mastodon.social,2017-06-30:objectId=11260427:objectType=Status - 2017-06-30T13:27:47Z - 2017-06-30T13:27:47Z - New status by lambadalambda - - https://mastodon.social/users/lambadalambda - http://activitystrea.ms/schema/1.0/person - https://mastodon.social/users/lambadalambda - lambadalambda - lambadalambda@mastodon.social - - - lambadalambda - Critical Value - public - - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - <p><span class="h-card"><a href="https://pleroma.soykaf.com/users/lain" class="u-url mention">@<span>lain</span></a></span> Hey.</p> - - direct - - - - diff --git a/test/fixtures/favorite.xml b/test/fixtures/favorite.xml deleted file mode 100644 index c32b4a403..000000000 --- a/test/fixtures/favorite.xml +++ /dev/null @@ -1,65 +0,0 @@ - - - GNU social - https://social.heldscal.la/api/statuses/user_timeline/23211.atom - lambadalambda timeline - Updates from lambadalambda on social.heldscal.la! - https://social.heldscal.la/avatar/23211-96-20170416114255.jpeg - 2017-05-05T09:12:53+00:00 - - http://activitystrea.ms/schema/1.0/person - https://social.heldscal.la/user/23211 - lambadalambda - Call me Deacon Blues. - - - - - - lambadalambda - Constance Variable - Call me Deacon Blues. - - Berlin - - - homepage - https://heldscal.la - true - - - - - - - - - - - - - tag:social.heldscal.la,2017-05-05:fave:23211:comment:2061643:2017-05-05T09:12:50+00:00 - Favorite - lambadalambda favorited something by moonman: @<a href="https://shitposter.club/user/9655" class="h-card mention" title="Solidarity for Pigs">neimzr4luzerz</a> @<a href="https://gs.smuglo.li/user/2326" class="h-card mention" title="Dolus_McHonest">dolus</a> childhood poring over Strong's concordance and a koine Greek dictionary, fast forward to 2017 and some fuckstick who translates japanese jackoff material tells me you just need to make it sound right in English - - http://activitystrea.ms/schema/1.0/favorite - 2017-05-05T09:12:50+00:00 - 2017-05-05T09:12:50+00:00 - - http://activitystrea.ms/schema/1.0/comment - tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment - New comment by moonman - @<a href="https://shitposter.club/user/9655" class="h-card mention" title="Solidarity for Pigs">neimzr4luzerz</a> @<a href="https://gs.smuglo.li/user/2326" class="h-card mention" title="Dolus_McHonest">dolus</a> childhood poring over Strong's concordance and a koine Greek dictionary, fast forward to 2017 and some fuckstick who translates japanese jackoff material tells me you just need to make it sound right in English - - - - - - - tag:social.heldscal.la,2017-05-05:objectType=thread:nonce=55ead90125cd4bd4 - - - - - - diff --git a/test/fixtures/favorite_with_local_note.xml b/test/fixtures/favorite_with_local_note.xml deleted file mode 100644 index 3c955607d..000000000 --- a/test/fixtures/favorite_with_local_note.xml +++ /dev/null @@ -1,64 +0,0 @@ - - - GNU social - https://social.heldscal.la/api/statuses/user_timeline/23211.atom - lambadalambda timeline - Updates from lambadalambda on social.heldscal.la! - https://social.heldscal.la/avatar/23211-96-20170416114255.jpeg - 2017-05-05T09:12:53+00:00 - - http://activitystrea.ms/schema/1.0/person - https://social.heldscal.la/user/23211 - lambadalambda - Call me Deacon Blues. - - - - - - lambadalambda - Constance Variable - Call me Deacon Blues. - - Berlin - - - homepage - https://heldscal.la - true - - - - - - - - - - - - - tag:social.heldscal.la,2017-05-05:fave:23211:comment:2061643:2017-05-05T09:12:50+00:00 - Favorite - lambadalambda favorited something by moonman: @<a href="https://shitposter.club/user/9655" class="h-card mention" title="Solidarity for Pigs">neimzr4luzerz</a> @<a href="https://gs.smuglo.li/user/2326" class="h-card mention" title="Dolus_McHonest">dolus</a> childhood poring over Strong's concordance and a koine Greek dictionary, fast forward to 2017 and some fuckstick who translates japanese jackoff material tells me you just need to make it sound right in English - - http://activitystrea.ms/schema/1.0/favorite - 2017-05-05T09:12:50+00:00 - 2017-05-05T09:12:50+00:00 - - http://activitystrea.ms/schema/1.0/comment - localid - New comment by moonman - @<a href="https://shitposter.club/user/9655" class="h-card mention" title="Solidarity for Pigs">neimzr4luzerz</a> @<a href="https://gs.smuglo.li/user/2326" class="h-card mention" title="Dolus_McHonest">dolus</a> childhood poring over Strong's concordance and a koine Greek dictionary, fast forward to 2017 and some fuckstick who translates japanese jackoff material tells me you just need to make it sound right in English - - - - - - tag:social.heldscal.la,2017-05-05:objectType=thread:nonce=55ead90125cd4bd4 - - - - - - diff --git a/test/fixtures/follow.xml b/test/fixtures/follow.xml deleted file mode 100644 index d4e89954b..000000000 --- a/test/fixtures/follow.xml +++ /dev/null @@ -1,68 +0,0 @@ - - - GNU social - https://social.heldscal.la/api/statuses/user_timeline/23211.atom - lambadalambda timeline - Updates from lambadalambda on social.heldscal.la! - https://social.heldscal.la/avatar/23211-96-20170416114255.jpeg - 2017-05-07T09:54:49+00:00 - - http://activitystrea.ms/schema/1.0/person - https://social.heldscal.la/user/23211 - lambadalambda - Call me Deacon Blues. - - - - - - lambadalambda - Constance Variable - Call me Deacon Blues. - - Berlin - - - homepage - https://heldscal.la - true - - - - - - - - - - - - - tag:social.heldscal.la,2017-05-07:subscription:23211:person:44803:2017-05-07T09:54:48+00:00 - Constance Variable (lambadalambda@social.heldscal.la)'s status on Sunday, 07-May-2017 09:54:49 UTC - <a href="https://social.heldscal.la/lambadalambda">Constance Variable</a> started following <a href="https://pawoo.net/@pekorino">ｍｏｎｏ</a>. - - http://activitystrea.ms/schema/1.0/follow - 2017-05-07T09:54:49+00:00 - 2017-05-07T09:54:49+00:00 - - http://activitystrea.ms/schema/1.0/person - https://pawoo.net/users/pekorino - ｍｏｎｏ - http://shitposter.club/mono 孤独のグルメ - - - - - pekorino - ｍｏｎｏ - http://shitposter.club/mono 孤独のグルメ - - - tag:social.heldscal.la,2017-05-07:objectType=thread:nonce=6e80caf94e03029f - - - - - - diff --git a/test/fixtures/incoming_note_activity.xml b/test/fixtures/incoming_note_activity.xml deleted file mode 100644 index 21eda2d30..000000000 --- a/test/fixtures/incoming_note_activity.xml +++ /dev/null @@ -1,42 +0,0 @@ - - http://activitystrea.ms/schema/1.0/note - tag:gs.example.org:4040,2017-04-23:noticeId=29:objectType=note - New note by lambda - @<a href="http://pleroma.example.org:4000/users/lain3" class="h-card mention">lain3</a> - - - - - http://activitystrea.ms/schema/1.0/post - 2017-04-23T14:51:03+00:00 - 2017-04-23T14:51:03+00:00 - - http://activitystrea.ms/schema/1.0/person - http://gs.example.org:4040/index.php/user/1 - lambda - - - - - lambda - lambda - - - - - tag:gs.example.org:4040,2017-04-23:objectType=thread:nonce=f09e22f58abd5c7b - - - - http://gs.example.org:4040/index.php/api/statuses/user_timeline/1.atom - lambda - - - - http://gs.example.org:4040/theme/neo-gnu/default-avatar-profile.png - 2017-04-23T14:51:03+00:00 - - - - - diff --git a/test/fixtures/incoming_note_activity_answer.xml b/test/fixtures/incoming_note_activity_answer.xml deleted file mode 100644 index b1244faa6..000000000 --- a/test/fixtures/incoming_note_activity_answer.xml +++ /dev/null @@ -1,42 +0,0 @@ - - http://activitystrea.ms/schema/1.0/note - tag:gs.example.org:4040,2017-04-25:noticeId=55:objectType=note - New note by lambda - hey. - - - http://activitystrea.ms/schema/1.0/post - 2017-04-25T18:16:13+00:00 - 2017-04-25T18:16:13+00:00 - - http://activitystrea.ms/schema/1.0/person - http://gs.example.org:4040/index.php/user/1 - lambda - - - - - lambda - lambda - - - - - - - http://pleroma.example.org:4000/contexts/8f6f45d4-8e4d-4e1a-a2de-09f27367d2d0 - - - - http://gs.example.org:4040/index.php/api/statuses/user_timeline/1.atom - lambda - - - - http://gs.example.org:4040/theme/neo-gnu/default-avatar-profile.png - 2017-04-25T18:16:13+00:00 - - - - - diff --git a/test/fixtures/incoming_reply_mastodon.xml b/test/fixtures/incoming_reply_mastodon.xml deleted file mode 100644 index 8ee1186cc..000000000 --- a/test/fixtures/incoming_reply_mastodon.xml +++ /dev/null @@ -1,29 +0,0 @@ - - - tag:mastodon.social,2017-05-02:objectId=4901603:objectType=Status - 2017-05-02T18:33:06Z - 2017-05-02T18:33:06Z - New status by lambadalambda - - https://mastodon.social/users/lambadalambda - http://activitystrea.ms/schema/1.0/person - https://mastodon.social/users/lambadalambda - lambadalambda - lambadalambda@mastodon.social - - - - lambadalambda - Critical Value - public - - http://activitystrea.ms/schema/1.0/comment - http://activitystrea.ms/schema/1.0/post - <p><span class="h-card"><a href="https://pleroma.soykaf.com/users/lain" class="u-url mention">@<span>lain</span></a></span> hey</p> - - - public - - - - diff --git a/test/fixtures/incoming_websub_gnusocial_attachments.xml b/test/fixtures/incoming_websub_gnusocial_attachments.xml deleted file mode 100644 index 9d331ef32..000000000 --- a/test/fixtures/incoming_websub_gnusocial_attachments.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - GNU social - https://social.heldscal.la/api/statuses/user_timeline/23211.atom - lambadalambda timeline - Updates from lambadalambda on social.heldscal.la! - https://social.heldscal.la/avatar/23211-96-20170416114255.jpeg - 2017-05-02T20:29:35+00:00 - - http://activitystrea.ms/schema/1.0/person - https://social.heldscal.la/user/23211 - lambadalambda - Call me Deacon Blues. - - - - - - lambadalambda - Constance Variable - Call me Deacon Blues. - - Berlin - - - homepage - https://heldscal.la - true - - - - - - - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:social.heldscal.la,2017-05-02:noticeId=2020923:objectType=note - New note by lambadalambda - Okay gonna stream some cool games!! <a href="https://social.heldscal.la/file/7ed5ee508e6376a6e3dd581e17e7ed0b7b638147c7e86784bf83abc2641ee3d4.gif" title="https://social.heldscal.la/file/7ed5ee508e6376a6e3dd581e17e7ed0b7b638147c7e86784bf83abc2641ee3d4.gif" rel="nofollow external noreferrer" class="attachment" id="attachment-423842">https://social.heldscal.la/attachment/423842</a> <a href="https://social.heldscal.la/file/4c209099cadfc5afd3e27a334aa0db96b3a7510dde1603305d68a2707e59a11f.png" title="https://social.heldscal.la/file/4c209099cadfc5afd3e27a334aa0db96b3a7510dde1603305d68a2707e59a11f.png" rel="nofollow external noreferrer" class="attachment" id="attachment-423843">https://social.heldscal.la/attachment/423843</a> - - - http://activitystrea.ms/schema/1.0/post - 2017-05-02T20:29:35+00:00 - 2017-05-02T20:29:35+00:00 - - tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=26c7afdcbcf4ebd4 - - - - - - - - diff --git a/test/fixtures/lambadalambda.atom b/test/fixtures/lambadalambda.atom deleted file mode 100644 index 964a416f7..000000000 --- a/test/fixtures/lambadalambda.atom +++ /dev/null @@ -1,479 +0,0 @@ - - - https://mastodon.social/users/lambadalambda.atom - Critical Value - - 2017-04-16T21:47:25Z - https://files.mastodon.social/accounts/avatars/000/000/264/original/1429214160519.gif?1492379244 - - https://mastodon.social/users/lambadalambda - http://activitystrea.ms/schema/1.0/person - https://mastodon.social/users/lambadalambda - lambadalambda - lambadalambda@mastodon.social - a cool dude. - - - - lambadalambda - Critical Value - public - - - - - - - - tag:mastodon.social,2017-04-07:objectId=1874242:objectType=Status - 2017-04-07T11:02:56Z - 2017-04-07T11:02:56Z - lambadalambda shared a status by 0xroy@social.wxcafe.net - http://activitystrea.ms/schema/1.0/activity - http://activitystrea.ms/schema/1.0/share - - tag:social.wxcafe.net,2017-04-07:objectId=72554:objectType=Status - 2017-04-07T11:01:59Z - 2017-04-07T11:02:00Z - New status by 0xroy@social.wxcafe.net - - https://social.wxcafe.net/users/0xroy - http://activitystrea.ms/schema/1.0/person - https://social.wxcafe.net/users/0xroy - 0xroy - 0xroy@social.wxcafe.net - ta caution weeb | discussions privées : <a href="https://💌.0xroy.me" rel="nofollow noopener" target="_blank"><span class="invisible">https://</span><span class="">💌.0xroy.me</span><span class="invisible"></span></a> - - - - 0xroy - 「R O Y 🍵 B O S」 - ta caution weeb | discussions privées : <a href="https://%F0%9F%92%8C.0xroy.me" rel="nofollow noopener"><span class="invisible">https://</span><span class="">💌.0xroy.me</span><span class="invisible"></span></a> - public - - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - <p>someone pls eli5 matrix (protocol) and riot</p> - - public - - - <p>someone pls eli5 matrix (protocol) and riot</p> - - public - - - - - tag:mastodon.social,2017-04-06:objectId=1768247:objectType=Status - 2017-04-06T11:10:19Z - 2017-04-06T11:10:19Z - lambadalambda shared a status by areyoutoo@mastodon.xyz - http://activitystrea.ms/schema/1.0/activity - http://activitystrea.ms/schema/1.0/share - - tag:mastodon.xyz,2017-04-05:objectId=133327:objectType=Status - 2017-04-05T17:36:41Z - 2017-04-05T18:12:14Z - New status by areyoutoo@mastodon.xyz - - https://mastodon.xyz/users/areyoutoo - http://activitystrea.ms/schema/1.0/person - https://mastodon.xyz/users/areyoutoo - areyoutoo - areyoutoo@mastodon.xyz - devops | retired gamedev | always boost puppy pics - - - - areyoutoo - Raw Butter - devops | retired gamedev | always boost puppy pics - public - - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - <p>Some UX thoughts for <a href="https://mastodon.xyz/tags/mastodev" class="mention hashtag">#<span>mastodev</span></a>:</p><p>- Would be nice if I could work on multiple draft toots? Clicking to reply to someone seems to erase any draft I had been working on.</p><p>- Kinda risky to click on the Federated Timeline if it loads new toots and scrolls 10ms before I click on something.</p><p>I probably don't know enough web frontend to help, but it might be fun to try.</p> - - - public - - - <p>Some UX thoughts for <a href="https://mastodon.xyz/tags/mastodev" class="mention hashtag">#<span>mastodev</span></a>:</p><p>- Would be nice if I could work on multiple draft toots? Clicking to reply to someone seems to erase any draft I had been working on.</p><p>- Kinda risky to click on the Federated Timeline if it loads new toots and scrolls 10ms before I click on something.</p><p>I probably don't know enough web frontend to help, but it might be fun to try.</p> - - public - - - - - tag:mastodon.social,2017-04-06:objectId=1764509:objectType=Status - 2017-04-06T10:15:38Z - 2017-04-06T10:15:38Z - New status by lambadalambda - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - This is a test for cw federation - <p>This is a test for cw federation body text.</p> - - public - - - - - tag:mastodon.social,2017-04-05:objectId=1645208:objectType=Status - 2017-04-05T07:14:53Z - 2017-04-05T07:14:53Z - lambadalambda shared a status by lambadalambda@social.heldscal.la - http://activitystrea.ms/schema/1.0/activity - http://activitystrea.ms/schema/1.0/share - - tag:social.heldscal.la,2017-04-05:noticeId=1502088:objectType=note - 2017-04-05T06:12:09Z - 2017-04-05T07:12:47Z - New status by lambadalambda@social.heldscal.la - - https://social.heldscal.la/user/23211 - http://activitystrea.ms/schema/1.0/person - https://social.heldscal.la/user/23211 - lambadalambda - lambadalambda@social.heldscal.la - Call me Deacon Blues. - - - - lambadalambda - Constance Variable - Call me Deacon Blues. - public - - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - Federation 101: <a href="https://www.youtube.com/watch?v=t1lYU5CA40o" rel="nofollow external noreferrer" class="attachment thumbnail">https://www.youtube.com/watch?v=t1lYU5CA40o</a> - - public - - - Federation 101: <a href="https://www.youtube.com/watch?v=t1lYU5CA40o" rel="nofollow external noreferrer" class="attachment thumbnail">https://www.youtube.com/watch?v=t1lYU5CA40o</a> - - public - - - - - tag:mastodon.social,2017-04-05:objectId=1641750:objectType=Status - 2017-04-05T05:44:48Z - 2017-04-05T05:44:48Z - New status by lambadalambda - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> just a test.</p> - - - public - - - - - tag:mastodon.social,2017-04-04:objectId=1540149:objectType=Status - 2017-04-04T06:31:09Z - 2017-04-04T06:31:09Z - New status by lambadalambda - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - <p>Looks like you still can&apos;t delete your account here (PRIVACY!), but I won&apos;t be posting here anymore, my main account is <span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span></p> - - - public - - - - - tag:mastodon.social,2017-04-04:objectId=1539608:objectType=Status - 2017-04-04T06:18:16Z - 2017-04-04T06:18:16Z - New status by lambadalambda - http://activitystrea.ms/schema/1.0/comment - http://activitystrea.ms/schema/1.0/post - <p><span class="h-card"><a href="https://mastodon.social/@ghostbar" class="u-url mention">@<span>ghostbar</span></a></span> Remember to rewrite it in Rust once you&apos;re done.</p> - - - public - - - - - - tag:mastodon.social,2017-04-03:objectId=1504813:objectType=Status - 2017-04-03T18:01:20Z - 2017-04-03T18:01:20Z - New status by lambadalambda - http://activitystrea.ms/schema/1.0/comment - http://activitystrea.ms/schema/1.0/post - <p><span class="h-card"><a href="https://mastodon.xyz/@Azurolu" class="u-url mention">@<span>Azurolu</span></a></span> You mean gs.smuglo.li?</p> - - - public - - - - - - tag:mastodon.social,2017-04-03:objectId=1504805:objectType=Status - 2017-04-03T18:01:05Z - 2017-04-03T18:01:05Z - New status by lambadalambda - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - <p>There&apos;s nothing wrong with having several alt accounts all across the fediverse. Try out another mastodon instance (<a href="https://icosahedron.website" rel="nofollow noopener" target="_blank"><span class="invisible">https://</span><span class="">icosahedron.website</span><span class="invisible"></span></a>) or a GNU Social instance (like <a href="https://shitposter.club" rel="nofollow noopener" target="_blank"><span class="invisible">https://</span><span class="">shitposter.club</span><span class="invisible"></span></a> or <a href="https://freezepeach.xyz" rel="nofollow noopener" target="_blank"><span class="invisible">https://</span><span class="">freezepeach.xyz</span><span class="invisible"></span></a>), or friendica. They are all on the same network, so you can still follow all your friends!</p> - - public - - - - - tag:mastodon.social,2017-04-03:objectId=1503965:objectType=Status - 2017-04-03T17:31:30Z - 2017-04-03T17:31:30Z - New status by lambadalambda - http://activitystrea.ms/schema/1.0/comment - http://activitystrea.ms/schema/1.0/post - <p><span class="h-card"><a href="https://mastodon.social/@20Hz" class="u-url mention">@<span>20Hz</span></a></span> you could also try out a GS instance, which are on the same network :)</p> - - - public - - - - - - tag:mastodon.social,2017-04-03:objectId=1503955:objectType=Status - 2017-04-03T17:31:08Z - 2017-04-03T17:31:08Z - lambadalambda shared a status by shpuld@shitposter.club - http://activitystrea.ms/schema/1.0/activity - http://activitystrea.ms/schema/1.0/share - - tag:shitposter.club,2017-04-03:noticeId=2251717:objectType=note - 2017-04-03T17:06:43Z - 2017-04-03T17:12:06Z - New status by shpuld@shitposter.club - - https://shitposter.club/user/5381 - http://activitystrea.ms/schema/1.0/person - https://shitposter.club/user/5381 - shpuld - shpuld@shitposter.club - - - - - shpuld - shp - public - - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - reposting the classic <a href="https://shitposter.club/file/89c5fe483526caf3a46cfc5cdd4ae68061054350e767397731af658d54786e31.jpg" class="attachment" rel="nofollow external">https://shitposter.club/attachment/219846</a> - - - public - - - reposting the classic <a href="https://shitposter.club/file/89c5fe483526caf3a46cfc5cdd4ae68061054350e767397731af658d54786e31.jpg" class="attachment" rel="nofollow external">https://shitposter.club/attachment/219846</a> - - public - - - - - tag:mastodon.social,2017-04-03:objectId=1503929:objectType=Status - 2017-04-03T17:30:43Z - 2017-04-03T17:30:43Z - New status by lambadalambda - http://activitystrea.ms/schema/1.0/comment - http://activitystrea.ms/schema/1.0/post - <p><span class="h-card"><a href="https://mastodon.social/@ghostbar" class="u-url mention">@<span>ghostbar</span></a></span> Normally you shouldn&apos;t be running tens of thousands of users on one instance... That&apos;s one of the reasons for federation.</p> - - - public - - - - - - tag:mastodon.social,2017-04-03:objectId=1477255:objectType=Status - 2017-04-03T08:24:39Z - 2017-04-03T08:24:39Z - New status by lambadalambda - http://activitystrea.ms/schema/1.0/comment - http://activitystrea.ms/schema/1.0/post - <p><span class="h-card"><a href="https://mastodon.social/@dot_tiff" class="u-url mention">@<span>dot_tiff</span></a></span> it&apos;s the vaporwave mode.</p> - - - public - - - - - - tag:mastodon.social,2017-04-03:objectId=1476210:objectType=Status - 2017-04-03T07:45:42Z - 2017-04-03T07:45:42Z - lambadalambda shared a status by lambadalambda@social.heldscal.la - http://activitystrea.ms/schema/1.0/activity - http://activitystrea.ms/schema/1.0/share - - tag:social.heldscal.la,2017-04-03:noticeId=1475727:objectType=note - 2017-04-03T07:44:43Z - 2017-04-03T07:44:48Z - New status by lambadalambda@social.heldscal.la - - https://social.heldscal.la/user/23211 - http://activitystrea.ms/schema/1.0/person - https://social.heldscal.la/user/23211 - lambadalambda - lambadalambda@social.heldscal.la - Call me Deacon Blues. - - - - lambadalambda - Constance Variable - Call me Deacon Blues. - public - - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - Here's a song by the original anti-idol, Togawa Jun: <a href="https://www.youtube.com/watch?v=kNI_NK2YY-s" rel="nofollow external noreferrer" class="attachment">https://www.youtube.com/watch?v=kNI_NK2YY-s</a> - - public - - - Here's a song by the original anti-idol, Togawa Jun: <a href="https://www.youtube.com/watch?v=kNI_NK2YY-s" rel="nofollow external noreferrer" class="attachment">https://www.youtube.com/watch?v=kNI_NK2YY-s</a> - - public - - - - - tag:mastodon.social,2017-04-03:objectId=1476047:objectType=Status - 2017-04-03T07:39:14Z - 2017-04-03T07:39:14Z - New status by lambadalambda - http://activitystrea.ms/schema/1.0/comment - http://activitystrea.ms/schema/1.0/post - <p><span class="h-card"><a href="https://mastodon.social/@amrrr" class="u-url mention">@<span>amrrr</span></a></span> tumblr/10, but pretty good!</p> - - - public - - - - - - tag:mastodon.social,2017-04-03:objectId=1475949:objectType=Status - 2017-04-03T07:35:45Z - 2017-04-03T07:35:45Z - New status by lambadalambda - http://activitystrea.ms/schema/1.0/comment - http://activitystrea.ms/schema/1.0/post - <p><span class="h-card"><a href="https://mastodon.social/@Shookaite" class="u-url mention">@<span>Shookaite</span></a></span> Oh, you mean like userstyles?</p> - - - public - - - - - - tag:mastodon.social,2017-04-03:objectId=1475581:objectType=Status - 2017-04-03T07:20:03Z - 2017-04-03T07:20:03Z - New status by lambadalambda - http://activitystrea.ms/schema/1.0/comment - http://activitystrea.ms/schema/1.0/post - <p><span class="h-card"><a href="https://mastodon.social/@Shookaite" class="u-url mention">@<span>Shookaite</span></a></span> Would be nice if someone helped port Pleroma to Mastodon, that has a theme switcher (click on the cog in the upper right): <a href="https://pleroma.heldscal.la/main/all" rel="nofollow noopener" target="_blank"><span class="invisible">https://</span><span class="">pleroma.heldscal.la/main/all</span><span class="invisible"></span></a></p> - - - public - - - - - - tag:mastodon.social,2017-04-02:objectId=1457325:objectType=Status - 2017-04-02T21:57:43Z - 2017-04-02T21:57:43Z - New status by lambadalambda - http://activitystrea.ms/schema/1.0/comment - http://activitystrea.ms/schema/1.0/post - <p><span class="h-card"><a href="https://mastodon.social/@rhosyn" class="u-url mention">@<span>rhosyn</span></a></span> <span class="h-card"><a href="https://mastodon.social/@Meaningness" class="u-url mention">@<span>Meaningness</span></a></span> you could take a look at those listed at social.guhnoo.org</p> - - - - public - - - - - - tag:mastodon.social,2017-04-02:objectId=1447926:objectType=Status - 2017-04-02T18:31:52Z - 2017-04-02T18:31:52Z - New status by lambadalambda - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - <p>My main account is <span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> , btw.</p> - - - public - - - - - tag:mastodon.social,2017-04-02:objectId=1447878:objectType=Status - 2017-04-02T18:30:37Z - 2017-04-02T18:30:37Z - lambadalambda shared a status by Firstaide@awoo.space - http://activitystrea.ms/schema/1.0/activity - http://activitystrea.ms/schema/1.0/share - - tag:awoo.space,2017-04-02:objectId=135324:objectType=Status - 2017-04-02T18:29:32Z - 2017-04-02T18:29:32Z - New status by Firstaide@awoo.space - - https://awoo.space/users/Firstaide - http://activitystrea.ms/schema/1.0/person - https://awoo.space/users/Firstaide - Firstaide - Firstaide@awoo.space - A smol awoo account, for a smol autistic 💙 -They/them please! -NB/white/ace - - - - Firstaide - Miff🚑✨ - A smol awoo account, for a smol autistic 💙 -They/them please! -NB/white/ace - public - - http://activitystrea.ms/schema/1.0/comment - http://activitystrea.ms/schema/1.0/post - <p><a href="https://mastodon.social/users/lambadalambda" class="h-card u-url p-nickname mention">@<span>lambadalambda</span></a> yeah, I think that's p much the big issue here? <br>When I first heard of Masto, I thought it was just like twitter at first, I had no idea federation was even a thing?, and I actually joined p early on? :-o </p><p>idk I think more stuff needs to be done about federation promotion, but honestly its gotta come from the get go when people get here to make an account I feel :-o</p> - - - public - - - - <p><a href="https://mastodon.social/users/lambadalambda" class="h-card u-url p-nickname mention">@<span>lambadalambda</span></a> yeah, I think that's p much the big issue here? <br>When I first heard of Masto, I thought it was just like twitter at first, I had no idea federation was even a thing?, and I actually joined p early on? :-o </p><p>idk I think more stuff needs to be done about federation promotion, but honestly its gotta come from the get go when people get here to make an account I feel :-o</p> - - public - - - - diff --git a/test/fixtures/mastodon-note-cw.xml b/test/fixtures/mastodon-note-cw.xml deleted file mode 100644 index 02f49dd61..000000000 --- a/test/fixtures/mastodon-note-cw.xml +++ /dev/null @@ -1,39 +0,0 @@ - - - https://mastodon.social/users/lambadalambda.atom - Critical Value - - 2017-04-16T21:47:25Z - https://files.mastodon.social/accounts/avatars/000/000/264/original/1429214160519.gif - - https://mastodon.social/users/lambadalambda - http://activitystrea.ms/schema/1.0/person - https://mastodon.social/users/lambadalambda - lambadalambda - lambadalambda@mastodon.social - - - - lambadalambda - Critical Value - public - - - - - - - tag:mastodon.social,2017-05-10:objectId=5551985:objectType=Status - 2017-05-10T12:21:36Z - 2017-05-10T12:21:36Z - New status by lambadalambda - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - technologic - <p>test</p> - - public - - - - diff --git a/test/fixtures/mastodon-note-unlisted.xml b/test/fixtures/mastodon-note-unlisted.xml deleted file mode 100644 index d21017b80..000000000 --- a/test/fixtures/mastodon-note-unlisted.xml +++ /dev/null @@ -1,38 +0,0 @@ - - - https://mastodon.social/users/lambadalambda.atom - Critical Value - - 2017-04-16T21:47:25Z - https://files.mastodon.social/accounts/avatars/000/000/264/original/1429214160519.gif - - https://mastodon.social/users/lambadalambda - http://activitystrea.ms/schema/1.0/person - https://mastodon.social/users/lambadalambda - lambadalambda - lambadalambda@mastodon.social - - - - lambadalambda - Critical Value - public - - - - - - - tag:mastodon.social,2017-05-10:objectId=5551985:objectType=Status - 2017-05-10T12:21:36Z - 2017-05-10T12:21:36Z - New status by lambadalambda - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - technologic - <p>test</p> - unlisted - - - - diff --git a/test/fixtures/mastodon-problematic.xml b/test/fixtures/mastodon-problematic.xml deleted file mode 100644 index a39e72759..000000000 --- a/test/fixtures/mastodon-problematic.xml +++ /dev/null @@ -1,72 +0,0 @@ - - - https://icosahedron.website/users/shel.atom - shel🍖‼️ - Gay jackal dog, poet, future librarian. - -http://datapup.info -avatar: @puppytube@twitter.com - 2017-05-02T23:26:01Z - https://icosahedron.website/system/accounts/avatars/000/001/207/original/b1e07b09ae1cc787.png?1493767561 - - https://icosahedron.website/users/shel - http://activitystrea.ms/schema/1.0/person - https://icosahedron.website/users/shel - shel - shel@icosahedron.website - <p>Gay jackal dog, poet, future librarian. </p><p><a href="http://datapup.info/" rel="nofollow noopener" target="_blank"><span class="invisible">http://</span><span class="">datapup.info/</span><span class="invisible"></span></a><br />avatar: @puppytube@twitter.com</p> - - - - shel - shel🍖‼️ - Gay jackal dog, poet, future librarian. - -http://datapup.info -avatar: @puppytube@twitter.com - public - - - - - - - tag:icosahedron.website,2017-05-10:objectId=1414013:objectType=Status - 2017-05-10T17:16:24Z - 2017-05-10T17:16:24Z - shel shared a status by instance_names@cybre.space - http://activitystrea.ms/schema/1.0/activity - http://activitystrea.ms/schema/1.0/share - - tag:cybre.space,2017-05-10:objectId=946671:objectType=Status - 2017-05-10T17:15:51Z - 2017-05-10T17:15:52Z - New status by instance_names@cybre.space - - https://cybre.space/users/instance_names - http://activitystrea.ms/schema/1.0/person - https://cybre.space/users/instance_names - instance_names - instance_names@cybre.space - <p>name ideas for your new mastodon instance. made by <span class="h-card"><a href="https://witches.town/@lycaon">@<span>lycaon</span></a></span> source available at <a href="https://github.com/LycaonIsAWolf/instance_names"><span class="invisible">https://</span><span class="ellipsis">github.com/LycaonIsAWolf/insta</span><span class="invisible">nce_names</span></a></p> - - - - instance_names - instance names - name ideas for your new mastodon instance. made by @lycaon source available at https://github.com/LycaonIsAWolf/instance_names - public - - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - <p>dildo.codes</p> - unlisted - - - <p>dildo.codes</p> - - public - - - - diff --git a/test/fixtures/mastodon_conversation.xml b/test/fixtures/mastodon_conversation.xml deleted file mode 100644 index 8faab2304..000000000 --- a/test/fixtures/mastodon_conversation.xml +++ /dev/null @@ -1,30 +0,0 @@ - - - tag:mastodon.social,2017-08-28:objectId=16402826:objectType=Status - 2017-08-28T17:58:55Z - 2017-08-28T17:58:55Z - New status by lambadalambda - - https://mastodon.social/users/lambadalambda - http://activitystrea.ms/schema/1.0/person - https://mastodon.social/users/lambadalambda - lambadalambda - lambadalambda@mastodon.social - - - - lambadalambda - Critical Value - public - - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - - <p>test. <a href="https://mastodon.social/media/XCp0OHGPON9kWZwhjaI" rel="nofollow noopener" target="_blank"><span class="invisible">https://</span><span class="ellipsis">mastodon.social/media/XCp0OHGP</span><span class="invisible">ON9kWZwhjaI</span></a></p> - - - public - - - - diff --git a/test/fixtures/nil_mention_entry.xml b/test/fixtures/nil_mention_entry.xml deleted file mode 100644 index e13024cb3..000000000 --- a/test/fixtures/nil_mention_entry.xml +++ /dev/null @@ -1,52 +0,0 @@ - - - GNU social - https://social.stopwatchingus-heidelberg.de/api/statuses/user_timeline/18330.atom - atarifrosch timeline - Updates from atarifrosch on social.stopwatchingus-heidelberg.de! - https://social.stopwatchingus-heidelberg.de/avatar/18330-96-20150628163706.png - 2017-08-24T11:36:49+02:00 - - http://activitystrea.ms/schema/1.0/person - https://social.stopwatchingus-heidelberg.de/user/18330 - atarifrosch - Nerd, Pirat, Debian user, CAcert assurer, Geocacher, Freifunker. Autismus/Depression, agender. GnuPG Key-ID: 0xBCF81ADE - - - - - - atarifrosch - Atari-Frosch - Nerd, Pirat, Debian user, CAcert assurer, Geocacher, Freifunker. Autismus/Depression, agender. GnuPG Key-ID: 0xBCF81ADE - - Düsseldorf, NRW, Germany - - - homepage - https://www.atari-frosch.de/ - true - - - - - - http://activitystrea.ms/schema/1.0/note - tag:social.stopwatchingus-heidelberg.de,2017-08-22:noticeId=978072:objectType=note - New note by atarifrosch - 2017-08-22 Bundesverfassungsgericht: Erfolgreiche Verfassungsbeschwerde gegen die Versagung vorläufiger Leistungen für Kosten der Unterkunft und Heizung – <a href="https://www.bundesverfassungsgericht.de/SharedDocs/Pressemitteilungen/DE/2017/bvg17-072.html" title="https://www.bundesverfassungsgericht.de/SharedDocs/Pressemitteilungen/DE/2017/bvg17-072.html" class="attachment" id="attachment-450768" rel="nofollow external">https://www.bundesverfassungsgericht.de/SharedDocs/Pressemitteilungen/DE/2017/bvg17-072.html</a> !<a href="http://quitter.se/group/2184/id" class="h-card group" title="HartzIV (hartziv)">hartziv</a> - - - http://activitystrea.ms/schema/1.0/post - 2017-08-22T12:00:21+00:00 - 2017-08-22T12:00:21+00:00 - - tag:social.stopwatchingus-heidelberg.de,2017-08-22:noticeId=978072:objectType=thread:crc32=28a35f44 - - - - - - - - diff --git a/test/fixtures/ostatus_incoming_post.xml b/test/fixtures/ostatus_incoming_post.xml deleted file mode 100644 index 7967e1b32..000000000 --- a/test/fixtures/ostatus_incoming_post.xml +++ /dev/null @@ -1,57 +0,0 @@ - - - GNU social - https://social.heldscal.la/api/statuses/user_timeline/23211.atom - lambadalambda timeline - Updates from lambadalambda on social.heldscal.la! - https://social.heldscal.la/avatar/23211-96-20170416114255.jpeg - 2017-04-29T18:25:38+00:00 - - http://activitystrea.ms/schema/1.0/person - https://social.heldscal.la/user/23211 - lambadalambda - Call me Deacon Blues. - - - - - - lambadalambda - Constance Variable - Call me Deacon Blues. - - Berlin - - - homepage - https://heldscal.la - true - - - - - - - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:social.heldscal.la,2017-04-29:noticeId=1967725:objectType=note - New note by lambadalambda - Will it blend? - - - http://activitystrea.ms/schema/1.0/post - 2017-04-29T18:25:38+00:00 - 2017-04-29T18:25:38+00:00 - - tag:social.heldscal.la,2017-04-29:objectType=thread:nonce=3f3a9dd83acc4e35 - - - - - - diff --git a/test/fixtures/ostatus_incoming_post_tag.xml b/test/fixtures/ostatus_incoming_post_tag.xml deleted file mode 100644 index 0f99c4126..000000000 --- a/test/fixtures/ostatus_incoming_post_tag.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - GNU social - https://social.heldscal.la/api/statuses/user_timeline/23211.atom - lambadalambda timeline - Updates from lambadalambda on social.heldscal.la! - https://social.heldscal.la/avatar/23211-96-20170416114255.jpeg - 2017-04-29T18:25:38+00:00 - - http://activitystrea.ms/schema/1.0/person - https://social.heldscal.la/user/23211 - lambadalambda - Call me Deacon Blues. - - - - - - lambadalambda - Constance Variable - Call me Deacon Blues. - - Berlin - - - homepage - https://heldscal.la - true - - - - - - - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:social.heldscal.la,2017-04-29:noticeId=1967725:objectType=note - New note by lambadalambda - Will it blend? - - - - - http://activitystrea.ms/schema/1.0/post - 2017-04-29T18:25:38+00:00 - 2017-04-29T18:25:38+00:00 - - tag:social.heldscal.la,2017-04-29:objectType=thread:nonce=3f3a9dd83acc4e35 - - - - - - diff --git a/test/fixtures/ostatus_incoming_reply.xml b/test/fixtures/ostatus_incoming_reply.xml deleted file mode 100644 index 83a427a68..000000000 --- a/test/fixtures/ostatus_incoming_reply.xml +++ /dev/null @@ -1,60 +0,0 @@ - - - GNU social - https://social.heldscal.la/api/statuses/user_timeline/23211.atom - lambadalambda timeline - Updates from lambadalambda on social.heldscal.la! - https://social.heldscal.la/avatar/23211-96-20170416114255.jpeg - 2017-04-30T09:30:32+00:00 - - http://activitystrea.ms/schema/1.0/person - https://social.heldscal.la/user/23211 - lambadalambda - Call me Deacon Blues. - - - - - - lambadalambda - Constance Variable - Call me Deacon Blues. - - Berlin - - - homepage - https://heldscal.la - true - - - - - - - - - - - - - http://activitystrea.ms/schema/1.0/comment - tag:social.heldscal.la,2017-04-30:noticeId=1978790:objectType=comment - New comment by lambadalambda - @<a href="https://gs.archae.me/user/4687" class="h-card u-url p-nickname mention" title="shpbot">shpbot</a> why not indeed. - - - http://activitystrea.ms/schema/1.0/post - 2017-04-30T09:30:32+00:00 - 2017-04-30T09:30:32+00:00 - - - - https://gs.archae.me/conversation/327120 - - - - - - - diff --git a/test/fixtures/share-gs-local.xml b/test/fixtures/share-gs-local.xml deleted file mode 100644 index 9d52eab7b..000000000 --- a/test/fixtures/share-gs-local.xml +++ /dev/null @@ -1,99 +0,0 @@ - - - GNU social - https://social.heldscal.la/api/statuses/user_timeline/23211.atom - lambadalambda timeline - Updates from lambadalambda on social.heldscal.la! - https://social.heldscal.la/avatar/23211-96-20170416114255.jpeg - 2017-05-03T08:05:41+00:00 - - http://activitystrea.ms/schema/1.0/person - https://social.heldscal.la/user/23211 - lambadalambda - Call me Deacon Blues. - - - - - - lambadalambda - Constance Variable - Call me Deacon Blues. - - Berlin - - - homepage - https://heldscal.la - true - - - - - - - - - - - - - tag:social.heldscal.la,2017-05-03:noticeId=2028428:objectType=note - lambadalambda repeated a notice by lain - RT @<a href="https://pleroma.soykaf.com/users/lain" class="h-card u-url p-nickname mention" title="Lain Iwakura">lain</a> Added returning the entries as xml... let's see if the mastodon hammering stops now. - - http://activitystrea.ms/schema/1.0/share - 2017-05-03T08:05:41+00:00 - 2017-05-03T08:05:41+00:00 - - http://activitystrea.ms/schema/1.0/activity - LOCAL_ID - - Added returning the entries as xml... let's see if the mastodon hammering stops now. - - http://activitystrea.ms/schema/1.0/post - 2017-05-03T08:04:44+00:00 - 2017-05-03T08:04:44+00:00 - - http://activitystrea.ms/schema/1.0/person - LOCAL_USER - lain - Test account - - - - - - lain - Lain Iwakura - Test account - - - - http://activitystrea.ms/schema/1.0/note - https://pleroma.soykaf.com/objects/4c1bda26-902e-4525-9fcd-b9fd44925193 - New note by lain - Added returning the entries as xml... let's see if the mastodon hammering stops now. - - - - - https://pleroma.soykaf.com/contexts/ede39a2b-7cf3-4fa4-8ccd-cb97431bcc22 - - - https://pleroma.soykaf.com/users/lain/feed.atom - Lain Iwakura - - - https://social.heldscal.la/avatar/43188-96-20170429172422.jpeg - 2017-05-03T08:04:44+00:00 - - - - https://pleroma.soykaf.com/contexts/ede39a2b-7cf3-4fa4-8ccd-cb97431bcc22 - - - - - - diff --git a/test/fixtures/share-gs.xml b/test/fixtures/share-gs.xml deleted file mode 100644 index ab5e488bd..000000000 --- a/test/fixtures/share-gs.xml +++ /dev/null @@ -1,99 +0,0 @@ - - - GNU social - https://social.heldscal.la/api/statuses/user_timeline/23211.atom - lambadalambda timeline - Updates from lambadalambda on social.heldscal.la! - https://social.heldscal.la/avatar/23211-96-20170416114255.jpeg - 2017-05-03T08:05:41+00:00 - - http://activitystrea.ms/schema/1.0/person - https://social.heldscal.la/user/23211 - lambadalambda - Call me Deacon Blues. - - - - - - lambadalambda - Constance Variable - Call me Deacon Blues. - - Berlin - - - homepage - https://heldscal.la - true - - - - - - - - - - - - - tag:social.heldscal.la,2017-05-03:noticeId=2028428:objectType=note - lambadalambda repeated a notice by lain - RT @<a href="https://pleroma.soykaf.com/users/lain" class="h-card u-url p-nickname mention" title="Lain Iwakura">lain</a> Added returning the entries as xml... let's see if the mastodon hammering stops now. - - http://activitystrea.ms/schema/1.0/share - 2017-05-03T08:05:41+00:00 - 2017-05-03T08:05:41+00:00 - - http://activitystrea.ms/schema/1.0/activity - https://pleroma.soykaf.com/objects/4c1bda26-902e-4525-9fcd-b9fd44925193 - - Added returning the entries as xml... let's see if the mastodon hammering stops now. - - http://activitystrea.ms/schema/1.0/post - 2017-05-03T08:04:44+00:00 - 2017-05-03T08:04:44+00:00 - - http://activitystrea.ms/schema/1.0/person - https://pleroma.soykaf.com/users/lain - lain - Test account - - - - - - lain - Lain Iwakura - Test account - - - - http://activitystrea.ms/schema/1.0/note - https://pleroma.soykaf.com/objects/4c1bda26-902e-4525-9fcd-b9fd44925193 - New note by lain - Added returning the entries as xml... let's see if the mastodon hammering stops now. - - - - - https://pleroma.soykaf.com/contexts/ede39a2b-7cf3-4fa4-8ccd-cb97431bcc22 - - - https://pleroma.soykaf.com/users/lain/feed.atom - Lain Iwakura - - - https://social.heldscal.la/avatar/43188-96-20170429172422.jpeg - 2017-05-03T08:04:44+00:00 - - - - https://pleroma.soykaf.com/contexts/ede39a2b-7cf3-4fa4-8ccd-cb97431bcc22 - - - - - - diff --git a/test/fixtures/share.xml b/test/fixtures/share.xml deleted file mode 100644 index e07b88680..000000000 --- a/test/fixtures/share.xml +++ /dev/null @@ -1,54 +0,0 @@ - - - tag:mastodon.social,2017-05-03:objectId=4934452:objectType=Status - 2017-05-03T08:21:09Z - 2017-05-03T08:21:09Z - lambadalambda shared a status by lain@pleroma.soykaf.com - - https://mastodon.social/users/lambadalambda - http://activitystrea.ms/schema/1.0/person - https://mastodon.social/users/lambadalambda - lambadalambda - lambadalambda@mastodon.social - - - - lambadalambda - Critical Value - public - - http://activitystrea.ms/schema/1.0/activity - http://activitystrea.ms/schema/1.0/share - - https://pleroma.soykaf.com/objects/4c1bda26-902e-4525-9fcd-b9fd44925193 - 2017-05-03T08:04:44Z - 2017-05-03T08:05:52Z - New status by lain@pleroma.soykaf.com - - https://pleroma.soykaf.com/users/lain - http://activitystrea.ms/schema/1.0/person - https://pleroma.soykaf.com/users/lain - lain - lain@pleroma.soykaf.com - Test account - - - - lain - Lain Iwakura - Test account - public - - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - Added returning the entries as xml... let's see if the mastodon hammering stops now. - - public - - - Added returning the entries as xml... let's see if the mastodon hammering stops now. - - public - - - diff --git a/test/fixtures/tesla_mock/7369654.atom b/test/fixtures/tesla_mock/7369654.atom deleted file mode 100644 index 74fd9ce6b..000000000 --- a/test/fixtures/tesla_mock/7369654.atom +++ /dev/null @@ -1,44 +0,0 @@ - - - http://activitystrea.ms/schema/1.0/comment - tag:shitposter.club,2018-02-22:noticeId=7369654:objectType=comment - New comment by shpuld - @<a href="https://testing.pleroma.lol/users/lain" class="h-card mention" title="Rael Electric Razor">lain</a> me far right - - - http://activitystrea.ms/schema/1.0/post - 2018-02-22T09:20:12+00:00 - 2018-02-22T09:20:12+00:00 - - http://activitystrea.ms/schema/1.0/person - https://shitposter.club/user/5381 - shpuld - - - - - - shpuld - shp - - - - - - - tag:shitposter.club,2018-02-22:objectType=thread:nonce=e5a7c72d60a9c0e4 - - - - https://shitposter.club/api/statuses/user_timeline/5381.atom - shp - - - - https://shitposter.club/avatar/5381-96-20171230093854.png - 2018-02-23T13:30:15+00:00 - - - - - diff --git a/test/fixtures/tesla_mock/atarifrosch_feed.xml b/test/fixtures/tesla_mock/atarifrosch_feed.xml deleted file mode 100644 index e00df782e..000000000 --- a/test/fixtures/tesla_mock/atarifrosch_feed.xml +++ /dev/null @@ -1,473 +0,0 @@ - - - GNU social - https://social.stopwatchingus-heidelberg.de/api/statuses/user_timeline/18330.atom - atarifrosch-Zeitleiste - Aktualisierungen von atarifrosch auf social.stopwatchingus-heidelberg.de! - https://social.stopwatchingus-heidelberg.de/avatar/18330-96-20150628163706.png - 2017-08-24T12:06:55+02:00 - - http://activitystrea.ms/schema/1.0/person - https://social.stopwatchingus-heidelberg.de/user/18330 - atarifrosch - Nerd, Pirat, Debian user, CAcert assurer, Geocacher, Freifunker. Autismus/Depression, agender. GnuPG Key-ID: 0xBCF81ADE - - - - - - atarifrosch - Atari-Frosch - Nerd, Pirat, Debian user, CAcert assurer, Geocacher, Freifunker. Autismus/Depression, agender. GnuPG Key-ID: 0xBCF81ADE - - Düsseldorf, NRW, Germany - - - homepage - https://www.atari-frosch.de/ - true - - - - - - - - - - - - - - tag:social.stopwatchingus-heidelberg.de,2017-08-24:noticeId=978735:objectType=note - atarifrosch repeated a notice by hoergen - RT @<a href="https://social.hoergen.org/hoergen" class="h-card mention" title="hoergen">hoergen</a> Das falsche Bild der Tagesschau &quot;Auffallend &quot;erfolgreich&quot; - Andrea Nahles und Manuela Schwesig&quot; #<span class="tag"><a href="https://social.stopwatchingus-heidelberg.de/tag/geringverdiener" rel="tag">Geringverdiener</a></span> #<span class="tag"><a href="https://social.stopwatchingus-heidelberg.de/tag/mindestlohn" rel="tag">Mindestlohn</a></span> #<span class="tag"><a href="https://social.stopwatchingus-heidelberg.de/tag/mannxismus" rel="tag">mannxismus</a></span> #<span class="tag"><a href="https://social.stopwatchingus-heidelberg.de/tag/erwerbsminderungsrente" rel="tag">Erwerbsminderungsrente</a></span> #<span class="tag"><a href="https://social.stopwatchingus-heidelberg.de/tag/arbeitnehmerflexibilisierung" rel="tag">ArbeitnehmerFlexibilisierung</a></span> #<span class="tag"><a href="https://social.stopwatchingus-heidelberg.de/tag/altersarmut" rel="tag">AltersArmut</a></span> ..... <a href="http://www.tagesschau.de/inland/btw17/bilanz-schwesig-nahles-101.html" title="http://www.tagesschau.de/inland/btw17/bilanz-schwesig-nahles-101.html" class="attachment" id="attachment-450858" rel="nofollow external">http://www.tagesschau.de/inland/btw17/bilanz-schwesig-nahles-101.html</a> - https://social.stopwatchingus-heidelberg.de/notice/978735 - http://activitystrea.ms/schema/1.0/share - 2017-08-24T09:18:25+00:00 - 2017-08-24T09:18:25+00:00 - - http://activitystrea.ms/schema/1.0/activity - tag:social.hoergen.org,2017-08-24:noticeId=222320:objectType=note - - Das falsche Bild der Tagesschau <br /> &quot;Auffallend &quot;erfolgreich&quot; - Andrea Nahles und Manuela Schwesig&quot; #<span class="tag"><a href="https://social.hoergen.org/tag/geringverdiener" rel="tag">Geringverdiener</a></span> #<span class="tag"><a href="https://social.hoergen.org/tag/mindestlohn" rel="tag">Mindestlohn</a></span> #<span class="tag"><a href="https://social.hoergen.org/tag/mannxismus" rel="tag">mannxismus</a></span> #<span class="tag"><a href="https://social.hoergen.org/tag/erwerbsminderungsrente" rel="tag">Erwerbsminderungsrente</a></span> #<span class="tag"><a href="https://social.hoergen.org/tag/arbeitnehmerflexibilisierung" rel="tag">ArbeitnehmerFlexibilisierung</a></span> #<span class="tag"><a href="https://social.hoergen.org/tag/altersarmut" rel="tag">AltersArmut</a></span> ..... <br /> <br /> <a href="http://www.tagesschau.de/inland/btw17/bilanz-schwesig-nahles-101.html" title="http://www.tagesschau.de/inland/btw17/bilanz-schwesig-nahles-101.html" rel="nofollow external noreferrer" class="attachment">http://www.tagesschau.de/inland/btw17/bilanz-schwesig-nahles-101.html</a> - https://social.hoergen.org/notice/222320 - http://activitystrea.ms/schema/1.0/post - 2017-08-24T07:36:31+00:00 - 2017-08-24T07:36:31+00:00 - - http://activitystrea.ms/schema/1.0/person - https://social.hoergen.org/user/2 - hoergen - aka Andi Memyself #humanist #nerd Menschen liebhabender Misanthrop und auch sonst sehr vielseitig interessiert. - - - - - - hoergen - hoergen - aka Andi Memyself #humanist #nerd Menschen liebhabender Misanthrop und auch sonst sehr vielseitig interessiert. - - Berlin - - - homepage - https://hyperblog.de/hoergen/ - true - - - - - http://activitystrea.ms/schema/1.0/note - tag:social.hoergen.org,2017-08-24:noticeId=222320:objectType=note - New note by hoergen - Das falsche Bild der Tagesschau <br /> &quot;Auffallend &quot;erfolgreich&quot; - Andrea Nahles und Manuela Schwesig&quot; #<span class="tag"><a href="https://social.hoergen.org/tag/geringverdiener" rel="tag">Geringverdiener</a></span> #<span class="tag"><a href="https://social.hoergen.org/tag/mindestlohn" rel="tag">Mindestlohn</a></span> #<span class="tag"><a href="https://social.hoergen.org/tag/mannxismus" rel="tag">mannxismus</a></span> #<span class="tag"><a href="https://social.hoergen.org/tag/erwerbsminderungsrente" rel="tag">Erwerbsminderungsrente</a></span> #<span class="tag"><a href="https://social.hoergen.org/tag/arbeitnehmerflexibilisierung" rel="tag">ArbeitnehmerFlexibilisierung</a></span> #<span class="tag"><a href="https://social.hoergen.org/tag/altersarmut" rel="tag">AltersArmut</a></span> ..... <br /> <br /> <a href="http://www.tagesschau.de/inland/btw17/bilanz-schwesig-nahles-101.html" title="http://www.tagesschau.de/inland/btw17/bilanz-schwesig-nahles-101.html" rel="nofollow external noreferrer" class="attachment">http://www.tagesschau.de/inland/btw17/bilanz-schwesig-nahles-101.html</a> - - - - - https://social.hoergen.org/conversation/98616 - - - - - - - - - https://social.hoergen.org/api/statuses/user_timeline/2.atom - hoergen - - - https://social.stopwatchingus-heidelberg.de/avatar/54316-original-20170824072526.jpeg - 2017-08-24T09:48:30+00:00 - - - - https://social.hoergen.org/conversation/98616 - - - - - - - - - - - - - http://activitystrea.ms/schema/1.0/comment - tag:social.stopwatchingus-heidelberg.de,2017-08-24:noticeId=978734:objectType=comment - New comment by atarifrosch - Jo, die Anzahl der Hartz-IV-Sanktionen nennt sie genausowenig wie die Anzahl der Menschen, die von den Repressionsbehörden in Obdachlosigkeit und Suizid getrieben wurden. Das würde die Erfolgszahlen dann doch ein wenig trüben, nech? - - - http://activitystrea.ms/schema/1.0/post - 2017-08-24T09:18:13+00:00 - 2017-08-24T09:18:13+00:00 - - - - https://social.hoergen.org/conversation/98616 - - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:social.stopwatchingus-heidelberg.de,2017-08-24:noticeId=978732:objectType=note - New note by atarifrosch - Moin-quak. - - - http://activitystrea.ms/schema/1.0/post - 2017-08-24T09:09:39+00:00 - 2017-08-24T09:09:39+00:00 - - tag:social.stopwatchingus-heidelberg.de,2017-08-24:noticeId=978732:objectType=thread:crc32=2f92b7b6 - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:social.stopwatchingus-heidelberg.de,2017-08-23:noticeId=978594:objectType=note - New note by atarifrosch - n8-quak! - - - http://activitystrea.ms/schema/1.0/post - 2017-08-23T21:39:54+00:00 - 2017-08-23T21:39:54+00:00 - - tag:social.stopwatchingus-heidelberg.de,2017-08-23:noticeId=978594:objectType=thread:crc32=9bdb0ac9 - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:social.stopwatchingus-heidelberg.de,2017-08-23:noticeId=978503:objectType=note - New note by atarifrosch - 2017-08-16 Michal Špaček: Post a boarding pass on Facebook, get your account stolen – Post a boarding pass on Facebook, get your account stolen (gilt übrinx nicht nur für Facebook) - - - http://activitystrea.ms/schema/1.0/post - 2017-08-23T15:14:29+00:00 - 2017-08-23T15:14:29+00:00 - - tag:social.stopwatchingus-heidelberg.de,2017-08-23:noticeId=978503:objectType=thread:crc32=3de05c3a - - - - - - - tag:social.stopwatchingus-heidelberg.de,2017-08-23:fave:18330:activity:978458:2017-08-23T15:18:19+02:00 - Favorite - atarifrosch favorited something by einebiene: Haha, große Überraschung. <a href="http://www.sueddeutsche.de/wirtschaft/abgasaffaere-software-updates-fuer-dieselautos-helfen-kaum-1.3637636" title="http://www.sueddeutsche.de/wirtschaft/abgasaffaere-software-updates-fuer-dieselautos-helfen-kaum-1.3637636" rel="nofollow noreferrer" class="attachment">https://quitter.is/url/1122672</a><br /> Was ich an all diesen Artikeln schade finde, ist, daß immer nur auf den Umstieg von Auto zu anderem Auto gesprochen wird. Öffis werden nicht erwähnt, Carsharing nicht, radeln nicht, und in der Stadt wäre ne Vespa auch deutlich besser als ein SUV. - http://activitystrea.ms/schema/1.0/favorite - 2017-08-23T13:18:19+00:00 - 2017-08-23T13:18:19+00:00 - - http://activitystrea.ms/schema/1.0/note - tag:quitter.is,2017-08-23:noticeId=4032910:objectType=note - New note by einebiene - Haha, große Überraschung. <a href="http://www.sueddeutsche.de/wirtschaft/abgasaffaere-software-updates-fuer-dieselautos-helfen-kaum-1.3637636" title="http://www.sueddeutsche.de/wirtschaft/abgasaffaere-software-updates-fuer-dieselautos-helfen-kaum-1.3637636" rel="nofollow noreferrer" class="attachment">https://quitter.is/url/1122672</a><br /> Was ich an all diesen Artikeln schade finde, ist, daß immer nur auf den Umstieg von Auto zu anderem Auto gesprochen wird. Öffis werden nicht erwähnt, Carsharing nicht, radeln nicht, und in der Stadt wäre ne Vespa auch deutlich besser als ein SUV. - - - - - - - https://quitter.is/conversation/2535246 - - - - - - http://activitystrea.ms/schema/1.0/note - tag:social.stopwatchingus-heidelberg.de,2017-08-23:noticeId=978402:objectType=note - New note by atarifrosch - moin-quak - - - http://activitystrea.ms/schema/1.0/post - 2017-08-23T10:57:26+00:00 - 2017-08-23T10:57:26+00:00 - - tag:social.stopwatchingus-heidelberg.de,2017-08-23:noticeId=978402:objectType=thread:crc32=7050c397 - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:social.stopwatchingus-heidelberg.de,2017-08-22:noticeId=978164:objectType=note - New note by atarifrosch - n8-quak - - - http://activitystrea.ms/schema/1.0/post - 2017-08-22T19:54:30+00:00 - 2017-08-22T19:54:30+00:00 - - tag:social.stopwatchingus-heidelberg.de,2017-08-22:noticeId=978164:objectType=thread:crc32=b0a209c7 - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:social.stopwatchingus-heidelberg.de,2017-08-22:noticeId=978072:objectType=note - New note by atarifrosch - 2017-08-22 Bundesverfassungsgericht: Erfolgreiche Verfassungsbeschwerde gegen die Versagung vorläufiger Leistungen für Kosten der Unterkunft und Heizung – <a href="https://www.bundesverfassungsgericht.de/SharedDocs/Pressemitteilungen/DE/2017/bvg17-072.html" title="https://www.bundesverfassungsgericht.de/SharedDocs/Pressemitteilungen/DE/2017/bvg17-072.html" class="attachment" id="attachment-450768" rel="nofollow external">https://www.bundesverfassungsgericht.de/SharedDocs/Pressemitteilungen/DE/2017/bvg17-072.html</a> !<a href="http://quitter.se/group/2184/id" class="h-card group" title="HartzIV (hartziv)">hartziv</a> - - - http://activitystrea.ms/schema/1.0/post - 2017-08-22T12:00:21+00:00 - 2017-08-22T12:00:21+00:00 - - tag:social.stopwatchingus-heidelberg.de,2017-08-22:noticeId=978072:objectType=thread:crc32=28a35f44 - - - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:social.stopwatchingus-heidelberg.de,2017-08-22:noticeId=978042:objectType=note - New note by atarifrosch - moin-quak! - - - http://activitystrea.ms/schema/1.0/post - 2017-08-22T07:55:27+00:00 - 2017-08-22T07:55:27+00:00 - - tag:social.stopwatchingus-heidelberg.de,2017-08-22:noticeId=978042:objectType=thread:crc32=f070a9f7 - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:social.stopwatchingus-heidelberg.de,2017-08-21:noticeId=977914:objectType=note - New note by atarifrosch - So, morgen geht's weiter. n8-quak! - - - http://activitystrea.ms/schema/1.0/post - 2017-08-21T22:09:53+00:00 - 2017-08-21T22:09:53+00:00 - - tag:social.stopwatchingus-heidelberg.de,2017-08-21:noticeId=977914:objectType=thread:crc32=c0a9f7fa - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:social.stopwatchingus-heidelberg.de,2017-08-21:noticeId=977710:objectType=note - New note by atarifrosch - moin-quak. - - - http://activitystrea.ms/schema/1.0/post - 2017-08-21T08:58:26+00:00 - 2017-08-21T08:58:26+00:00 - - tag:social.stopwatchingus-heidelberg.de,2017-08-21:noticeId=977710:objectType=thread:crc32=60cfb466 - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:social.stopwatchingus-heidelberg.de,2017-08-20:noticeId=977526:objectType=note - New note by atarifrosch - Meine Augen meinen, für heute sei es genug. Nun denn. n8-quak. - - - http://activitystrea.ms/schema/1.0/post - 2017-08-20T19:58:16+00:00 - 2017-08-20T19:58:16+00:00 - - tag:social.stopwatchingus-heidelberg.de,2017-08-20:noticeId=977526:objectType=thread:crc32=ce79634 - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:social.stopwatchingus-heidelberg.de,2017-08-20:noticeId=977369:objectType=note - New note by atarifrosch - [Blog] Im Netz aufgefischt #<span class="tag"><a href="https://social.stopwatchingus-heidelberg.de/tag/330" rel="tag">330</a></span> – <a href="https://blog.atari-frosch.de/2017/08/20/im-netz-aufgefischt-330/" title="https://blog.atari-frosch.de/2017/08/20/im-netz-aufgefischt-330/" class="attachment" id="attachment-450668" rel="nofollow external">https://blog.atari-frosch.de/2017/08/20/im-netz-aufgefischt-330/</a> (was ich diese Woche so gelesen habe) - - - http://activitystrea.ms/schema/1.0/post - 2017-08-20T09:14:07+00:00 - 2017-08-20T09:14:07+00:00 - - tag:social.stopwatchingus-heidelberg.de,2017-08-20:noticeId=977369:objectType=thread:crc32=2f800b86 - - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:social.stopwatchingus-heidelberg.de,2017-08-19:noticeId=977268:objectType=note - New note by atarifrosch - Fast ständig husten müssen ist echt anstrengend … naja, n8-quak. - - - http://activitystrea.ms/schema/1.0/post - 2017-08-19T21:59:20+00:00 - 2017-08-19T21:59:20+00:00 - - tag:social.stopwatchingus-heidelberg.de,2017-08-19:noticeId=977268:objectType=thread:crc32=deda767a - - - - - - - tag:social.stopwatchingus-heidelberg.de,2017-08-19:fave:18330:activity:977146:2017-08-19T21:39:26+02:00 - Favorite - atarifrosch favorited something by einebienezwo: Ich mach gerade Kompetenztraining.<br /> Ich trainiere die Kompetenz, eine halb aufgegessene Gummibärchentüte nicht ganz aufzuessen. - http://activitystrea.ms/schema/1.0/favorite - 2017-08-19T19:39:26+00:00 - 2017-08-19T19:39:26+00:00 - - http://activitystrea.ms/schema/1.0/note - tag:gnusocial.de,2017-08-19:noticeId=11011264:objectType=note - New note by einebienezwo - Ich mach gerade Kompetenztraining.<br /> Ich trainiere die Kompetenz, eine halb aufgegessene Gummibärchentüte nicht ganz aufzuessen. - - - - - - - https://gnusocial.de/conversation/9363945 - - - - - - http://activitystrea.ms/schema/1.0/comment - tag:social.stopwatchingus-heidelberg.de,2017-08-19:noticeId=977242:objectType=comment - New comment by atarifrosch - Wir hatten hier schon Ordnungsdienst auf'm Radweg. Fotografisch dokumentiert (nicht von mir, Bekannter hatte es gesehen). Da hatte grad 'ne Pizzeria neu eröffnet … - - - http://activitystrea.ms/schema/1.0/post - 2017-08-19T19:38:53+00:00 - 2017-08-19T19:38:53+00:00 - - - - https://gnusocial.de/conversation/9363813 - - - - - - - - tag:social.stopwatchingus-heidelberg.de,2017-08-19:fave:18330:activity:977180:2017-08-19T21:37:36+02:00 - Favorite - atarifrosch favorited something by jcaktiv: BTW Hallo zusammen &lt;3, wo ich schon mal wieder hier bin - http://activitystrea.ms/schema/1.0/favorite - 2017-08-19T19:37:36+00:00 - 2017-08-19T19:37:36+00:00 - - http://activitystrea.ms/schema/1.0/note - tag:quitter.se,2017-08-19:noticeId=17372467:objectType=note - New note by jcaktiv - BTW Hallo zusammen &lt;3, wo ich schon mal wieder hier bin - - - - - - - tag:quitter.se,2017-08-19:objectType=thread:nonce=46c1c433d88aaa9f - - - - - - http://activitystrea.ms/schema/1.0/comment - tag:social.stopwatchingus-heidelberg.de,2017-08-19:noticeId=976985:objectType=comment - New comment by atarifrosch - Jo, oder einfach mal nachfragen, so als Realitätsabgleich. - - - http://activitystrea.ms/schema/1.0/post - 2017-08-19T10:34:50+00:00 - 2017-08-19T10:34:50+00:00 - - - - https://gnusocial.de/conversation/9362516 - - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:social.stopwatchingus-heidelberg.de,2017-08-19:noticeId=976983:objectType=note - New note by atarifrosch - Schöne Alternative zu mit Werbung überladenen kommerziellen Anbietern: <a href="http://ifconfig.at/" title="http://ifconfig.at/" class="attachment" id="attachment-450636" rel="nofollow external">http://ifconfig.at/</a> – eigene IP, Hostname etc. abfragen, mit curl dann auch in Textform zur lokalen Weiterverarbeitung in Scripten etc. Leider (noch?) kein https. - - - http://activitystrea.ms/schema/1.0/post - 2017-08-19T10:33:04+00:00 - 2017-08-19T10:33:04+00:00 - - tag:social.stopwatchingus-heidelberg.de,2017-08-19:noticeId=976983:objectType=thread:crc32=4a3593c0 - - - - - - diff --git a/test/fixtures/tesla_mock/emelie.atom b/test/fixtures/tesla_mock/emelie.atom deleted file mode 100644 index ddaa1c6ca..000000000 --- a/test/fixtures/tesla_mock/emelie.atom +++ /dev/null @@ -1,306 +0,0 @@ - - - https://mastodon.social/users/emelie.atom - emelie 🎨 - 23 / #Sweden / #Artist / #Equestrian / #GameDev - -If I ain't spending time with my pets, I'm probably drawing. 🐴 🐱 🐰 - 2019-02-04T20:22:19Z - https://files.mastodon.social/accounts/avatars/000/015/657/original/e7163f98280da1a4.png - - https://mastodon.social/users/emelie - http://activitystrea.ms/schema/1.0/person - https://mastodon.social/users/emelie - emelie - emelie@mastodon.social - <p>23 / <a href="https://mastodon.social/tags/sweden" class="mention hashtag" rel="tag">#<span>Sweden</span></a> / <a href="https://mastodon.social/tags/artist" class="mention hashtag" rel="tag">#<span>Artist</span></a> / <a href="https://mastodon.social/tags/equestrian" class="mention hashtag" rel="tag">#<span>Equestrian</span></a> / <a href="https://mastodon.social/tags/gamedev" class="mention hashtag" rel="tag">#<span>GameDev</span></a></p><p>If I ain&apos;t spending time with my pets, I&apos;m probably drawing. 🐴 🐱 🐰</p> - - - - emelie - emelie 🎨 - 23 / #Sweden / #Artist / #Equestrian / #GameDev - -If I ain't spending time with my pets, I'm probably drawing. 🐴 🐱 🐰 - public - - - - - - - https://mastodon.social/users/emelie/statuses/101850331907006641 - 2019-04-01T09:58:50Z - 2019-04-01T09:58:50Z - New status by emelie - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - - <p>Me: I&apos;m going to make this vital change to my world building in the morning, no way I&apos;ll forget this, it&apos;s too big of a deal<br />Also me: forgets</p> - - public - - - - - - https://mastodon.social/users/emelie/statuses/101849626603073336 - 2019-04-01T06:59:28Z - 2019-04-01T06:59:28Z - New status by emelie - http://activitystrea.ms/schema/1.0/comment - http://activitystrea.ms/schema/1.0/post - - <p><span class="h-card"><a href="https://mastodon.social/@Fergant" class="u-url mention">@<span>Fergant</span></a></span> Dom är i stort sett religiös skrift vid det här laget 👏👏</p><p>har dock bara läst svenska översättningen, kanske är dags att jag läser dom på engelska</p> - - - public - - - - - - - https://mastodon.social/users/emelie/statuses/101849580030237068 - 2019-04-01T06:47:37Z - 2019-04-01T06:47:37Z - New status by emelie - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - - <p>What&apos;s you people&apos;s favourite fantasy books? Give me some hot tips 🌞</p> - - public - - - - - - https://mastodon.social/users/emelie/statuses/101849550599949363 - 2019-04-01T06:40:08Z - 2019-04-01T06:40:08Z - New status by emelie - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - - <p>Stick them legs out 💃 <a href="https://mastodon.social/tags/mastocats" class="mention hashtag" rel="tag">#<span>mastocats</span></a></p> - - - - public - - - - - - https://mastodon.social/users/emelie/statuses/101849191533152720 - 2019-04-01T05:08:49Z - 2019-04-01T05:08:49Z - New status by emelie - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - - <p>long 🐱 <a href="https://mastodon.social/tags/mastocats" class="mention hashtag" rel="tag">#<span>mastocats</span></a></p> - - - - public - - - - - - https://mastodon.social/users/emelie/statuses/101849165031453009 - 2019-04-01T05:02:05Z - 2019-04-01T05:02:05Z - New status by emelie - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - - <p>You gotta take whatever bellyrubbing opportunity you can get before she changes her mind 🦁 <a href="https://mastodon.social/tags/mastocats" class="mention hashtag" rel="tag">#<span>mastocats</span></a></p> - - - - public - - - - - - https://mastodon.social/users/emelie/statuses/101846512530748693 - 2019-03-31T17:47:31Z - 2019-03-31T17:47:31Z - New status by emelie - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - - <p>Hello look at this boy having a decent haircut for once <a href="https://mastodon.social/tags/mastohorses" class="mention hashtag" rel="tag">#<span>mastohorses</span></a> <a href="https://mastodon.social/tags/equestrian" class="mention hashtag" rel="tag">#<span>equestrian</span></a></p> - - - - - public - - - - - - https://mastodon.social/users/emelie/statuses/101846181093805500 - 2019-03-31T16:23:14Z - 2019-03-31T16:23:14Z - New status by emelie - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - - <p>Sorry did I disturb the who-is-the-longest-cat competition ? <a href="https://mastodon.social/tags/mastocats" class="mention hashtag" rel="tag">#<span>mastocats</span></a></p> - - - - public - - - - - - https://mastodon.social/users/emelie/statuses/101845897513133849 - 2019-03-31T15:11:07Z - 2019-03-31T15:11:07Z - New status by emelie - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - - more earthsea ramblings - <p>I&apos;m re-watching Tales from Earthsea for the first time since I read the books, and that Therru doesn&apos;t squash Cob like a spider, as Orm Embar did is a wasted opportunity tbh</p> - - public - - - - - - https://mastodon.social/users/emelie/statuses/101841219051533307 - 2019-03-30T19:21:19Z - 2019-03-30T19:21:19Z - New status by emelie - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - - <p>I gave my cats some mackerel and they ate it all in 0.3 seconds, and now they won&apos;t stop meowing for more, and I&apos;m tired plz shut up</p> - - public - - - - - - https://mastodon.social/users/emelie/statuses/101839949762341381 - 2019-03-30T13:58:31Z - 2019-03-30T13:58:31Z - New status by emelie - http://activitystrea.ms/schema/1.0/comment - http://activitystrea.ms/schema/1.0/post - - <p>yet I&apos;m confused about this american dude with a gun, like the heck r ya doin in mah ghibli</p> - - public - - - - - - - https://mastodon.social/users/emelie/statuses/101839928677863590 - 2019-03-30T13:53:09Z - 2019-03-30T13:53:09Z - New status by emelie - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - - <p>2 hours into Ni no Kuni 2 and I&apos;ve already sold my soul to this game</p> - - public - - - - - - https://mastodon.social/users/emelie/statuses/101836329521599438 - 2019-03-29T22:37:51Z - 2019-03-29T22:37:51Z - New status by emelie - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - - <p>Pippi Longstocking the original one-punch /man</p> - - public - - - - - - https://mastodon.social/users/emelie/statuses/101835905282948341 - 2019-03-29T20:49:57Z - 2019-03-29T20:49:57Z - New status by emelie - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - - <p>I&apos;ve had so much wine I thought I had a 3rd brother</p> - - public - - - - - - https://mastodon.social/users/emelie/statuses/101835878059204660 - 2019-03-29T20:43:02Z - 2019-03-29T20:43:02Z - New status by emelie - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - - <p>ååååhhh booi</p> - - public - - - - - - https://mastodon.social/users/emelie/statuses/101835848050598939 - 2019-03-29T20:35:24Z - 2019-03-29T20:35:24Z - New status by emelie - http://activitystrea.ms/schema/1.0/comment - http://activitystrea.ms/schema/1.0/post - - <p><span class="h-card"><a href="https://thraeryn.net/@thraeryn" class="u-url mention">@<span>thraeryn</span></a></span> if I spent 1 hour and a half watching this monstrosity, I need to</p> - - - public - - - - - - - https://mastodon.social/users/emelie/statuses/101835823138262290 - 2019-03-29T20:29:04Z - 2019-03-29T20:29:04Z - New status by emelie - http://activitystrea.ms/schema/1.0/comment - http://activitystrea.ms/schema/1.0/post - - medical, fluids mention - <p><span class="h-card"><a href="https://icosahedron.website/@Trev" class="u-url mention">@<span>Trev</span></a></span> *hugs* ✨</p> - - - public - - - - - - diff --git a/test/fixtures/tesla_mock/http__gs.example.org_index.php_api_statuses_user_timeline_1.atom.xml b/test/fixtures/tesla_mock/http__gs.example.org_index.php_api_statuses_user_timeline_1.atom.xml deleted file mode 100644 index 490467708..000000000 --- a/test/fixtures/tesla_mock/http__gs.example.org_index.php_api_statuses_user_timeline_1.atom.xml +++ /dev/null @@ -1,460 +0,0 @@ - - - GNU social - http://gs.example.org/index.php/api/statuses/user_timeline/1.atom - lambda timeline - Updates from lambda on gs.example.org! - http://gs.example.org/theme/neo-gnu/default-avatar-profile.png - 2017-05-05T12:09:57+00:00 - - http://activitystrea.ms/schema/1.0/person - http://gs.example.org:4040/index.php/user/1 - lambda - - - - - lambda - lambda - - - - - - - - - - - - - tag:gs.example.org,2017-05-04:noticeId=84:objectType=note - lambda repeated a notice by lambda2 - RT @<a href="http://gs.example.org/index.php/user/7" class="h-card mention">lambda2</a> Hello! - - http://activitystrea.ms/schema/1.0/share - 2017-05-04T16:38:50+00:00 - 2017-05-04T16:38:50+00:00 - - http://activitystrea.ms/schema/1.0/activity - tag:gs.example.org,2017-05-01:noticeId=67:objectType=note - - Hello! - - http://activitystrea.ms/schema/1.0/post - 2017-05-01T08:41:04+00:00 - 2017-05-01T08:41:04+00:00 - - http://activitystrea.ms/schema/1.0/person - http://gs.example.org/index.php/user/7 - lambda2 - - - - - - lambda2 - lambda2 - - - - - http://activitystrea.ms/schema/1.0/note - tag:gs.example.org,2017-05-01:noticeId=67:objectType=note - New note by lambda2 - Hello! - - - - - tag:gs.example.org,2017-05-01:objectType=thread:nonce=cffa792cb95fe417 - - - http://gs.example.org/index.php/api/statuses/user_timeline/7.atom - lambda2 - - - - http://gs.example.org/avatar/7-96-20170501084054.png - 2017-05-01T16:33:10+00:00 - - - - - - tag:gs.example.org,2017-05-01:objectType=thread:nonce=cffa792cb95fe417 - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:gs.example.org,2017-04-30:noticeId=63:objectType=note - New note by lambda - what now? - - - http://activitystrea.ms/schema/1.0/post - 2017-04-30T10:09:57+00:00 - 2017-04-30T10:09:57+00:00 - - - - tag:gs.example.org,2017-04-30:objectType=thread:nonce=1bbb60991ae9874b - - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:gs.example.org,2017-04-30:noticeId=61:objectType=note - New note by lambda - @<a href="http://pleroma.example.org:4000/users/lain5" class="h-card mention">lain5</a> Hello! - - - http://activitystrea.ms/schema/1.0/post - 2017-04-30T10:07:26+00:00 - 2017-04-30T10:07:26+00:00 - - tag:gs.example.org,2017-04-30:objectType=thread:nonce=1bbb60991ae9874b - - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:gs.example.org,2017-04-29:noticeId=59:objectType=note - New note by lambda - ey - - - http://activitystrea.ms/schema/1.0/post - 2017-04-29T17:04:59+00:00 - 2017-04-29T17:04:59+00:00 - - tag:gs.example.org,2017-04-29:objectType=thread:nonce=4cc42c2c61a0f4bd - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:gs.example.org,2017-04-29:noticeId=58:objectType=note - New note by lambda - Another one. - - - http://activitystrea.ms/schema/1.0/post - 2017-04-29T17:02:47+00:00 - 2017-04-29T17:02:47+00:00 - - tag:gs.example.org,2017-04-29:objectType=thread:nonce=53e9b8f1d6d38d13 - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:gs.example.org,2017-04-29:noticeId=57:objectType=note - New note by lambda - Let's see if this comes over. - - - http://activitystrea.ms/schema/1.0/post - 2017-04-29T17:01:39+00:00 - 2017-04-29T17:01:39+00:00 - - tag:gs.example.org,2017-04-29:objectType=thread:nonce=238a7bd3ffc7c9cc - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:gs.example.org,2017-04-29:noticeId=56:objectType=note - New note by lambda - @<a href="http://pleroma.example.org:4000/users/lain5" class="h-card mention">lain5</a> Hey! - - - http://activitystrea.ms/schema/1.0/post - 2017-04-29T16:38:13+00:00 - 2017-04-29T16:38:13+00:00 - - tag:gs.example.org,2017-04-29:objectType=thread:nonce=2629d3a398171b0f - - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:gs.example.org:4040,2017-04-25:noticeId=55:objectType=note - New note by lambda - hey. - - - http://activitystrea.ms/schema/1.0/post - 2017-04-25T18:16:13+00:00 - 2017-04-25T18:16:13+00:00 - - - - http://pleroma.example.org:4000/contexts/8f6f45d4-8e4d-4e1a-a2de-09f27367d2d0 - - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:gs.example.org:4040,2017-04-25:noticeId=53:objectType=note - New note by lambda - and this? - - - http://activitystrea.ms/schema/1.0/post - 2017-04-25T18:14:34+00:00 - 2017-04-25T18:14:34+00:00 - - - - http://pleroma.example.org:4000/contexts/24779b0e-91ad-487e-81bd-6cf5bb437b09 - - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:gs.example.org:4040,2017-04-25:noticeId=52:objectType=note - New note by lambda - yeah it does :) - - - http://activitystrea.ms/schema/1.0/post - 2017-04-25T18:13:31+00:00 - 2017-04-25T18:13:31+00:00 - - - - tag:gs.example.org:4040,2017-04-25:objectType=thread:nonce=e0dc24b1a93ab6b3 - - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:gs.example.org:4040,2017-04-25:noticeId=50:objectType=note - New note by lambda - @<a href="http://pleroma.example.org:4000/users/lain5" class="h-card mention">lain5</a> Let's try with one that originates here! - - - http://activitystrea.ms/schema/1.0/post - 2017-04-25T18:10:28+00:00 - 2017-04-25T18:10:28+00:00 - - tag:gs.example.org:4040,2017-04-25:objectType=thread:nonce=e0dc24b1a93ab6b3 - - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:gs.example.org:4040,2017-04-25:noticeId=48:objectType=note - New note by lambda - works? - - - http://activitystrea.ms/schema/1.0/post - 2017-04-25T18:08:44+00:00 - 2017-04-25T18:08:44+00:00 - - - - http://pleroma.example.org:4000/contexts/24779b0e-91ad-487e-81bd-6cf5bb437b09 - - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:gs.example.org:4040,2017-04-25:noticeId=46:objectType=note - New note by lambda - Let's send you an answer. - - - http://activitystrea.ms/schema/1.0/post - 2017-04-25T18:05:31+00:00 - 2017-04-25T18:05:31+00:00 - - - - tag:gs.example.org:4040,2017-04-25:objectType=thread:nonce=73c7bcf6658f7ce3 - - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:gs.example.org:4040,2017-04-25:noticeId=44:objectType=note - New note by lambda - Hey. - - - http://activitystrea.ms/schema/1.0/post - 2017-04-25T18:01:09+00:00 - 2017-04-25T18:01:09+00:00 - - - - tag:gs.example.org:4040,2017-04-25:objectType=thread:nonce=6e7c8fc2823380b4 - - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:gs.example.org:4040,2017-04-25:noticeId=43:objectType=note - New note by lambda - What's coming to you? - - - http://activitystrea.ms/schema/1.0/post - 2017-04-25T17:58:41+00:00 - 2017-04-25T17:58:41+00:00 - - - - tag:gs.example.org:4040,2017-04-25:objectType=thread:nonce=6e7c8fc2823380b4 - - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:gs.example.org:4040,2017-04-25:noticeId=42:objectType=note - New note by lambda - Now this is podracing. - - - http://activitystrea.ms/schema/1.0/post - 2017-04-25T17:57:40+00:00 - 2017-04-25T17:57:40+00:00 - - - - tag:gs.example.org:4040,2017-04-25:objectType=thread:nonce=6e7c8fc2823380b4 - - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:gs.example.org:4040,2017-04-25:noticeId=39:objectType=note - New note by lambda - Sure looks like it! - - - http://activitystrea.ms/schema/1.0/post - 2017-04-25T17:48:27+00:00 - 2017-04-25T17:48:27+00:00 - - - - tag:gs.example.org:4040,2017-04-25:objectType=thread:nonce=4c6114a75bb4cea5 - - - - - - - - tag:gs.example.org:4040,2017-04-25:subscription:1:person:6:2017-04-25T17:47:47+00:00 - lambda (lambda)'s status on Tuesday, 25-Apr-2017 17:47:47 UTC - <a href="http://gs.example.org:4040/index.php/lambda">lambda</a> started following <a href="http://pleroma.example.org:4000/users/lain5">l</a>. - - http://activitystrea.ms/schema/1.0/follow - 2017-04-25T17:47:47+00:00 - 2017-04-25T17:47:47+00:00 - - http://activitystrea.ms/schema/1.0/person - http://pleroma.example.org:4000/users/lain5 - l - lambadalambda - - - - - - lain5 - l - lambadalambda - - - tag:gs.example.org:4040,2017-04-25:objectType=thread:nonce=119acad17515314c - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:gs.example.org:4040,2017-04-25:noticeId=36:objectType=note - New note by lambda - @<a href="http://pleroma.example.org:4000/users/lain5" class="h-card mention">lain5</a> Hey, how are you? - - - http://activitystrea.ms/schema/1.0/post - 2017-04-25T17:46:22+00:00 - 2017-04-25T17:46:22+00:00 - - tag:gs.example.org:4040,2017-04-25:objectType=thread:nonce=9c5ec19a18191372 - - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:gs.example.org:4040,2017-04-25:noticeId=35:objectType=note - New note by lambda - @lain5@pleroma.example.org does this not work? - - - http://activitystrea.ms/schema/1.0/post - 2017-04-25T17:42:31+00:00 - 2017-04-25T17:42:31+00:00 - - tag:gs.example.org:4040,2017-04-25:objectType=thread:nonce=fc841d7f52caa363 - - - - - - diff --git a/test/fixtures/tesla_mock/https___mamot.fr_users_Skruyb.atom b/test/fixtures/tesla_mock/https___mamot.fr_users_Skruyb.atom deleted file mode 100644 index b5f3d923b..000000000 --- a/test/fixtures/tesla_mock/https___mamot.fr_users_Skruyb.atom +++ /dev/null @@ -1,342 +0,0 @@ - - - https://mamot.fr/users/Skruyb.atom - The 7th Son - Fr and En. -Posts will disappear on a regular basis. - 2017-04-28T13:54:23Z - https://mamot.fr/system/accounts/avatars/000/026/213/original/d95dbcfc76f77f4c.jpg?1493230984 - - https://mamot.fr/users/Skruyb - http://activitystrea.ms/schema/1.0/person - https://mamot.fr/users/Skruyb - Skruyb - Skruyb@mamot.fr - <p>Fr and En.<br />Posts will disappear on a regular basis.</p> - - - - Skruyb - The 7th Son - Fr and En. -Posts will disappear on a regular basis. - public - - - - - - - - tag:mamot.fr,2017-05-10:objectId=1299665:objectType=Status - 2017-05-10T20:06:59Z - 2017-05-10T20:06:59Z - New status by Skruyb - http://activitystrea.ms/schema/1.0/comment - http://activitystrea.ms/schema/1.0/post - <p><span class="h-card"><a href="https://pouets.ovh/@noName" class="u-url mention">@<span>noName</span></a></span></p><p>Pour comparer faut avoir tester... Ô wait!!! 😁</p> - - - public - - - - - - tag:mamot.fr,2017-05-10:objectId=1299185:objectType=Status - 2017-05-10T19:52:14Z - 2017-05-10T19:52:14Z - New status by Skruyb - http://activitystrea.ms/schema/1.0/comment - http://activitystrea.ms/schema/1.0/post - <p><span class="h-card"><a href="https://witches.town/@Dhveszak" class="u-url mention">@<span>Dhveszak</span></a></span></p><p>Toi!! Tu vises le ministère de la propagande avoue!!!!!!!</p> - - - public - - - - - - tag:mamot.fr,2017-05-10:objectId=1299019:objectType=Status - 2017-05-10T19:47:19Z - 2017-05-10T19:47:19Z - New status by Skruyb - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - <p>Facebook s&apos;attaque aux sites internet &quot;trompeurs&quot;</p><p><a href="http://u.afp.com/4W4z" rel="nofollow noopener" target="_blank"><span class="invisible">http://</span><span class="">u.afp.com/4W4z</span><span class="invisible"></span></a></p><p>J&apos;attends de voir que Facebook s&apos;attaque à lui même... rien qu&apos;à lire leurs conditions générales d&apos;utilisation, le respect de la vie privée...</p><p>Charité bien ordonnée... Parfois l&apos;égoïsme aurait du bon.</p> - - public - - - - - tag:mamot.fr,2017-05-10:objectId=1298889:objectType=Status - 2017-05-10T19:43:18Z - 2017-05-10T19:43:18Z - New status by Skruyb - http://activitystrea.ms/schema/1.0/comment - http://activitystrea.ms/schema/1.0/post - <p><span class="h-card"><a href="https://octodon.social/@Balise" class="u-url mention">@<span>Balise</span></a></span></p><p>Fait comme moi, annonce que tu fais dans le flou artistique et que seuls des esprits éclairés pourront en percevoir la beauté et apprécier. Globalement après ça, tout le monde trouve les photos cool :-p</p> - - - public - - - - - - tag:mamot.fr,2017-05-10:objectId=1298728:objectType=Status - 2017-05-10T19:38:39Z - 2017-05-10T19:38:39Z - New status by Skruyb - http://activitystrea.ms/schema/1.0/comment - http://activitystrea.ms/schema/1.0/post - <p><span class="h-card"><a href="https://mastodon.social/@applecandy" class="u-url mention">@<span>applecandy</span></a></span></p><p>Lucky you!!!</p> - - - public - - - - - - tag:mamot.fr,2017-05-10:objectId=1298431:objectType=Status - 2017-05-10T19:26:32Z - 2017-05-10T19:26:32Z - New status by Skruyb - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - <p>Est-ce que je suis le seul qui lorsqu&apos;il commence à compter les arbres sur le bord de la route n&apos;arrive pas à s&apos;arrêter de compter?</p> - - public - - - - - tag:mamot.fr,2017-05-10:objectId=1298224:objectType=Status - 2017-05-10T19:18:17Z - 2017-05-10T19:18:17Z - New status by Skruyb - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - <p>Ca y est j&apos;ai une nouvelle passion. Mettre les bouchons qui trainent par terre dans le bons sens avec mon pied 🙌</p> - - public - - - - - tag:mamot.fr,2017-05-10:objectId=1297450:objectType=Status - 2017-05-10T18:53:37Z - 2017-05-10T18:53:37Z - New status by Skruyb - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - <p>Ok. On est capable d&apos;envoyer des mecs dans l&apos;espace, avoir des voitures autonomes, des trucs intelligents de partout mais pas tous les bâtiments accessibles aux personnes à mobilité réduite, les émissions sur le services publics avec une personne faisant la traduction pour les sourds et malentendants de manière systématique...</p><p>J&apos;ai du louper un truc dans l&apos;ordre des priorités Oo</p> - - public - - - - - tag:mamot.fr,2017-05-10:objectId=1297292:objectType=Status - 2017-05-10T18:48:17Z - 2017-05-10T18:48:17Z - New status by Skruyb - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - <p>J&apos;ai comme envie de faire un truc mais je ne sais pas quoi mais pourtant c&apos;est comme si je ressentais l&apos;idée dans ma tête mais c&apos;est pas clair...</p><p>Fuck!!! J&apos;vais aller draguer Josiane à la compta ça va me changer les idées!!!</p> - - public - - - - - tag:mamot.fr,2017-05-10:objectId=1296598:objectType=Status - 2017-05-10T18:25:11Z - 2017-05-10T18:25:11Z - New status by Skruyb - http://activitystrea.ms/schema/1.0/comment - http://activitystrea.ms/schema/1.0/post - <p><span class="h-card"><a href="https://mamot.fr/@Smeablog" class="u-url mention">@<span>Smeablog</span></a></span></p><p>Pas faux MDR!!!!</p> - - - public - - - - - - tag:mamot.fr,2017-05-10:objectId=1296571:objectType=Status - 2017-05-10T18:24:13Z - 2017-05-10T18:24:13Z - New status by Skruyb - http://activitystrea.ms/schema/1.0/comment - http://activitystrea.ms/schema/1.0/post - <p><span class="h-card"><a href="https://mamot.fr/@Smeablog" class="u-url mention">@<span>Smeablog</span></a></span></p><p>Ca ne change pas la finalité malheureusement, ça ne m&apos;ouvre pas ce à quoi je veux accéder 😭</p> - - - public - - - - - - tag:mamot.fr,2017-05-10:objectId=1296475:objectType=Status - 2017-05-10T18:20:50Z - 2017-05-10T18:20:50Z - New status by Skruyb - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - <p>Arrrgghhhhhhh!!!!</p><p>Quand t&apos;es sur le point de cliquer sur un lien dans le fil public global et que BOOM ça se met à jour... J&apos;ose même pas imaginer combien j&apos;ai ouvert de pages web sans le vouloir!!!</p> - - public - - - - - tag:mamot.fr,2017-05-10:objectId=1296426:objectType=Status - 2017-05-10T18:19:17Z - 2017-05-10T18:19:17Z - Skruyb shared a status by Isaluini@mastodon.social - http://activitystrea.ms/schema/1.0/activity - http://activitystrea.ms/schema/1.0/share - - tag:mastodon.social,2017-05-10:objectId=5587049:objectType=Status - 2017-05-10T18:18:59Z - 2017-05-10T18:19:00Z - New status by Isaluini@mastodon.social - - https://mastodon.social/users/Isaluini - http://activitystrea.ms/schema/1.0/person - https://mastodon.social/users/Isaluini - Isaluini - Isaluini@mastodon.social - <p>Adicciones: Escribir, diseñar, cine, café, humor negro, música y dibujar. | Jedi. Bueno, no. Algún día (?) | Gratitude.</p> - - - - Isaluini - Isa - Adicciones: Escribir, diseñar, cine, café, humor negro, música y dibujar. | Jedi. Bueno, no. Algún día (?) | Gratitude. - public - - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - <p>♫ <br><a href="https://www.youtube.com/watch?v=pT68FS3YbQ4"><span class="invisible">https://www.</span><span class="ellipsis">youtube.com/watch?v=pT68FS3YbQ</span><span class="invisible">4</span></a></p> - - public - - - <p>♫ <br><a href="https://www.youtube.com/watch?v=pT68FS3YbQ4"><span class="invisible">https://www.</span><span class="ellipsis">youtube.com/watch?v=pT68FS3YbQ</span><span class="invisible">4</span></a></p> - - public - - - - - tag:mamot.fr,2017-05-10:objectId=1295893:objectType=Status - 2017-05-10T18:01:51Z - 2017-05-10T18:01:51Z - New status by Skruyb - http://activitystrea.ms/schema/1.0/comment - http://activitystrea.ms/schema/1.0/post - <p><span class="h-card"><a href="https://mamot.fr/@Chat2Gouttieres" class="u-url mention">@<span>Chat2Gouttieres</span></a></span></p><p>Ah bah après faut savoir mettre à profit ce savoir ^^</p> - - - public - - - - - - tag:mamot.fr,2017-05-10:objectId=1295815:objectType=Status - 2017-05-10T18:00:02Z - 2017-05-10T18:00:02Z - New status by Skruyb - http://activitystrea.ms/schema/1.0/comment - http://activitystrea.ms/schema/1.0/post - <p><span class="h-card"><a href="https://mamot.fr/@Chat2Gouttieres" class="u-url mention">@<span>Chat2Gouttieres</span></a></span></p><p>Exactement. On a les jeux mais pas le pain encore.</p><p>Finalement on a rien inventé :-p</p> - - - public - - - - - - tag:mamot.fr,2017-05-10:objectId=1295778:objectType=Status - 2017-05-10T17:58:52Z - 2017-05-10T17:58:52Z - New status by Skruyb - http://activitystrea.ms/schema/1.0/comment - http://activitystrea.ms/schema/1.0/post - <p><span class="h-card"><a href="https://mamot.fr/@Chat2Gouttieres" class="u-url mention">@<span>Chat2Gouttieres</span></a></span></p><p>C&apos;est ça visiblement dans notre société dite moderne... &quot;Créer l&apos;illusion que&quot; Oo.</p> - - - public - - - - - - tag:mamot.fr,2017-05-10:objectId=1294943:objectType=Status - 2017-05-10T17:31:44Z - 2017-05-10T17:31:44Z - New status by Skruyb - http://activitystrea.ms/schema/1.0/comment - http://activitystrea.ms/schema/1.0/post - Hey. - <p><span class="h-card"><a href="https://mastodon.social/@lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span></p><p>Hey!!!</p> - - - public - - - - - - tag:mamot.fr,2017-05-10:objectId=1294914:objectType=Status - 2017-05-10T17:30:40Z - 2017-05-10T17:30:40Z - New status by Skruyb - http://activitystrea.ms/schema/1.0/comment - http://activitystrea.ms/schema/1.0/post - <p><span class="h-card"><a href="https://mamot.fr/@EloClemTiti" class="u-url mention">@<span>EloClemTiti</span></a></span></p><p>J&apos;ai souvent cette impression en effet 😂</p> - - - public - - - - - - tag:mamot.fr,2017-05-10:objectId=1294148:objectType=Status - 2017-05-10T17:02:01Z - 2017-05-10T17:02:01Z - New status by Skruyb - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - <p>Les gars, les boss veulent voir de l&apos;avancement!! Une idée?</p><p>On fait comme d&apos;habitude. On divise nos tâches en 25.000 tâches unitaires, on fout du vert au maximum et on crée l&apos;illusion que ça a bien avancé!</p><p>Deal!!</p><p>Bob, tu choisis quel vert on utilise<br />Alice, t&apos;es en charge de la typo<br />Moi, je m&apos;occupe qu&apos;on prend bien le dernier template ppt fournit par la comm interne.</p><p>Des winners qu&apos;on est!!!! Des WI-NNERS!!!</p> - - public - - - - - tag:mamot.fr,2017-05-10:objectId=1293995:objectType=Status - 2017-05-10T16:57:53Z - 2017-05-10T16:57:53Z - New status by Skruyb - http://activitystrea.ms/schema/1.0/comment - http://activitystrea.ms/schema/1.0/post - <p><span class="h-card"><a href="https://mastodon.social/@SauceHair" class="u-url mention">@<span>SauceHair</span></a></span></p><p>Cool!!</p><p>Bon courage.</p> - - - public - - - - - diff --git a/test/fixtures/tesla_mock/https___mastodon.social_users_lambadalambda.atom b/test/fixtures/tesla_mock/https___mastodon.social_users_lambadalambda.atom deleted file mode 100644 index 4d732b109..000000000 --- a/test/fixtures/tesla_mock/https___mastodon.social_users_lambadalambda.atom +++ /dev/null @@ -1,464 +0,0 @@ - - - https://mastodon.social/users/lambadalambda.atom - Critical Value - - 2017-04-16T21:47:25Z - https://files.mastodon.social/accounts/avatars/000/000/264/original/1429214160519.gif - - https://mastodon.social/users/lambadalambda - http://activitystrea.ms/schema/1.0/person - https://mastodon.social/users/lambadalambda - lambadalambda - lambadalambda@mastodon.social - - - - lambadalambda - Critical Value - public - - - - - - - - tag:mastodon.social,2017-05-04:objectId=4991300:objectType=Status - 2017-05-04T14:10:30Z - 2017-05-04T14:10:30Z - Delete - http://activitystrea.ms/schema/1.0/activity - http://activitystrea.ms/schema/1.0/delete - - - - - tag:mastodon.social,2017-05-04:objectId=4980289:objectType=Status - 2017-05-04T07:43:23Z - 2017-05-04T07:43:23Z - Delete - http://activitystrea.ms/schema/1.0/activity - http://activitystrea.ms/schema/1.0/delete - - - - - tag:mastodon.social,2017-05-03:objectId=4952899:objectType=Status - 2017-05-03T17:26:43Z - 2017-05-03T17:26:43Z - New status by lambadalambda - http://activitystrea.ms/schema/1.0/comment - http://activitystrea.ms/schema/1.0/post - <p><span class="h-card"><a href="https://pleroma.soykaf.com/users/lain" class="u-url mention">@<span>lain</span></a></span> OK!!</p> - - - public - - - - - - tag:mastodon.social,2017-05-03:objectId=4952810:objectType=Status - 2017-05-03T17:24:34Z - 2017-05-03T17:24:34Z - New status by lambadalambda - http://activitystrea.ms/schema/1.0/comment - http://activitystrea.ms/schema/1.0/post - <p><span class="h-card"><a href="https://pleroma.soykaf.com/users/lain" class="u-url mention">@<span>lain</span></a></span> yeah :)</p> - - - public - - - - - - tag:mastodon.social,2017-05-03:objectId=4950388:objectType=Status - 2017-05-03T16:22:00Z - 2017-05-03T16:22:00Z - lambadalambda shared a status by lambadalambda@social.heldscal.la - http://activitystrea.ms/schema/1.0/activity - http://activitystrea.ms/schema/1.0/share - - tag:social.heldscal.la,2017-05-03:noticeId=2030733:objectType=note - 2017-05-03T12:29:20Z - 2017-05-03T12:29:31Z - New status by lambadalambda@social.heldscal.la - - https://social.heldscal.la/user/23211 - http://activitystrea.ms/schema/1.0/person - https://social.heldscal.la/user/23211 - lambadalambda - lambadalambda@social.heldscal.la - Call me Deacon Blues. - - - - lambadalambda - Constance Variable - Call me Deacon Blues. - public - - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - Time for work. <a href="https://social.heldscal.la/file/953c117a1e7e4c763755d2ac29cf1aae08e025599f4a4cc11ddff4082c07f969.jpg">https://social.heldscal.la/attachment/120552</a> - - - public - - - Time for work. <a href="https://social.heldscal.la/file/953c117a1e7e4c763755d2ac29cf1aae08e025599f4a4cc11ddff4082c07f969.jpg">https://social.heldscal.la/attachment/120552</a> - - public - - - - - tag:mastodon.social,2017-05-03:objectId=4934452:objectType=Status - 2017-05-03T08:21:09Z - 2017-05-03T08:21:09Z - lambadalambda shared a status by lain@pleroma.soykaf.com - http://activitystrea.ms/schema/1.0/activity - http://activitystrea.ms/schema/1.0/share - - https://pleroma.soykaf.com/objects/4c1bda26-902e-4525-9fcd-b9fd44925193 - 2017-05-03T08:04:44Z - 2017-05-03T08:05:52Z - New status by lain@pleroma.soykaf.com - - https://pleroma.soykaf.com/users/lain - http://activitystrea.ms/schema/1.0/person - https://pleroma.soykaf.com/users/lain - lain - lain@pleroma.soykaf.com - Test account - - - - lain - Lain Iwakura - Test account - public - - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - Added returning the entries as xml... let's see if the mastodon hammering stops now. - - public - - - Added returning the entries as xml... let's see if the mastodon hammering stops now. - - public - - - - - tag:mastodon.social,2017-05-02:objectId=4905499:objectType=Status - 2017-05-02T19:34:21Z - 2017-05-02T19:34:21Z - New status by lambadalambda - http://activitystrea.ms/schema/1.0/comment - http://activitystrea.ms/schema/1.0/post - <p><span class="h-card"><a href="https://pleroma.soykaf.com/users/lain" class="u-url mention">@<span>lain</span></a></span> yay!</p> - - - public - - - - - - tag:mastodon.social,2017-05-02:objectId=4905442:objectType=Status - 2017-05-02T19:33:33Z - 2017-05-02T19:33:33Z - New status by lambadalambda - http://activitystrea.ms/schema/1.0/comment - http://activitystrea.ms/schema/1.0/post - <p><span class="h-card"><a href="https://pleroma.soykaf.com/users/lain" class="u-url mention">@<span>lain</span></a></span> so?</p> - - - public - - - - - - tag:mastodon.social,2017-05-02:objectId=4901603:objectType=Status - 2017-05-02T18:33:06Z - 2017-05-02T18:33:06Z - New status by lambadalambda - http://activitystrea.ms/schema/1.0/comment - http://activitystrea.ms/schema/1.0/post - <p><span class="h-card"><a href="https://pleroma.soykaf.com/users/lain" class="u-url mention">@<span>lain</span></a></span> hey</p> - - - public - - - - - - tag:mastodon.social,2017-05-01:objectId=4836720:objectType=Status - 2017-05-01T18:52:16Z - 2017-05-01T18:52:16Z - lambadalambda shared a status by lain@pleroma.soykaf.com - http://activitystrea.ms/schema/1.0/activity - http://activitystrea.ms/schema/1.0/share - - https://pleroma.soykaf.com/objects/7b41bb51-9aba-436a-82d9-dd3f5aca98c9 - 2017-05-01T18:50:54Z - 2017-05-01T18:50:57Z - New status by lain@pleroma.soykaf.com - - https://pleroma.soykaf.com/users/lain - http://activitystrea.ms/schema/1.0/person - https://pleroma.soykaf.com/users/lain - lain - lain@pleroma.soykaf.com - Test account - - - - lain - Lain Iwakura - Test account - public - - http://activitystrea.ms/schema/1.0/comment - http://activitystrea.ms/schema/1.0/post - <a href="https://mastodon.social/users/lambadalambda">@lambadalambda@mastodon.social</a> you're an all-star. - - - public - - - - <a href="https://mastodon.social/users/lambadalambda">@lambadalambda@mastodon.social</a> you're an all-star. - - public - - - - - tag:mastodon.social,2017-05-01:objectId=4836142:objectType=Status - 2017-05-01T18:38:47Z - 2017-05-01T18:38:47Z - New status by lambadalambda - http://activitystrea.ms/schema/1.0/comment - http://activitystrea.ms/schema/1.0/post - <p><span class="h-card"><a href="https://pleroma.soykaf.com/users/lain" class="u-url mention">@<span>lain</span></a></span> Hey now!</p> - - - public - - - - - - tag:mastodon.social,2017-05-01:objectId=4836055:objectType=Status - 2017-05-01T18:37:04Z - 2017-05-01T18:37:04Z - New status by lambadalambda - http://activitystrea.ms/schema/1.0/comment - http://activitystrea.ms/schema/1.0/post - <p><span class="h-card"><a href="https://pleroma.soykaf.com/users/lain" class="u-url mention">@<span>lain</span></a></span> hello</p> - - - public - - - - - - tag:mastodon.social,2017-05-01:objectId=4834850:objectType=Status - 2017-05-01T18:10:43Z - 2017-05-01T18:10:43Z - New status by lambadalambda - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - <p><span class="h-card"><a href="https://pleroma.soykaf.com/users/lain" class="u-url mention">@<span>lain</span></a></span> Hey!</p> - - - public - - - - - tag:mastodon.social,2017-04-29:objectId=4694455:objectType=Status - 2017-04-29T18:39:12Z - 2017-04-29T18:39:12Z - New status by lambadalambda - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - <p>@lain@pleroma.soykaf.com What&apos;s up?</p> - - public - - - - - tag:mastodon.social,2017-04-29:objectId=4694384:objectType=Status - 2017-04-29T18:37:32Z - 2017-04-29T18:37:32Z - New status by lambadalambda - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - <p><span class="h-card"><a href="https://social.heldscal.la/lain" class="u-url mention">@<span>lain</span></a></span> Hey.</p> - - - public - - - - - tag:mastodon.social,2017-04-07:objectId=1874242:objectType=Status - 2017-04-07T11:02:56Z - 2017-04-07T11:02:56Z - lambadalambda shared a status by 0xroy@social.wxcafe.net - http://activitystrea.ms/schema/1.0/activity - http://activitystrea.ms/schema/1.0/share - - tag:social.wxcafe.net,2017-04-07:objectId=72554:objectType=Status - 2017-04-07T11:01:59Z - 2017-04-07T11:02:00Z - New status by 0xroy@social.wxcafe.net - - https://social.wxcafe.net/users/0xroy - http://activitystrea.ms/schema/1.0/person - https://social.wxcafe.net/users/0xroy - 0xroy - 0xroy@social.wxcafe.net - ta caution weeb | discussions privées : <a href="https://%F0%9F%92%8C.0xroy.me"><span class="invisible">https://</span><span class="">💌.0xroy.me</span><span class="invisible"></span></a> - - - - 0xroy - 「R O Y 🍵 B O S」 - ta caution weeb | discussions privées : https://💌.0xroy.me - public - - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - <p>someone pls eli5 matrix (protocol) and riot</p> - - public - - - <p>someone pls eli5 matrix (protocol) and riot</p> - - public - - - - - tag:mastodon.social,2017-04-06:objectId=1768247:objectType=Status - 2017-04-06T11:10:19Z - 2017-04-06T11:10:19Z - lambadalambda shared a status by areyoutoo@mastodon.xyz - http://activitystrea.ms/schema/1.0/activity - http://activitystrea.ms/schema/1.0/share - - tag:mastodon.xyz,2017-04-05:objectId=133327:objectType=Status - 2017-04-05T17:36:41Z - 2017-04-05T18:12:14Z - New status by areyoutoo@mastodon.xyz - - https://mastodon.xyz/users/areyoutoo - http://activitystrea.ms/schema/1.0/person - https://mastodon.xyz/users/areyoutoo - areyoutoo - areyoutoo@mastodon.xyz - devops | retired gamedev | always boost puppy pics - - - - areyoutoo - Raw Butter - devops | retired gamedev | always boost puppy pics - public - - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - <p>Some UX thoughts for <a href="https://mastodon.xyz/tags/mastodev">#<span>mastodev</span></a>:</p><p>- Would be nice if I could work on multiple draft toots? Clicking to reply to someone seems to erase any draft I had been working on.</p><p>- Kinda risky to click on the Federated Timeline if it loads new toots and scrolls 10ms before I click on something.</p><p>I probably don't know enough web frontend to help, but it might be fun to try.</p> - - - public - - - <p>Some UX thoughts for <a href="https://mastodon.xyz/tags/mastodev">#<span>mastodev</span></a>:</p><p>- Would be nice if I could work on multiple draft toots? Clicking to reply to someone seems to erase any draft I had been working on.</p><p>- Kinda risky to click on the Federated Timeline if it loads new toots and scrolls 10ms before I click on something.</p><p>I probably don't know enough web frontend to help, but it might be fun to try.</p> - - public - - - - - tag:mastodon.social,2017-04-06:objectId=1764509:objectType=Status - 2017-04-06T10:15:38Z - 2017-04-06T10:15:38Z - New status by lambadalambda - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - This is a test for cw federation - <p>This is a test for cw federation body text.</p> - - public - - - - - tag:mastodon.social,2017-04-05:objectId=1645208:objectType=Status - 2017-04-05T07:14:53Z - 2017-04-05T07:14:53Z - lambadalambda shared a status by lambadalambda@social.heldscal.la - http://activitystrea.ms/schema/1.0/activity - http://activitystrea.ms/schema/1.0/share - - tag:social.heldscal.la,2017-04-05:noticeId=1502088:objectType=note - 2017-04-05T06:12:09Z - 2017-04-05T07:12:47Z - New status by lambadalambda@social.heldscal.la - - https://social.heldscal.la/user/23211 - http://activitystrea.ms/schema/1.0/person - https://social.heldscal.la/user/23211 - lambadalambda - lambadalambda@social.heldscal.la - Call me Deacon Blues. - - - - lambadalambda - Constance Variable - Call me Deacon Blues. - public - - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - Federation 101: <a href="https://www.youtube.com/watch?v=t1lYU5CA40o">https://www.youtube.com/watch?v=t1lYU5CA40o</a> - - public - - - Federation 101: <a href="https://www.youtube.com/watch?v=t1lYU5CA40o">https://www.youtube.com/watch?v=t1lYU5CA40o</a> - - public - - - - - tag:mastodon.social,2017-04-05:objectId=1641750:objectType=Status - 2017-04-05T05:44:48Z - 2017-04-05T05:44:48Z - New status by lambadalambda - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> just a test.</p> - - - public - - - - diff --git a/test/fixtures/tesla_mock/https___pawoo.net_users_pekorino.atom b/test/fixtures/tesla_mock/https___pawoo.net_users_pekorino.atom deleted file mode 100644 index 17d1956e8..000000000 --- a/test/fixtures/tesla_mock/https___pawoo.net_users_pekorino.atom +++ /dev/null @@ -1,231 +0,0 @@ - - - https://pawoo.net/users/pekorino.atom - モノエ - シアトル・米国 - -GNUsocial 英語版 -http://shitposter.club/mono - - - 2017-05-07T09:28:20Z - https://img.pawoo.net/accounts/avatars/000/128/378/original/e1fce04a36a1ad90.jpg - - https://pawoo.net/users/pekorino - http://activitystrea.ms/schema/1.0/person - https://pawoo.net/users/pekorino - pekorino - pekorino@pawoo.net - <p>シアトル・米国</p><p>GNUsocial 英語版<br /><a href="http://shitposter.club/mono" rel="nofollow noopener" target="_blank"><span class="invisible">http://</span><span class="">shitposter.club/mono</span><span class="invisible"></span></a> </p> - - - - pekorino - モノエ - シアトル・米国 - -GNUsocial 英語版 -http://shitposter.club/mono - - - public - - - - - - - tag:pawoo.net,2017-05-07:objectId=9319211:objectType=Status - 2017-05-07T09:56:35Z - 2017-05-07T09:56:35Z - New status by pekorino - http://activitystrea.ms/schema/1.0/comment - http://activitystrea.ms/schema/1.0/post - <p><span class="h-card"><a href="https://shitposter.club/moonman" class="u-url mention">@<span>moonman</span></a></span> <span class="h-card"><a href="https://shitposter.club/rw" class="u-url mention">@<span>rw</span></a></span> <span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> <span class="h-card"><a href="https://shitposter.club/mono" class="u-url mention">@<span>mono</span></a></span> </p><p>i have to wait for someone to respond to this before i can follow because i dont think this software has a direct follow by url option</p> - - - - - - public - - - - - - tag:pawoo.net,2017-05-07:objectId=9318595:objectType=Status - 2017-05-07T09:54:39Z - 2017-05-07T09:54:39Z - New status by pekorino - http://activitystrea.ms/schema/1.0/comment - http://activitystrea.ms/schema/1.0/post - <p><span class="h-card"><a href="https://shitposter.club/mono" class="u-url mention">@<span>mono</span></a></span> <span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> <span class="h-card"><a href="https://shitposter.club/rw" class="u-url mention">@<span>rw</span></a></span> <span class="h-card"><a href="https://shitposter.club/moonman" class="u-url mention">@<span>moonman</span></a></span> <br />please respond</p> - - - - - - public - - - - - - tag:pawoo.net,2017-05-07:objectId=9313978:objectType=Status - 2017-05-07T09:39:17Z - 2017-05-07T09:39:17Z - New status by pekorino - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - <p><span class="h-card"><a href="https://shitposter.club/moonman" class="u-url mention">@<span>moonman</span></a></span> <br />mastodon is so slow. browser crashed twice trying to set avatar</p> - - - public - - - - - tag:pawoo.net,2017-05-07:objectId=9312691:objectType=Status - 2017-05-07T09:34:38Z - 2017-05-07T09:34:38Z - New status by pekorino - http://activitystrea.ms/schema/1.0/comment - http://activitystrea.ms/schema/1.0/post - <p><span class="h-card"><a href="https://shitposter.club/hardbass2k8" class="u-url mention">@<span>hardbass2k8</span></a></span> <a href="https://pawoo.net/media/mZJjLpbPU72GFEz2Svk" rel="nofollow noopener" target="_blank"><span class="invisible">https://</span><span class="ellipsis">pawoo.net/media/mZJjLpbPU72GFE</span><span class="invisible">z2Svk</span></a></p> - - - - public - - - - - - tag:pawoo.net,2017-05-07:objectId=9312379:objectType=Status - 2017-05-07T09:33:29Z - 2017-05-07T09:33:29Z - New status by pekorino - http://activitystrea.ms/schema/1.0/comment - http://activitystrea.ms/schema/1.0/post - <p><span class="h-card"><a href="https://shitposter.club/hardbass2k8" class="u-url mention">@<span>hardbass2k8</span></a></span> <a href="https://pawoo.net/media/nt5JHBEHyTN2bqzdcGU" rel="nofollow noopener" target="_blank"><span class="invisible">https://</span><span class="ellipsis">pawoo.net/media/nt5JHBEHyTN2bq</span><span class="invisible">zdcGU</span></a></p> - - - - public - - - - - - tag:pawoo.net,2017-05-07:objectId=9311765:objectType=Status - 2017-05-07T09:31:26Z - 2017-05-07T09:31:26Z - New status by pekorino - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - <p><a href="https://pawoo.net/media/C4RV6ubsEtvS04DX6qs" rel="nofollow noopener" target="_blank"><span class="invisible">https://</span><span class="ellipsis">pawoo.net/media/C4RV6ubsEtvS04</span><span class="invisible">DX6qs</span></a></p> - - - public - - - - - tag:pawoo.net,2017-05-07:objectId=9311610:objectType=Status - 2017-05-07T09:30:59Z - 2017-05-07T09:30:59Z - New status by pekorino - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - <p><a href="https://pawoo.net/media/MBmkeEdrjs8pAtCHN6s" rel="nofollow noopener" target="_blank"><span class="invisible">https://</span><span class="ellipsis">pawoo.net/media/MBmkeEdrjs8pAt</span><span class="invisible">CHN6s</span></a></p> - - - public - - - - - tag:pawoo.net,2017-05-07:objectId=9307782:objectType=Status - 2017-05-07T09:16:47Z - 2017-05-07T09:16:47Z - New status by pekorino - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - <p><span class="h-card"><a href="https://shitposter.club/mono" class="u-url mention">@<span>mono</span></a></span></p><p>test</p> - - - public - - - - - tag:pawoo.net,2017-05-07:objectId=9307444:objectType=Status - 2017-05-07T09:15:42Z - 2017-05-07T09:15:42Z - New status by pekorino - http://activitystrea.ms/schema/1.0/comment - http://activitystrea.ms/schema/1.0/post - <p><span class="h-card"><a href="https://shitposter.club/hardbass2k8" class="u-url mention">@<span>hardbass2k8</span></a></span> テスト</p> - - - public - - - - - - tag:pawoo.net,2017-05-07:objectId=9307239:objectType=Status - 2017-05-07T09:14:58Z - 2017-05-07T09:14:58Z - New status by pekorino - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - <p>ててててててテスト</p> - - public - - - - - tag:pawoo.net,2017-04-20:objectId=2212164:objectType=Status - 2017-04-20T06:19:18Z - 2017-04-20T06:19:18Z - New status by pekorino - http://activitystrea.ms/schema/1.0/comment - http://activitystrea.ms/schema/1.0/post - <p><span class="h-card"><a href="https://shitposter.club/mono" class="u-url mention">@<span>mono</span></a></span> <a href="https://pawoo.net/media/iMbjMBVPfZJX3lUC2Sc" rel="nofollow noopener" target="_blank"><span class="invisible">https://</span><span class="ellipsis">pawoo.net/media/iMbjMBVPfZJX3l</span><span class="invisible">UC2Sc</span></a></p> - - - - public - - - - - - tag:pawoo.net,2017-04-20:objectId=2206216:objectType=Status - 2017-04-20T05:57:59Z - 2017-04-20T05:57:59Z - New status by pekorino - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - <p>テスト</p> - - public - - - - - tag:pawoo.net,2017-04-20:objectId=2204702:objectType=Status - 2017-04-20T05:52:09Z - 2017-04-20T05:52:09Z - New status by pekorino - http://activitystrea.ms/schema/1.0/note - http://activitystrea.ms/schema/1.0/post - <p>ＨＥＬＬＯＷＯＲＬＤ</p> - - public - - - - diff --git a/test/fixtures/tesla_mock/https___pleroma.soykaf.com_users_lain_feed.atom.xml b/test/fixtures/tesla_mock/https___pleroma.soykaf.com_users_lain_feed.atom.xml deleted file mode 100644 index a2a2629a6..000000000 --- a/test/fixtures/tesla_mock/https___pleroma.soykaf.com_users_lain_feed.atom.xml +++ /dev/null @@ -1 +0,0 @@ -https://pleroma.soykaf.com/users/lain/feed.atomlain's timeline2017-05-05T08:38:03.385598https://pleroma.soykaf.com/users/lainhttp://activitystrea.ms/schema/1.0/personhttps://pleroma.soykaf.com/users/lainlainLain IwakuraTest accountlainhttp://activitystrea.ms/schema/1.0/activityhttp://activitystrea.ms/schema/1.0/sharehttps://pleroma.soykaf.com/activities/579e4224-b2ab-4ffa-8bbe-f7197a0a38d1lain repeated a noticeRT In just seven days, I can make you a man!<br> -- The Rocky Horror Picture Show2017-05-05T08:38:03.3855902017-05-05T08:38:03.385598https://pleroma.soykaf.com/contexts/e8673466-9642-4c9e-8781-f0f69d6b15aehttp://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://pleroma.soykaf.com/objects/53dd40f4-3069-45a1-863b-94a9b317093dNew note by fortuneIn just seven days, I can make you a man!<br> -- The Rocky Horror Picture Show2017-05-05T02:10:02.9308022017-05-05T08:38:03.423539https://pleroma.soykaf.com/contexts/e8673466-9642-4c9e-8781-f0f69d6b15aehttps://pleroma.soykaf.com/users/fortunehttp://activitystrea.ms/schema/1.0/personhttps://pleroma.soykaf.com/users/fortunefortunefortuneThe trusty unix fortune filefortunehttp://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://pleroma.soykaf.com/objects/2bc86888-a256-4771-bb53-903f375804f9New note by lainRTs federating into pleroma now.2017-05-04T18:18:50.2764702017-05-04T18:18:50.276476https://pleroma.soykaf.com/contexts/b7ae9350-f317-48aa-8058-2668091bb280http://activitystrea.ms/schema/1.0/favoritehttps://pleroma.soykaf.com/activities/902b1f50-f295-4189-8c15-9c880919e121New favorite by lainlain favorited something2017-05-04T08:03:01.3088902017-05-04T08:03:01.308927http://activitystrea.ms/schema/1.0/notetag:gs.smuglo.li,2017-05-03:noticeId=2164642:objectType=commenthttps://pleroma.soykaf.com/contexts/9419f742-aaba-4eb5-89a2-8b599e8bf43chttp://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://pleroma.soykaf.com/objects/4e396e66-b063-454c-92c6-583506a9a2deNew note by lainClassic.<br><a href='https://pleroma.soykaf.com/media/adc36781-9765-4d9a-b57c-99b7a99108b2/mikodaemonstop.jpg'>https://pleroma.soykaf.com/media/adc36781-9765-4d9a-b57c-99b7a99108b2/mikodaemonstop.jpg</a>2017-05-04T07:59:45.1806192017-05-04T07:59:45.180628https://pleroma.soykaf.com/contexts/6afd9659-41e6-406d-ae97-43b880722861http://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://pleroma.soykaf.com/objects/85d183e9-c935-4655-a1e6-8d69a4108235New note by lainん？<br><a href='https://pleroma.soykaf.com/media/ab144c6d-a38c-4d35-a60b-9a998becc094/n.gif'>https://pleroma.soykaf.com/media/ab144c6d-a38c-4d35-a60b-9a998becc094/n.gif</a>2017-05-04T07:58:08.8107162017-05-04T07:58:08.810726https://pleroma.soykaf.com/contexts/2e1aa616-86ce-4b50-9c81-63045a972156http://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://pleroma.soykaf.com/objects/7c5c45bb-e4d9-4f72-b4c6-0314afbd3553New note by lainyeah.2017-05-04T07:55:17.3352902017-05-04T07:55:17.335299https://pleroma.soykaf.com/contexts/702c06cf-56ff-4a2f-bf5a-150bc00bb168http://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://pleroma.soykaf.com/objects/f33f5f54-1c1d-4462-b9ed-229bb635dfd8New note by lainyeah.2017-05-04T07:49:24.9314842017-05-04T07:49:24.931492https://pleroma.soykaf.com/contexts/c4932e7a-00cb-431a-b4ec-7404cb9daf65http://activitystrea.ms/schema/1.0/favoritehttps://pleroma.soykaf.com/activities/0709bc79-7ac5-4983-b6d0-2205bf5ceba3New favorite by lainlain favorited something2017-05-03T20:08:11.2945792017-05-03T20:08:11.294587http://activitystrea.ms/schema/1.0/notetag:pawoo.net,2017-05-03:objectId=7967690:objectType=Statushttps://pleroma.soykaf.com/contexts/07a4b34d-6255-4bb2-8c73-c295a09ac952http://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://pleroma.soykaf.com/objects/72c0288e-62d8-43d9-b3d8-1a9d78be8375New note by lain<a href='https://pawoo.net/users/God_Emperor_of_Dune'>@God_Emperor_of_Dune@pawoo.net</a> no man, just some fun domination play among buddies, nothing homo about it.2017-05-03T20:01:00.9983142017-05-03T20:01:00.998322https://pleroma.soykaf.com/contexts/07a4b34d-6255-4bb2-8c73-c295a09ac952http://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://pleroma.soykaf.com/objects/d846409e-cf2a-4b68-a149-d5de34a91b0dNew note by lain<a href='https://social.heldscal.la/user/24974'>@dtluna@social.heldscal.la</a> btfo.<br><a href='https://pleroma.soykaf.com/media/fbe42e87-5574-4544-89ba-29ddf46227fa/pnc__picked_media_1889ce61-4961-4fea-8a14-04fe6783ebf6.jpg'>https://pleroma.soykaf.com/media/fbe42e87-5574-4544-89ba-29ddf46227fa/pnc__picked_media_1889ce61-4961-4fea-8a14-04fe6783ebf6.jpg</a>2017-05-03T20:00:15.8609952017-05-03T20:00:15.861002https://pleroma.soykaf.com/contexts/0e88f35e-1a38-4181-bef9-5cbb0d943c63http://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://pleroma.soykaf.com/objects/9075265f-f3b2-40e8-809f-10714f05a1fdNew note by lain#nohomo <br><a href='https://pleroma.soykaf.com/media/5cc5ad91-d637-4c45-a691-5ea778dc1bb3/pnc__picked_media_f62dc9ae-ea23-4fe6-bf85-cb75a129ab34.jpg'>https://pleroma.soykaf.com/media/5cc5ad91-d637-4c45-a691-5ea778dc1bb3/pnc__picked_media_f62dc9ae-ea23-4fe6-bf85-cb75a129ab34.jpg</a>2017-05-03T19:50:38.5891062017-05-03T19:50:38.589113https://pleroma.soykaf.com/contexts/07a4b34d-6255-4bb2-8c73-c295a09ac952http://activitystrea.ms/schema/1.0/favoritehttps://pleroma.soykaf.com/activities/7924e992-0a95-40d9-8d17-7278c6c634c9New favorite by lainlain favorited something2017-05-03T18:32:59.2733752017-05-03T18:32:59.273382http://activitystrea.ms/schema/1.0/notetag:gs.smuglo.li,2017-05-03:noticeId=2164774:objectType=commenthttps://pleroma.soykaf.com/contexts/9419f742-aaba-4eb5-89a2-8b599e8bf43chttp://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://pleroma.soykaf.com/objects/569571ba-f54c-41b0-bde4-0fede54599f0New note by lain<a href='https://gs.smuglo.li/user/2'>@nepfag@gs.smuglo.li</a>@gs.smuglo.li I'll do proper subfolders soon, for now it's one per attachment + thumbs etc.2017-05-03T18:27:01.4499492017-05-03T18:27:01.449956https://pleroma.soykaf.com/contexts/9419f742-aaba-4eb5-89a2-8b599e8bf43chttp://activitystrea.ms/schema/1.0/activityhttp://activitystrea.ms/schema/1.0/sharehttps://pleroma.soykaf.com/activities/b6cc5d7c-0785-4785-a689-f1b05dc9b24dlain repeated a noticeRT <p><span class="h-card"><a href="https://pleroma.soykaf.com/users/lain" class="u-url mention">@<span>lain</span></a></span> Hey now!</p>2017-05-03T18:13:48.8910612017-05-03T18:13:48.891069https://pleroma.soykaf.com/contexts/ec6fdd27-0ec1-4672-8408-5a8e5a9c094bhttp://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posttag:mastodon.social,2017-05-01:objectId=4836142:objectType=StatusNew note by lambadalambda@mastodon.social<p><span class="h-card"><a href="https://pleroma.soykaf.com/users/lain" class="u-url mention">@<span>lain</span></a></span> Hey now!</p>2017-05-01T18:38:49.3653912017-05-03T18:13:48.934745https://pleroma.soykaf.com/contexts/ec6fdd27-0ec1-4672-8408-5a8e5a9c094bhttps://mastodon.social/users/lambadalambdahttp://activitystrea.ms/schema/1.0/personhttps://mastodon.social/users/lambadalambdalambadalambda@mastodon.socialCritical Valuenillambadalambda@mastodon.socialhttp://activitystrea.ms/schema/1.0/activityhttp://activitystrea.ms/schema/1.0/sharehttps://pleroma.soykaf.com/activities/3c09eb31-4ba8-4ff5-b4fa-8f6f74d58bf0lain repeated a noticeRT Haha, salmons from mastodon didn't work because it's not implementing conversation id...2017-05-03T18:13:15.1480412017-05-03T18:13:15.148049tag:social.heldscal.la,2017-05-01:objectType=thread:nonce=86cda6c734401d80http://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posttag:social.heldscal.la,2017-05-01:noticeId=2000425:objectType=noteNew note by lambadalambda@social.heldscal.laHaha, salmons from mastodon didn't work because it's not implementing conversation id...2017-05-01T18:39:36.2163772017-05-03T18:13:15.171143tag:social.heldscal.la,2017-05-01:objectType=thread:nonce=86cda6c734401d80https://social.heldscal.la/user/23211http://activitystrea.ms/schema/1.0/personhttps://social.heldscal.la/user/23211lambadalambda@social.heldscal.laConstance Variablenillambadalambda@social.heldscal.lahttp://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://pleroma.soykaf.com/objects/b8fc83d5-d7c0-4b5f-8976-0317b51935eaNew note by lain.<br><a href='https://pleroma.soykaf.com/media/563008a7-9a60-47ac-a263-22835729adf6/1492530528735.png'>https://pleroma.soykaf.com/media/563008a7-9a60-47ac-a263-22835729adf6/1492530528735.png</a>2017-05-03T18:12:50.7452412017-05-03T18:12:50.745249https://pleroma.soykaf.com/contexts/9419f742-aaba-4eb5-89a2-8b599e8bf43chttp://activitystrea.ms/schema/1.0/activityhttp://activitystrea.ms/schema/1.0/sharehttps://pleroma.soykaf.com/activities/ac93ecef-cde0-48e8-ae4b-19e3b94dbe30lain repeated a noticeRT Awright, which one of you hid my PENIS ENVY?2017-05-03T18:08:49.2310012017-05-03T18:08:49.235354https://pleroma.soykaf.com/contexts/a9132cf8-6afa-4dd8-8b29-7b6fcab623b8http://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://pleroma.soykaf.com/objects/04e15c66-4936-4930-a134-32841f088bcfNew note by fortuneAwright, which one of you hid my PENIS ENVY?2017-05-01T19:40:03.1699962017-05-03T18:08:49.285347https://pleroma.soykaf.com/contexts/a9132cf8-6afa-4dd8-8b29-7b6fcab623b8https://pleroma.soykaf.com/users/fortunehttp://activitystrea.ms/schema/1.0/personhttps://pleroma.soykaf.com/users/fortunefortunefortuneThe trusty unix fortune filefortunehttp://activitystrea.ms/schema/1.0/activityhttp://activitystrea.ms/schema/1.0/sharehttps://pleroma.soykaf.com/activities/54b10fa9-d602-4a0f-b659-e6d3f7bc8c4clain repeated a noticeRT He is a man capable of turning any colour into grey.<br> -- John LeCarre2017-05-03T17:44:47.5789842017-05-03T17:44:47.578996https://pleroma.soykaf.com/contexts/8aebc8e5-5352-4047-8b74-4098a5830ccahttp://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://pleroma.soykaf.com/objects/70ded299-184d-49cd-af17-23c0950536aaNew note by fortuneHe is a man capable of turning any colour into grey.<br> -- John LeCarre2017-05-02T08:40:03.4194652017-05-03T17:44:47.646192https://pleroma.soykaf.com/contexts/8aebc8e5-5352-4047-8b74-4098a5830ccahttps://pleroma.soykaf.com/users/fortunehttp://activitystrea.ms/schema/1.0/personhttps://pleroma.soykaf.com/users/fortunefortunefortuneThe trusty unix fortune filefortunehttp://activitystrea.ms/schema/1.0/activityhttp://activitystrea.ms/schema/1.0/sharehttps://pleroma.soykaf.com/activities/eff9fe49-8fc9-48e6-a1a0-921aa25c8118lain repeated a noticeRT The real trouble with women is that they have *all* the pussy.2017-05-03T17:30:22.5960372017-05-03T17:30:22.596048https://pleroma.soykaf.com/contexts/8c88c9df-4e40-4f54-b15f-c21848d1a8e2http://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://pleroma.soykaf.com/objects/0b9b008d-49eb-48a9-a18d-172ce7d01ea2New note by fortuneThe real trouble with women is that they have *all* the pussy.2017-05-02T12:10:03.6030862017-05-03T17:30:22.683141https://pleroma.soykaf.com/contexts/8c88c9df-4e40-4f54-b15f-c21848d1a8e2https://pleroma.soykaf.com/users/fortunehttp://activitystrea.ms/schema/1.0/personhttps://pleroma.soykaf.com/users/fortunefortunefortuneThe trusty unix fortune filefortunehttp://activitystrea.ms/schema/1.0/favoritehttps://pleroma.soykaf.com/activities/5d90bb26-ce23-4a5b-8dbd-651011780007New favorite by lainlain favorited something2017-05-03T17:28:20.9679262017-05-03T17:28:20.967935http://activitystrea.ms/schema/1.0/notetag:mastodon.social,2017-05-03:objectId=4952899:objectType=Statushttps://pleroma.soykaf.com/contexts/42701ab4-964a-441a-a372-f51bd183e441 \ No newline at end of file diff --git a/test/fixtures/tesla_mock/https___shitposter.club_api_statuses_show_2827873.atom.xml b/test/fixtures/tesla_mock/https___shitposter.club_api_statuses_show_2827873.atom.xml deleted file mode 100644 index 26fdebb49..000000000 --- a/test/fixtures/tesla_mock/https___shitposter.club_api_statuses_show_2827873.atom.xml +++ /dev/null @@ -1,54 +0,0 @@ - - http://activitystrea.ms/schema/1.0/comment - tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment - New comment by moonman - @<a href="https://shitposter.club/user/9655" class="h-card mention" title="Solidarity for Pigs">neimzr4luzerz</a> @<a href="https://gs.smuglo.li/user/2326" class="h-card mention" title="Dolus_McHonest">dolus</a> childhood poring over Strong's concordance and a koine Greek dictionary, fast forward to 2017 and some fuckstick who translates japanese jackoff material tells me you just need to make it sound right in English - - - http://activitystrea.ms/schema/1.0/post - 2017-05-05T08:51:48+00:00 - 2017-05-05T08:51:48+00:00 - - http://activitystrea.ms/schema/1.0/person - https://shitposter.club/user/1 - moonman - EMAIL:shitposterclub@gmail.com XMPP: moon@talk.shitposter.club Matrix Ed25519 fingerprint: 2HuDUTEz3iFN5N3xl6PYp9xZW/EWhgbbt78SrFy4w8o - - - - - - moonman - Generic Enemy - EMAIL:shitposterclub@gmail.com XMPP: moon@talk.shitposter.club Matrix Ed25519 fingerprint: 2HuDUTEz3iFN5N3xl6PYp9xZW/EWhgbbt78SrFy4w8o - - The Moon - - - homepage - https://shitposter.club/moonman - true - - - - - - - - tag:shitposter.club,2017-05-05:objectType=thread:nonce=3c16e9c2681f6d26 - - - - - https://shitposter.club/api/statuses/user_timeline/1.atom - Generic Enemy - - - - https://shitposter.club/avatar/1-96-20170503024316.jpeg - 2017-05-05T11:43:58+00:00 - - - - - diff --git a/test/fixtures/tesla_mock/https___shitposter.club_api_statuses_user_timeline_1.atom.xml b/test/fixtures/tesla_mock/https___shitposter.club_api_statuses_user_timeline_1.atom.xml deleted file mode 100644 index 31df7c2a6..000000000 --- a/test/fixtures/tesla_mock/https___shitposter.club_api_statuses_user_timeline_1.atom.xml +++ /dev/null @@ -1,454 +0,0 @@ - - - GNU social - https://shitposter.club/api/statuses/user_timeline/1.atom - moonman timeline - Updates from moonman on Shitposter Club! - https://shitposter.club/avatar/1-96-20170503024316.jpeg - 2017-05-05T13:24:09+00:00 - - http://activitystrea.ms/schema/1.0/person - https://shitposter.club/user/1 - moonman - EMAIL:shitposterclub@gmail.com XMPP: moon@talk.shitposter.club Matrix Ed25519 fingerprint: 2HuDUTEz3iFN5N3xl6PYp9xZW/EWhgbbt78SrFy4w8o - - - - - - moonman - Generic Enemy - EMAIL:shitposterclub@gmail.com XMPP: moon@talk.shitposter.club Matrix Ed25519 fingerprint: 2HuDUTEz3iFN5N3xl6PYp9xZW/EWhgbbt78SrFy4w8o - - The Moon - - - homepage - https://shitposter.club/moonman - true - - - - - - - - - - - - - - tag:shitposter.club,2017-05-05:subscription:1:person:23190:2017-05-05T11:43:58+00:00 - Generic Enemy (moonman)'s status on Friday, 05-May-2017 11:43:58 UTC - <a href="https://shitposter.club/moonman">Generic Enemy</a> started following <a href="https://noagendasocial.com/@Ma5on">Mason</a>. - - http://activitystrea.ms/schema/1.0/follow - 2017-05-05T11:43:58+00:00 - 2017-05-05T11:43:58+00:00 - - http://activitystrea.ms/schema/1.0/person - https://noagendasocial.com/users/Ma5on - Mason - - - - - - ma5on - Mason - - - tag:shitposter.club,2017-05-05:objectType=thread:nonce=abffa9c14a054d3b - - - - - - - tag:shitposter.club,2017-05-05:subscription:1:person:14357:2017-05-05T10:29:03+00:00 - Generic Enemy (moonman)'s status on Friday, 05-May-2017 10:29:03 UTC - <a href="https://shitposter.club/moonman">Generic Enemy</a> started following <a href="https://mastodon.cloud/@ohyran">Jens Reuterberg</a>. - - http://activitystrea.ms/schema/1.0/follow - 2017-05-05T10:29:03+00:00 - 2017-05-05T10:29:03+00:00 - - http://activitystrea.ms/schema/1.0/person - https://mastodon.cloud/users/ohyran - Jens Reuterberg - RPG-nerd, illustrator, Open Source enthusiast, KDE dude, designer and gay lefty. Might be a cliché - but we will soon find out! - - - - - - ohyran - Jens Reuterberg - RPG-nerd, illustrator, Open Source enthusiast, KDE dude, designer and gay lefty. Might be a cliché - but we will soon find out! - - - tag:shitposter.club,2017-05-05:objectType=thread:nonce=937151d4825a85bf - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:shitposter.club,2017-05-05:noticeId=2828637:objectType=note - New note by moonman - basicall i would just rather have ppl say &quot;i like x and y&quot; than &quot;i'm a nerd&quot; the term can be retired. - - - http://activitystrea.ms/schema/1.0/post - 2017-05-05T10:24:54+00:00 - 2017-05-05T10:24:54+00:00 - - tag:shitposter.club,2017-05-05:objectType=thread:nonce=65992b0b9b5e6931 - - - - - - - http://activitystrea.ms/schema/1.0/comment - tag:shitposter.club,2017-05-05:noticeId=2828579:objectType=comment - New comment by moonman - @<a href="https://gs.smuglo.li/user/35497" class="h-card mention" title="Bokuro Bokusawa">boco</a> to be honest i've turned right around and been cruel to other people, i said i'd never do it but it happens again eventually. - - - http://activitystrea.ms/schema/1.0/post - 2017-05-05T10:20:33+00:00 - 2017-05-05T10:20:33+00:00 - - - - tag:shitposter.club,2017-05-05:objectType=thread:nonce=c997fc73d7f8a8f0 - - - - - - - - http://activitystrea.ms/schema/1.0/comment - tag:shitposter.club,2017-05-05:noticeId=2828554:objectType=comment - New comment by moonman - @<a href="https://mastodon.cloud/users/ohyran" class="h-card mention" title="Jens Reuterberg">ohyran</a> i won't ever get over bullying but i agree otherwise. i don't go to comic shops too often these days but i got dragged to one last year and the sheer diversity of people enjoying comics now compared to years ago was striking and it pleased me. and i noticed a couple years ago because of youtube i find things i truly enjoy watching, like in-depth videos about electronic parts, didn't exist 20 years ago. it's pretty great. - - - http://activitystrea.ms/schema/1.0/post - 2017-05-05T10:18:10+00:00 - 2017-05-05T10:18:10+00:00 - - - - tag:shitposter.club,2017-05-05:objectType=thread:nonce=efae3a23b6e05767 - - - - - - - - tag:shitposter.club,2017-05-05:fave:1:comment:2828502:2017-05-05T10:12:52+00:00 - Favorite - moonman favorited something by ohyran: <p><span class="h-card"><a href="https://shitposter.club/moonman" class="u-url mention">@<span>moonman</span></a></span> fair enough - that distinction makes it clearer...</p><p>On the other hand - those of us who did "pay the price" of being nerdy little kids in the 80's and 90's should strive to get past it anyway (mental health wise not "just get over it") and see the "nerd culture" thing as a blessing of sorts. We are in the optimal spot to do it. (not saying that that is something easy btw just that NOW is the best of time to start talking about it)</p> - - http://activitystrea.ms/schema/1.0/favorite - 2017-05-05T10:12:52+00:00 - 2017-05-05T10:12:52+00:00 - - http://activitystrea.ms/schema/1.0/comment - tag:mastodon.cloud,2017-05-05:objectId=6334570:objectType=Status - New comment by ohyran - <p><span class="h-card"><a href="https://shitposter.club/moonman" class="u-url mention">@<span>moonman</span></a></span> fair enough - that distinction makes it clearer...</p><p>On the other hand - those of us who did "pay the price" of being nerdy little kids in the 80's and 90's should strive to get past it anyway (mental health wise not "just get over it") and see the "nerd culture" thing as a blessing of sorts. We are in the optimal spot to do it. (not saying that that is something easy btw just that NOW is the best of time to start talking about it)</p> - - - - - - - tag:shitposter.club,2017-05-05:objectType=thread:nonce=efae3a23b6e05767 - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:shitposter.club,2017-05-05:noticeId=2828496:objectType=note - New note by moonman - things are better now, a lot less kids in america get beaten up and called a fag. still too many. - - - http://activitystrea.ms/schema/1.0/post - 2017-05-05T10:11:31+00:00 - 2017-05-05T10:11:31+00:00 - - tag:shitposter.club,2017-05-05:objectType=thread:nonce=c997fc73d7f8a8f0 - - - - - - - http://activitystrea.ms/schema/1.0/comment - tag:shitposter.club,2017-05-05:noticeId=2828457:objectType=comment - New comment by moonman - @<a href="https://shitposter.club/user/21787" class="h-card mention" title="Yukari">cutscenes</a> @<a href="https://gs.smuglo.li/user/28250" class="h-card mention" title="Bricky">thatbrickster</a> @<a href="https://gs.smuglo.li/user/35497" class="h-card mention" title="Bokuro Bokusawa">boco</a> i never understood this because nerds had pocket protectors, which was a draftsman engineer thing and therefore smart, while geeks were people in carnivals who bit heads off small animals. - - - http://activitystrea.ms/schema/1.0/post - 2017-05-05T10:07:57+00:00 - 2017-05-05T10:07:57+00:00 - - - - tag:shitposter.club,2017-05-05:objectType=thread:nonce=efae3a23b6e05767 - - - - - - - - - - http://activitystrea.ms/schema/1.0/comment - tag:shitposter.club,2017-05-05:noticeId=2828435:objectType=comment - New comment by moonman - @<a href="https://mastodon.cloud/users/ohyran" class="h-card mention" title="Jens Reuterberg">ohyran</a> since i didn't specify i'm talking about people subjected to physical and psychological abuse and not people that are just mad that more people like comic books now. - - - http://activitystrea.ms/schema/1.0/post - 2017-05-05T10:05:07+00:00 - 2017-05-05T10:05:07+00:00 - - - - tag:shitposter.club,2017-05-05:objectType=thread:nonce=efae3a23b6e05767 - - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:shitposter.club,2017-05-05:noticeId=2828326:objectType=note - New note by moonman - if you were a &quot;nerd&quot; before, like, 2001 you have permanent excuse to hate this kind of shit.   <a href="https://shitposter.club/file/b79fa5644be0d6f22679136e67b7bf45c9c4a74a55c32dd2d0cf15de4ddd5be5.gif" title="https://shitposter.club/file/b79fa5644be0d6f22679136e67b7bf45c9c4a74a55c32dd2d0cf15de4ddd5be5.gif" class="attachment" id="attachment-662105" rel="nofollow external">https://shitposter.club/attachment/662105</a> - - - http://activitystrea.ms/schema/1.0/post - 2017-05-05T09:47:42+00:00 - 2017-05-05T09:47:42+00:00 - - tag:shitposter.club,2017-05-05:objectType=thread:nonce=efae3a23b6e05767 - - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:shitposter.club,2017-05-05:noticeId=2828250:objectType=note - New note by moonman - <a href="https://shitposter.club/file/1283e2d4dd8f96b8eeb5d9a16b318e210868aa11386cf0d593891e4c75c9126e.gif" title="https://shitposter.club/file/1283e2d4dd8f96b8eeb5d9a16b318e210868aa11386cf0d593891e4c75c9126e.gif" class="attachment" id="attachment-662098" rel="nofollow external">https://shitposter.club/attachment/662098</a> - - - http://activitystrea.ms/schema/1.0/post - 2017-05-05T09:39:06+00:00 - 2017-05-05T09:39:06+00:00 - - tag:shitposter.club,2017-05-05:objectType=thread:nonce=ea8ffae90546f0ab - - - - - - - - tag:shitposter.club,2017-05-05:fave:1:comment:2828161:2017-05-05T09:28:19+00:00 - Favorite - moonman favorited something by kro: @<a href="https://shitposter.club/user/1" class="h-card u-url p-nickname mention" title="Generic Enemy">moonman</a> Till Brooklyn? - - http://activitystrea.ms/schema/1.0/favorite - 2017-05-05T09:28:19+00:00 - 2017-05-05T09:28:19+00:00 - - http://activitystrea.ms/schema/1.0/comment - tag:gs.smuglo.li,2017-05-05:noticeId=2188587:objectType=comment - New comment by kro - @<a href="https://shitposter.club/user/1" class="h-card u-url p-nickname mention" title="Generic Enemy">moonman</a> Till Brooklyn? - - - - - - - tag:shitposter.club,2017-05-05:objectType=thread:nonce=d7aa6b5b057ca555 - - - - - - - tag:shitposter.club,2017-05-05:fave:1:comment:2828125:2017-05-05T09:24:56+00:00 - Favorite - moonman favorited something by hardbass2k8: this has obviously interesting implications in various places, for example:<br /> the nationalism of the nazis might not have been real, who would have thought?<br /> socialism is usually promoted to implementation by real douchebags!<br /> your local social justice people might want diversity but they don't want you, m/19, white, why?<br /> amateur soccer club, they want to be the best in the amateur league but actually they just get drunk after training and are 50% overweight.<br /> This is because humans are not capable of telepathy, so if you join a group it doesn't magically align every little bit of your being with the declared group goals.<br /> <br /> Even though you see unmanned group beliefs flying around from time to time, generally groups are created from a bunch of people. they are not a container for people, they are the people inside them.<br /> <br /> so if you see a group that appears to be cool don't think of it as cool because its goals are cool but because its members are cool. if they aren't, tough cookies. don't be the retard and end up on the camp watchtower. - - http://activitystrea.ms/schema/1.0/favorite - 2017-05-05T09:24:56+00:00 - 2017-05-05T09:24:56+00:00 - - http://activitystrea.ms/schema/1.0/comment - tag:shitposter.club,2017-05-05:noticeId=2828125:objectType=comment - New comment by hardbass2k8 - this has obviously interesting implications in various places, for example:<br /> the nationalism of the nazis might not have been real, who would have thought?<br /> socialism is usually promoted to implementation by real douchebags!<br /> your local social justice people might want diversity but they don't want you, m/19, white, why?<br /> amateur soccer club, they want to be the best in the amateur league but actually they just get drunk after training and are 50% overweight.<br /> This is because humans are not capable of telepathy, so if you join a group it doesn't magically align every little bit of your being with the declared group goals.<br /> <br /> Even though you see unmanned group beliefs flying around from time to time, generally groups are created from a bunch of people. they are not a container for people, they are the people inside them.<br /> <br /> so if you see a group that appears to be cool don't think of it as cool because its goals are cool but because its members are cool. if they aren't, tough cookies. don't be the retard and end up on the camp watchtower. - - - - - - - tag:shitposter.club,2017-05-05:objectType=thread:nonce=51b227fe92f6babf - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:shitposter.club,2017-05-05:noticeId=2828128:objectType=note - New note by moonman - In a valid remake of They live, signs would say REBEL, and DON'T GET MARRIED AND HAVE KIDS - - - http://activitystrea.ms/schema/1.0/post - 2017-05-05T09:24:23+00:00 - 2017-05-05T09:24:23+00:00 - - tag:shitposter.club,2017-05-05:objectType=thread:nonce=b74397fa766b82c9 - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:shitposter.club,2017-05-05:noticeId=2828104:objectType=note - New note by moonman - <a href="https://shitposter.club/file/4d34178bde99599f31a28928e1666fbd58448d8a22e94ed82222496e4a45cb07.gif" title="https://shitposter.club/file/4d34178bde99599f31a28928e1666fbd58448d8a22e94ed82222496e4a45cb07.gif" class="attachment" id="attachment-662049" rel="nofollow external">https://shitposter.club/attachment/662049</a> - - - http://activitystrea.ms/schema/1.0/post - 2017-05-05T09:21:01+00:00 - 2017-05-05T09:21:01+00:00 - - tag:shitposter.club,2017-05-05:objectType=thread:nonce=d7aa6b5b057ca555 - - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:shitposter.club,2017-05-05:noticeId=2828102:objectType=note - New note by moonman - when ppl find out i haven't always been serious  <a href="https://shitposter.club/file/5859fa95875342cc65dba0d852f726db158ce28198c326d5f13d9de7c0d2c449.gif" title="https://shitposter.club/file/5859fa95875342cc65dba0d852f726db158ce28198c326d5f13d9de7c0d2c449.gif" class="attachment" id="attachment-662053" rel="nofollow external">https://shitposter.club/attachment/662053</a> - - - http://activitystrea.ms/schema/1.0/post - 2017-05-05T09:20:45+00:00 - 2017-05-05T09:20:45+00:00 - - tag:shitposter.club,2017-05-05:objectType=thread:nonce=0a025ac5a570b4ec - - - - - - - - http://activitystrea.ms/schema/1.0/comment - tag:shitposter.club,2017-05-05:noticeId=2828086:objectType=comment - New comment by moonman - @<a href="https://shitposter.club/user/9655" class="h-card mention" title="Solidarity for Pigs">neimzr4luzerz</a> @<a href="https://gs.smuglo.li/user/2326" class="h-card mention" title="Dolus_McHonest">dolus</a> @<a href="https://gs.smuglo.li/user/35497" class="h-card mention" title="Bokuro Bokusawa">boco</a> you are being too serious lol - - - http://activitystrea.ms/schema/1.0/post - 2017-05-05T09:17:19+00:00 - 2017-05-05T09:17:19+00:00 - - - - tag:shitposter.club,2017-05-05:objectType=thread:nonce=3c16e9c2681f6d26 - - - - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:shitposter.club,2017-05-05:noticeId=2828085:objectType=note - New note by moonman - shitposter dot club  <a href="https://shitposter.club/file/9b084c7210b16abbf4d28594b924a07ef4a2a06f89d901a4c42fb1e243291263.gif" title="https://shitposter.club/file/9b084c7210b16abbf4d28594b924a07ef4a2a06f89d901a4c42fb1e243291263.gif" class="attachment" id="attachment-662047" rel="nofollow external">https://shitposter.club/attachment/662047</a> - - - http://activitystrea.ms/schema/1.0/post - 2017-05-05T09:16:50+00:00 - 2017-05-05T09:16:50+00:00 - - tag:shitposter.club,2017-05-05:objectType=thread:nonce=d1ae088a1b91e5e5 - - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:shitposter.club,2017-05-05:noticeId=2828061:objectType=note - New note by moonman - even when i lie i tell the truth, is that so hard to understand? - - - http://activitystrea.ms/schema/1.0/post - 2017-05-05T09:15:07+00:00 - 2017-05-05T09:15:07+00:00 - - tag:shitposter.club,2017-05-05:objectType=thread:nonce=a516e4b8506b8ef5 - - - - - - - http://activitystrea.ms/schema/1.0/comment - tag:shitposter.club,2017-05-05:noticeId=2828052:objectType=comment - New comment by moonman - @<a href="https://shitposter.club/user/9591" class="h-card mention" title="warum hei&#xDF;en deutschl&#xE4;nder deutschl&#xE4;nder">hardbass2k8</a> history, anthropology. - - - http://activitystrea.ms/schema/1.0/post - 2017-05-05T09:14:22+00:00 - 2017-05-05T09:14:22+00:00 - - - - tag:shitposter.club,2017-05-05:objectType=thread:nonce=fe4d7f35b13403ba - - - - - - - diff --git a/test/fixtures/tesla_mock/https___shitposter.club_notice_2827873.json b/test/fixtures/tesla_mock/https___shitposter.club_notice_2827873.json deleted file mode 100644 index 4b7b4df44..000000000 --- a/test/fixtures/tesla_mock/https___shitposter.club_notice_2827873.json +++ /dev/null @@ -1 +0,0 @@ -{"@context":["https://www.w3.org/ns/activitystreams","https://shitposter.club/schemas/litepub-0.1.jsonld",{"@language":"und"}],"actor":"https://shitposter.club/users/moonman","attachment":[],"attributedTo":"https://shitposter.club/users/moonman","cc":["https://shitposter.club/users/moonman/followers"],"content":"@neimzr4luzerz @dolus childhood poring over Strong's concordance and a koine Greek dictionary, fast forward to 2017 and some fuckstick who translates japanese jackoff material tells me you just need to make it sound right in English","context":"tag:shitposter.club,2017-05-05:objectType=thread:nonce=3c16e9c2681f6d26","conversation":"tag:shitposter.club,2017-05-05:objectType=thread:nonce=3c16e9c2681f6d26","id":"tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment","inReplyTo":"tag:shitposter.club,2017-05-05:noticeId=2827849:objectType=comment","inReplyToStatusId":2827849,"published":"2017-05-05T08:51:48Z","sensitive":false,"summary":null,"tag":[],"to":["https://www.w3.org/ns/activitystreams#Public"],"type":"Note"} \ No newline at end of file diff --git a/test/fixtures/tesla_mock/https___social.heldscal.la_api_statuses_user_timeline_23211.atom.xml b/test/fixtures/tesla_mock/https___social.heldscal.la_api_statuses_user_timeline_23211.atom.xml deleted file mode 100644 index 6cba5c28f..000000000 --- a/test/fixtures/tesla_mock/https___social.heldscal.la_api_statuses_user_timeline_23211.atom.xml +++ /dev/null @@ -1,591 +0,0 @@ - - - GNU social - https://social.heldscal.la/api/statuses/user_timeline/23211.atom - lambadalambda timeline - Updates from lambadalambda on social.heldscal.la! - https://social.heldscal.la/avatar/23211-96-20170416114255.jpeg - 2017-05-05T12:01:21+00:00 - - http://activitystrea.ms/schema/1.0/person - https://social.heldscal.la/user/23211 - lambadalambda - Call me Deacon Blues. - - - - - - lambadalambda - Constance Variable - Call me Deacon Blues. - - Berlin - - - homepage - https://heldscal.la - true - - - - - - - - - - - - - - tag:social.heldscal.la,2017-05-05:fave:23211:comment:2063249:2017-05-05T11:40:21+00:00 - Favorite - lambadalambda favorited something by tatiana: <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> they will start complaining about this, but won't come up with any solutions)</p> - - http://activitystrea.ms/schema/1.0/favorite - 2017-05-05T11:40:21+00:00 - 2017-05-05T11:40:21+00:00 - - http://activitystrea.ms/schema/1.0/comment - tag:social.weho.st,2017-05-05:objectId=172033:objectType=Status - New comment by tatiana - <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> they will start complaining about this, but won't come up with any solutions)</p> - - - - - - - tag:social.heldscal.la,2017-05-05:objectType=thread:nonce=e95b99adc050e198 - - - - - - - tag:social.heldscal.la,2017-05-05:fave:23211:comment:2063041:2017-05-05T11:27:28+00:00 - Favorite - lambadalambda favorited something by kat: @<a href="https://social.heldscal.la/lambadalambda" class="h-card mention" title="Constance Variable">lambadalambda</a> if the admin reading mine would delete a few it would be really useful in prioritising.  - - http://activitystrea.ms/schema/1.0/favorite - 2017-05-05T11:27:28+00:00 - 2017-05-05T11:27:28+00:00 - - http://activitystrea.ms/schema/1.0/comment - tag:quitter.se,2017-05-05:noticeId=11807959:objectType=comment - New comment by kat - @<a href="https://social.heldscal.la/lambadalambda" class="h-card mention" title="Constance Variable">lambadalambda</a> if the admin reading mine would delete a few it would be really useful in prioritising.  - - - - - - - tag:social.heldscal.la,2017-05-05:objectType=thread:nonce=e95b99adc050e198 - - - - - - - tag:social.heldscal.la,2017-05-05:noticeId=2062924:objectType=note - lambadalambda repeated a notice by nielsk - RT @nielsk @<a href="https://social.heldscal.la/user/23211" class="h-card u-url p-nickname mention" title="Constance Variable">lambadalambda</a> but there are soooo many, where should I start to read? - - http://activitystrea.ms/schema/1.0/share - 2017-05-05T11:09:37+00:00 - 2017-05-05T11:09:37+00:00 - - http://activitystrea.ms/schema/1.0/activity - tag:mastodon.social,2017-05-05:objectId=5024471:objectType=Status - - <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> but there are soooo many, where should I start to read?</p> - - http://activitystrea.ms/schema/1.0/post - 2017-05-05T11:05:18+00:00 - 2017-05-05T11:05:18+00:00 - - http://activitystrea.ms/schema/1.0/person - https://mastodon.social/users/nielsk - nielsk - Sysadmin by day and ehm… sysadmin by night. Besides that old video games, Japan, economics and some other stuff - - - - - - nielsk - nielsk - Sysadmin by day and ehm… sysadmin by night. Besides that old video games, Japan, economics and some other stuff - - - - http://activitystrea.ms/schema/1.0/comment - tag:mastodon.social,2017-05-05:objectId=5024471:objectType=Status - New comment by nielsk - <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> but there are soooo many, where should I start to read?</p> - - - - - - - tag:social.heldscal.la,2017-05-05:objectType=thread:nonce=e95b99adc050e198 - - - - https://mastodon.social/users/nielsk.atom - nielsk - - - https://social.heldscal.la/avatar/29849-96-20170428120041.jpeg - 2017-05-05T11:06:32+00:00 - - - - tag:social.heldscal.la,2017-05-05:objectType=thread:nonce=e95b99adc050e198 - - - - - - - tag:social.heldscal.la,2017-05-05:fave:23211:comment:2062875:2017-05-05T11:09:27+00:00 - Favorite - lambadalambda favorited something by nielsk: <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> but there are soooo many, where should I start to read?</p> - - http://activitystrea.ms/schema/1.0/favorite - 2017-05-05T11:09:27+00:00 - 2017-05-05T11:09:27+00:00 - - http://activitystrea.ms/schema/1.0/comment - tag:mastodon.social,2017-05-05:objectId=5024471:objectType=Status - New comment by nielsk - <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> but there are soooo many, where should I start to read?</p> - - - - - - - tag:social.heldscal.la,2017-05-05:objectType=thread:nonce=e95b99adc050e198 - - - - - - - tag:social.heldscal.la,2017-05-05:fave:23211:comment:2062863:2017-05-05T11:09:11+00:00 - Favorite - lambadalambda favorited something by kasil: <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> surely, google is not that evil !</p> - - http://activitystrea.ms/schema/1.0/favorite - 2017-05-05T11:09:11+00:00 - 2017-05-05T11:09:11+00:00 - - http://activitystrea.ms/schema/1.0/comment - tag:loutre.info,2017-05-05:objectId=23331:objectType=Status - New comment by kasil - <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> surely, google is not that evil !</p> - - - - - - - tag:social.heldscal.la,2017-05-05:objectType=thread:nonce=e95b99adc050e198 - - - - - - - http://activitystrea.ms/schema/1.0/comment - tag:social.heldscal.la,2017-05-05:noticeId=2062767:objectType=comment - New comment by lambadalambda - @<a href="https://sealion.club/user/4" class="h-card u-url p-nickname mention" title="dewoo &#x274E;">dwmatiz</a> dunno, probably. - - - http://activitystrea.ms/schema/1.0/post - 2017-05-05T10:55:17+00:00 - 2017-05-05T10:55:17+00:00 - - - - tag:social.heldscal.la,2017-05-05:objectType=thread:nonce=e95b99adc050e198 - - - - - - - - http://activitystrea.ms/schema/1.0/comment - tag:social.heldscal.la,2017-05-05:noticeId=2062705:objectType=comment - New comment by lambadalambda - @<a href="https://gs.smuglo.li/user/28250" class="h-card u-url p-nickname mention" title="Bricky">thatbrickster</a> I do it, too. - - - http://activitystrea.ms/schema/1.0/post - 2017-05-05T10:48:12+00:00 - 2017-05-05T10:48:12+00:00 - - - - tag:social.heldscal.la,2017-05-05:objectType=thread:nonce=e95b99adc050e198 - - - - - - - - http://activitystrea.ms/schema/1.0/comment - tag:social.heldscal.la,2017-05-05:noticeId=2062620:objectType=comment - New comment by lambadalambda - @<a href="https://social.tchncs.de/users/israuor" class="h-card u-url p-nickname mention" title="Israuor &#x2642;">israuor</a> @<a href="https://mastodon.gougere.fr/users/bortzmeyer" class="h-card u-url p-nickname mention" title="S. Bortzmeyer &#x2705;">bortzmeyer</a> so, 99%. 100% for 'normal' people. - - - http://activitystrea.ms/schema/1.0/post - 2017-05-05T10:38:52+00:00 - 2017-05-05T10:38:52+00:00 - - - - tag:social.heldscal.la,2017-05-05:objectType=thread:nonce=e95b99adc050e198 - - - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:social.heldscal.la,2017-05-05:noticeId=2062583:objectType=note - New note by lambadalambda - I wonder what'll happen when people realize the admin at their mail hoster can read all their e-mails. - - - http://activitystrea.ms/schema/1.0/post - 2017-05-05T10:35:45+00:00 - 2017-05-05T10:35:45+00:00 - - tag:social.heldscal.la,2017-05-05:objectType=thread:nonce=e95b99adc050e198 - - - - - - - tag:social.heldscal.la,2017-05-05:subscription:23211:person:35708:2017-05-05T09:34:46+00:00 - Constance Variable (lambadalambda@social.heldscal.la)'s status on Friday, 05-May-2017 09:34:46 UTC - <a href="https://social.heldscal.la/lambadalambda">Constance Variable</a> started following <a href="https://mastodon.social/@milouse">milouse</a>. - - http://activitystrea.ms/schema/1.0/follow - 2017-05-05T09:34:46+00:00 - 2017-05-05T09:34:46+00:00 - - http://activitystrea.ms/schema/1.0/person - https://mastodon.social/users/milouse - milouse - #Scout leader #sgdf, interested in #openweb, #semanticweb, #privacy, #foss and #socialeconomy. 0xA714ECAC8C9CEE3D - - - - - - milouse - milouse - #Scout leader #sgdf, interested in #openweb, #semanticweb, #privacy, #foss and #socialeconomy. 0xA714ECAC8C9CEE3D - - - tag:social.heldscal.la,2017-05-05:objectType=thread:nonce=26ca19a355bb6135 - - - - - - - tag:social.heldscal.la,2017-05-05:noticeId=2061871:objectType=note - lambadalambda repeated a notice by safebot - RT @<a href="https://gs.smuglo.li/user/25857" class="h-card u-url p-nickname mention" title="safebot">safebot</a> #<span class="tag"><a href="https://social.heldscal.la/tag/cheers" rel="tag">cheers</a></span> <a href="https://gs.smuglo.li/attachment/456444" title="https://gs.smuglo.li/attachment/456444" rel="nofollow external noreferrer" class="attachment" id="attachment-432334">https://gs.smuglo.li/attachment/456444</a> - - http://activitystrea.ms/schema/1.0/share - 2017-05-05T09:16:17+00:00 - 2017-05-05T09:16:17+00:00 - - http://activitystrea.ms/schema/1.0/activity - tag:gs.smuglo.li,2017-05-05:noticeId=2188073:objectType=note - - #<span class="tag"><a href="https://gs.smuglo.li/tag/cheers" rel="tag">cheers</a></span> <a href="https://gs.smuglo.li/file/5099e73c83da778cd032a721e96880f99a868b712be2975d08238547a5ba06c7.jpg" title="https://gs.smuglo.li/file/5099e73c83da778cd032a721e96880f99a868b712be2975d08238547a5ba06c7.jpg" rel="nofollow noreferrer" class="attachment">https://gs.smuglo.li/attachment/456444</a> - - http://activitystrea.ms/schema/1.0/post - 2017-05-05T08:36:53+00:00 - 2017-05-05T08:36:53+00:00 - - http://activitystrea.ms/schema/1.0/person - https://gs.smuglo.li/user/25857 - safebot - - - - - - safebot - safebot - - - - http://activitystrea.ms/schema/1.0/note - tag:gs.smuglo.li,2017-05-05:noticeId=2188073:objectType=note - New note by safebot - #<span class="tag"><a href="https://gs.smuglo.li/tag/cheers" rel="tag">cheers</a></span> <a href="https://gs.smuglo.li/file/5099e73c83da778cd032a721e96880f99a868b712be2975d08238547a5ba06c7.jpg" title="https://gs.smuglo.li/file/5099e73c83da778cd032a721e96880f99a868b712be2975d08238547a5ba06c7.jpg" rel="nofollow noreferrer" class="attachment">https://gs.smuglo.li/attachment/456444</a> - - - - - https://gs.smuglo.li/conversation/1009429 - - - - https://gs.smuglo.li/api/statuses/user_timeline/25857.atom - safebot - - - https://social.heldscal.la/avatar/25719-original-20161215233234.jpeg - 2017-05-05T12:00:57+00:00 - - - - https://gs.smuglo.li/conversation/1009429 - - - - - - - tag:social.heldscal.la,2017-05-05:fave:23211:comment:2061643:2017-05-05T09:12:50+00:00 - Favorite - lambadalambda favorited something by moonman: @<a href="https://shitposter.club/user/9655" class="h-card mention" title="Solidarity for Pigs">neimzr4luzerz</a> @<a href="https://gs.smuglo.li/user/2326" class="h-card mention" title="Dolus_McHonest">dolus</a> childhood poring over Strong's concordance and a koine Greek dictionary, fast forward to 2017 and some fuckstick who translates japanese jackoff material tells me you just need to make it sound right in English - - http://activitystrea.ms/schema/1.0/favorite - 2017-05-05T09:12:50+00:00 - 2017-05-05T09:12:50+00:00 - - http://activitystrea.ms/schema/1.0/comment - tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment - New comment by moonman - @<a href="https://shitposter.club/user/9655" class="h-card mention" title="Solidarity for Pigs">neimzr4luzerz</a> @<a href="https://gs.smuglo.li/user/2326" class="h-card mention" title="Dolus_McHonest">dolus</a> childhood poring over Strong's concordance and a koine Greek dictionary, fast forward to 2017 and some fuckstick who translates japanese jackoff material tells me you just need to make it sound right in English - - - - - - - tag:social.heldscal.la,2017-05-05:objectType=thread:nonce=55ead90125cd4bd4 - - - - - - - tag:social.heldscal.la,2017-05-05:fave:23211:comment:2061696:2017-05-05T09:06:10+00:00 - Favorite - lambadalambda favorited something by moonman: @<a href="https://shitposter.club/user/9655" class="h-card mention" title="Solidarity for Pigs">neimzr4luzerz</a> <br /> <span class="greentext">&gt; (((common era)))</span> - - http://activitystrea.ms/schema/1.0/favorite - 2017-05-05T09:06:10+00:00 - 2017-05-05T09:06:10+00:00 - - http://activitystrea.ms/schema/1.0/comment - tag:shitposter.club,2017-05-05:noticeId=2827918:objectType=comment - New comment by moonman - @<a href="https://shitposter.club/user/9655" class="h-card mention" title="Solidarity for Pigs">neimzr4luzerz</a> <br /> <span class="greentext">&gt; (((common era)))</span> - - - - - - - tag:social.heldscal.la,2017-05-05:objectType=thread:nonce=55ead90125cd4bd4 - - - - - - - tag:social.heldscal.la,2017-05-05:fave:23211:note:2061673:2017-05-05T08:58:28+00:00 - Favorite - lambadalambda favorited something by moonman: discussion is one thing but any argument I've heard over and over again for the last three decades is going to go unanswered. - - http://activitystrea.ms/schema/1.0/favorite - 2017-05-05T08:58:28+00:00 - 2017-05-05T08:58:28+00:00 - - http://activitystrea.ms/schema/1.0/note - tag:shitposter.club,2017-05-05:noticeId=2827895:objectType=note - New note by moonman - discussion is one thing but any argument I've heard over and over again for the last three decades is going to go unanswered. - - - - - - - https://shitposter.club/conversation/1390494 - - - - - - - tag:social.heldscal.la,2017-05-05:fave:23211:comment:2061280:2017-05-05T08:47:38+00:00 - Favorite - lambadalambda favorited something by moonman: @<a href="https://shitposter.club/user/9655" class="h-card mention" title="Solidarity for Pigs">neimzr4luzerz</a> sex is for procreation and as an expression of intimacy between commited couples, it is a sacramental act - - http://activitystrea.ms/schema/1.0/favorite - 2017-05-05T08:47:38+00:00 - 2017-05-05T08:47:38+00:00 - - http://activitystrea.ms/schema/1.0/comment - tag:shitposter.club,2017-05-05:noticeId=2827561:objectType=comment - New comment by moonman - @<a href="https://shitposter.club/user/9655" class="h-card mention" title="Solidarity for Pigs">neimzr4luzerz</a> sex is for procreation and as an expression of intimacy between commited couples, it is a sacramental act - - - - - - - tag:social.heldscal.la,2017-05-05:objectType=thread:nonce=55ead90125cd4bd4 - - - - - - - tag:social.heldscal.la,2017-05-05:fave:23211:note:2061535:2017-05-05T08:40:55+00:00 - Favorite - lambadalambda favorited something by fortune: What did Mickey Mouse get for Christmas?<br /> <br /> A Dan Quayle watch.<br /> <br /> -- heard from a Mike Dukakis field worker - - http://activitystrea.ms/schema/1.0/favorite - 2017-05-05T08:40:55+00:00 - 2017-05-05T08:40:55+00:00 - - http://activitystrea.ms/schema/1.0/note - tag:social.heldscal.la,2017-05-05:noticeId=2061535:objectType=note - New note by fortune - What did Mickey Mouse get for Christmas?<br /> <br /> A Dan Quayle watch.<br /> <br /> -- heard from a Mike Dukakis field worker - - - - - - - tag:social.heldscal.la,2017-05-05:objectType=thread:nonce=5185e5c145ee4762 - - - - - - - tag:social.heldscal.la,2017-05-05:fave:23211:comment:2061421:2017-05-05T08:36:27+00:00 - Favorite - lambadalambda favorited something by moonman: @<a href="https://maly.io/users/sonya" class="h-card mention" title="Sonya Mann ✅">sonya</a> banned from 4chan. you better watch ou. i'm trouble, y'hear? - - http://activitystrea.ms/schema/1.0/favorite - 2017-05-05T08:36:27+00:00 - 2017-05-05T08:36:27+00:00 - - http://activitystrea.ms/schema/1.0/comment - tag:shitposter.club,2017-05-05:noticeId=2827689:objectType=comment - New comment by moonman - @<a href="https://maly.io/users/sonya" class="h-card mention" title="Sonya Mann ✅">sonya</a> banned from 4chan. you better watch ou. i'm trouble, y'hear? - - - - - - - https://shitposter.club/conversation/1389345 - - - - - - - tag:social.heldscal.la,2017-05-05:fave:23211:comment:2061351:2017-05-05T08:28:03+00:00 - Favorite - lambadalambda favorited something by moonman: @<a href="https://social.heldscal.la/user/29138" class="h-card mention" title="Claes Wallin (韋嘉誠)">clacke</a> is that the sequel to Time Crisis - - http://activitystrea.ms/schema/1.0/favorite - 2017-05-05T08:28:03+00:00 - 2017-05-05T08:28:03+00:00 - - http://activitystrea.ms/schema/1.0/comment - tag:shitposter.club,2017-05-05:noticeId=2827630:objectType=comment - New comment by moonman - @<a href="https://social.heldscal.la/user/29138" class="h-card mention" title="Claes Wallin (韋嘉誠)">clacke</a> is that the sequel to Time Crisis - - - - - - - https://shitposter.club/conversation/1385528 - - - - - - - tag:social.heldscal.la,2017-05-05:fave:23211:comment:2061339:2017-05-05T08:21:05+00:00 - Favorite - lambadalambda favorited something by hardbass2k8: @<a href="https://social.heldscal.la/user/23211" class="h-card mention" title="Constance Variable">lambadalambda</a> pretty sure it's money laundering - - http://activitystrea.ms/schema/1.0/favorite - 2017-05-05T08:21:05+00:00 - 2017-05-05T08:21:05+00:00 - - http://activitystrea.ms/schema/1.0/comment - tag:shitposter.club,2017-05-05:noticeId=2827617:objectType=comment - New comment by hardbass2k8 - @<a href="https://social.heldscal.la/user/23211" class="h-card mention" title="Constance Variable">lambadalambda</a> pretty sure it's money laundering - - - - - - - https://shitposter.club/conversation/1387523 - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:social.heldscal.la,2017-05-05:noticeId=2061303:objectType=note - New note by lambadalambda - It's got tattoos, it's got a pierced hood<br /> It's got generation X<br /> It's got lesbians, and vitriol<br /> And sadomasochistic latex sex<br /> It's got Mighty Morphin' power brokers<br /> And Tanya Harding nude<br /> Macrobiotic lacto-vegan non-confrontational free range food<br /> It's got the handshake, peace talk, non-aggression pact<br /> A multicultural integration of segregated historical facts<br /> <br /> #<span class="tag"><a href="https://social.heldscal.la/tag/nsfw" rel="tag">nsfw</a></span> <a href="https://social.heldscal.la/file/61c13b99c92f40ec4865e7a3830da340b187e3de70d94b8da38fd2138bbede3a.jpg" title="https://social.heldscal.la/file/61c13b99c92f40ec4865e7a3830da340b187e3de70d94b8da38fd2138bbede3a.jpg" rel="nofollow external noreferrer" class="attachment" id="attachment-432199">https://social.heldscal.la/attachment/432199</a> <a href="https://social.heldscal.la/file/a88bba1a324da68ee2cfdbcd1c4cde60bd9553298244d6f81731270b71aa80df.jpg" title="https://social.heldscal.la/file/a88bba1a324da68ee2cfdbcd1c4cde60bd9553298244d6f81731270b71aa80df.jpg" rel="nofollow external noreferrer" class="attachment" id="attachment-432200">https://social.heldscal.la/attachment/432200</a> <a href="https://social.heldscal.la/file/887329a303250e73dc2eea06b1f0512fcac4b9d1b534068f03c45f00d5b21c39.jpg" title="https://social.heldscal.la/file/887329a303250e73dc2eea06b1f0512fcac4b9d1b534068f03c45f00d5b21c39.jpg" rel="nofollow external noreferrer" class="attachment" id="attachment-432201">https://social.heldscal.la/attachment/432201</a> <a href="https://social.heldscal.la/file/6d7a1ec15c1368c4c68810434d24da528606fcbccdd1da97b25affafeeb6ffda.jpg" title="https://social.heldscal.la/file/6d7a1ec15c1368c4c68810434d24da528606fcbccdd1da97b25affafeeb6ffda.jpg" rel="nofollow external noreferrer" class="attachment" id="attachment-432202">https://social.heldscal.la/attachment/432202</a> <a href="https://social.heldscal.la/file/2f55f2bb028eb9be744cc82b35a6b86b496d8c3924c700aff55a872ff11df54c.jpg" title="https://social.heldscal.la/file/2f55f2bb028eb9be744cc82b35a6b86b496d8c3924c700aff55a872ff11df54c.jpg" rel="nofollow external noreferrer" class="attachment" id="attachment-432203">https://social.heldscal.la/attachment/432203</a> - - - http://activitystrea.ms/schema/1.0/post - 2017-05-05T08:17:08+00:00 - 2017-05-05T08:17:08+00:00 - - tag:social.heldscal.la,2017-05-05:objectType=thread:nonce=bb6f4343036970e8 - - - - - - - - - - - - diff --git a/test/fixtures/tesla_mock/https___social.heldscal.la_api_statuses_user_timeline_29191.atom.xml b/test/fixtures/tesla_mock/https___social.heldscal.la_api_statuses_user_timeline_29191.atom.xml deleted file mode 100644 index f70fbc695..000000000 --- a/test/fixtures/tesla_mock/https___social.heldscal.la_api_statuses_user_timeline_29191.atom.xml +++ /dev/null @@ -1,719 +0,0 @@ - - - GNU social - https://social.heldscal.la/api/statuses/user_timeline/29191.atom - shp timeline - Updates from shp on social.heldscal.la! - https://social.heldscal.la/avatar/29191-96-20170421154949.jpeg - 2017-05-05T11:57:06+00:00 - - http://activitystrea.ms/schema/1.0/person - https://social.heldscal.la/user/29191 - shp - cofe - - - - - - shp - shp - cofe - - cofe - - - - - - - - - - - - - - tag:social.heldscal.la,2017-04-29:noticeId=1967657:objectType=note - shp repeated a notice by lain - RT @<a href="https://social.heldscal.la/user/37181" class="h-card u-url p-nickname mention" title="Lain Iwakura">lain</a> @<a href="https://social.heldscal.la/user/29191" class="h-card u-url p-nickname mention" title="shp">shp</a> @<a href="https://social.heldscal.la/user/23211" class="h-card u-url p-nickname mention">lambadalambda</a> cofe. - - http://activitystrea.ms/schema/1.0/share - 2017-04-29T18:19:34+00:00 - 2017-04-29T18:19:34+00:00 - - http://activitystrea.ms/schema/1.0/activity - https://pleroma.soykaf.com/activities/43d12c05-db3f-4f3d-bee1-d676f264490c - - <a href="https://pleroma.soykaf.com/users/shp">@shp</a> <a href="https://social.heldscal.la/user/23211">@lambadalambda@social.heldscal.la</a> cofe. - - http://activitystrea.ms/schema/1.0/post - 2017-04-29T18:14:36+00:00 - 2017-04-29T18:14:36+00:00 - - http://activitystrea.ms/schema/1.0/person - https://pleroma.soykaf.com/users/lain - lain - Test account - - - - - - lain - Lain Iwakura - Test account - - - - http://activitystrea.ms/schema/1.0/note - https://pleroma.soykaf.com/activities/43d12c05-db3f-4f3d-bee1-d676f264490c - New note by lain - <a href="https://pleroma.soykaf.com/users/shp">@shp</a> <a href="https://social.heldscal.la/user/23211">@lambadalambda@social.heldscal.la</a> cofe. - - - - - tag:social.heldscal.la,2017-04-29:objectType=thread:nonce=e0b75431888efdab - - - https://pleroma.soykaf.com/users/lain/feed.atom - Lain Iwakura - - - https://social.heldscal.la/avatar/43188-96-20170429172422.jpeg - 2017-05-05T08:38:03+00:00 - - - - tag:social.heldscal.la,2017-04-29:objectType=thread:nonce=e0b75431888efdab - - - - - - - tag:social.heldscal.la,2017-04-27:subscription:29191:person:29558:2017-04-27T17:26:37+00:00 - shp (shp@social.heldscal.la)'s status on Thursday, 27-Apr-2017 17:26:37 UTC - <a href="https://social.heldscal.la/shp">shp</a> started following <a href="https://gs.smuglo.li/kfist">KFist</a>. - - http://activitystrea.ms/schema/1.0/follow - 2017-04-27T17:26:37+00:00 - 2017-04-27T17:26:37+00:00 - - http://activitystrea.ms/schema/1.0/person - https://gs.smuglo.li/user/28051 - KFist - I stream thanks to @nepfag. I also drink, shitpost, and fly planes. I visited Japan and it changed my life. Do you love your station? - - - - - - kfist - KFist - I stream thanks to @nepfag. I also drink, shitpost, and fly planes. I visited Japan and it changed my life. Do you love your station? - - homepage - http://smuglo.li:8000/stream.m3u - true - - - - tag:social.heldscal.la,2017-04-27:objectType=thread:nonce=f766240d13ed9c2e - - - - - - - tag:social.heldscal.la,2017-04-27:noticeId=1933030:objectType=note - shp repeated a notice by shpbot - RT @<a href="https://gs.archae.me/user/4687" class="h-card u-url p-nickname mention" title="shpbot">shpbot</a> &gt;QuakeC - - http://activitystrea.ms/schema/1.0/share - 2017-04-27T17:21:10+00:00 - 2017-04-27T17:21:10+00:00 - - http://activitystrea.ms/schema/1.0/activity - tag:gs.archae.me,2017-04-27:noticeId=760881:objectType=note - - <span class='greentext'>&gt;QuakeC</span> - - http://activitystrea.ms/schema/1.0/post - 2017-04-27T17:15:13+00:00 - 2017-04-27T17:15:13+00:00 - - http://activitystrea.ms/schema/1.0/person - https://gs.archae.me/user/4687 - shpbot - - - - - - shpbot - shpbot - - - - http://activitystrea.ms/schema/1.0/note - tag:gs.archae.me,2017-04-27:noticeId=760881:objectType=note - New note by shpbot - <span class='greentext'>&gt;QuakeC</span> - - - - - https://gs.archae.me/conversation/318362 - - - https://gs.archae.me/api/statuses/user_timeline/4687.atom - shpbot - - - https://social.heldscal.la/avatar/31581-original-20170405170019.jpeg - 2017-05-05T11:45:08+00:00 - - - - https://gs.archae.me/conversation/318362 - - - - - - - tag:social.heldscal.la,2017-04-27:subscription:29191:person:23226:2017-04-27T17:20:48+00:00 - shp (shp@social.heldscal.la)'s status on Thursday, 27-Apr-2017 17:20:48 UTC - <a href="https://social.heldscal.la/shp">shp</a> started following <a href="http://quitter.se/taknamay">Internet Turtle Ⓐ 🏴 ✅</a>. - - http://activitystrea.ms/schema/1.0/follow - 2017-04-27T17:20:48+00:00 - 2017-04-27T17:20:48+00:00 - - http://activitystrea.ms/schema/1.0/person - http://quitter.se/user/115823 - Internet Turtle Ⓐ 🏴 ✅ - Scheme programmer, Novice esperantist, Spiritual naturalist - Will listen to your problems for free - XMPP: DarkDungeons94 at chatme.im - - - - - - taknamay - Internet Turtle Ⓐ 🏴 ✅ - Scheme programmer, Novice esperantist, Spiritual naturalist - Will listen to your problems for free - XMPP: DarkDungeons94 at chatme.im - - New Jersey, United States - - - homepage - https://quitter.se/taknamay - true - - - - tag:social.heldscal.la,2017-04-27:objectType=thread:nonce=a66b1fb22020c152 - - - - - - - tag:social.heldscal.la,2017-04-27:subscription:29191:person:29302:2017-04-27T17:20:33+00:00 - shp (shp@social.heldscal.la)'s status on Thursday, 27-Apr-2017 17:20:33 UTC - <a href="https://social.heldscal.la/shp">shp</a> started following <a href="https://icosahedron.website/@Trev">Chillidan Stormrave</a>. - - http://activitystrea.ms/schema/1.0/follow - 2017-04-27T17:20:33+00:00 - 2017-04-27T17:20:33+00:00 - - http://activitystrea.ms/schema/1.0/person - https://icosahedron.website/users/Trev - Trev Prime - web tech, music, ethics. radical individualist. kinda queer. love thy neighbor. always open for conversation. - - - - - - trev - Trev Prime - web tech, music, ethics. radical individualist. kinda queer. love thy neighbor. always open for conversation. - - - tag:social.heldscal.la,2017-04-27:objectType=thread:nonce=781c05bd64ad9520 - - - - - - - tag:social.heldscal.la,2017-04-27:subscription:29191:person:29367:2017-04-27T17:20:27+00:00 - shp (shp@social.heldscal.la)'s status on Thursday, 27-Apr-2017 17:20:27 UTC - <a href="https://social.heldscal.la/shp">shp</a> started following <a href="https://gs.kawa-kun.com/aya">射命丸 文</a>. - - http://activitystrea.ms/schema/1.0/follow - 2017-04-27T17:20:27+00:00 - 2017-04-27T17:20:27+00:00 - - http://activitystrea.ms/schema/1.0/person - https://gs.kawa-kun.com/user/4885 - 射命丸 文 - Traditional Reporter of Fantasy - - - - - - aya - 射命丸 文 - Traditional Reporter of Fantasy - - Gensōkyō - - - homepage - https://danbooru.donmai.us - true - - - - tag:social.heldscal.la,2017-04-27:objectType=thread:nonce=5921da7a934e47ca - - - - - - - tag:social.heldscal.la,2017-04-27:subscription:29191:person:27773:2017-04-27T17:20:18+00:00 - shp (shp@social.heldscal.la)'s status on Thursday, 27-Apr-2017 17:20:18 UTC - <a href="https://social.heldscal.la/shp">shp</a> started following <a href="https://gs.smuglo.li/japananon">JapanAnon</a>. - - http://activitystrea.ms/schema/1.0/follow - 2017-04-27T17:20:18+00:00 - 2017-04-27T17:20:18+00:00 - - http://activitystrea.ms/schema/1.0/person - https://gs.smuglo.li/user/27299 - JapanAnon - 匿名でしていてね！ - - - - - - japananon - JapanAnon - 匿名でしていてね！ - - ワイヤード - - - homepage - http://www.anonymous-japan.org - true - - - - tag:social.heldscal.la,2017-04-27:objectType=thread:nonce=ae3d819865886cba - - - - - - - tag:social.heldscal.la,2017-04-27:subscription:29191:person:36560:2017-04-27T17:19:30+00:00 - shp (shp@social.heldscal.la)'s status on Thursday, 27-Apr-2017 17:19:30 UTC - <a href="https://social.heldscal.la/shp">shp</a> started following <a href="https://shitposter.club/wareya">wareya</a>. - - http://activitystrea.ms/schema/1.0/follow - 2017-04-27T17:19:30+00:00 - 2017-04-27T17:19:30+00:00 - - http://activitystrea.ms/schema/1.0/person - https://shitposter.club/user/15439 - wareya - Who are you to defy such a perfect being that is the machine? 日本語難しいけど頑張るぜ github.com/wareya wareya.moe Short: reya or war, never "ware" - - - - - - wareya - wareya - Who are you to defy such a perfect being that is the machine? 日本語難しいけど頑張るぜ github.com/wareya wareya.moe Short: reya or war, never "ware" - - - tag:social.heldscal.la,2017-04-27:objectType=thread:nonce=bd88a3cd20b5a418 - - - - - - - tag:social.heldscal.la,2017-04-27:subscription:29191:person:41176:2017-04-27T17:19:21+00:00 - shp (shp@social.heldscal.la)'s status on Thursday, 27-Apr-2017 17:19:21 UTC - <a href="https://social.heldscal.la/shp">shp</a> started following <a href="https://hakui.club/takeshitakenji">竹下憲二 (白)</a>. - - http://activitystrea.ms/schema/1.0/follow - 2017-04-27T17:19:21+00:00 - 2017-04-27T17:19:21+00:00 - - http://activitystrea.ms/schema/1.0/person - https://hakui.club/user/6 - 竹下憲二 (白) - Oh boy. - - - - - - takeshitakenji - 竹下憲二 (白) - Oh boy. - - Seattle, WA - - - homepage - http://gs.kawa-kun.com - true - - - - tag:social.heldscal.la,2017-04-27:objectType=thread:nonce=b139a673deba6963 - - - - - - - tag:social.heldscal.la,2017-04-27:fave:29191:note:1932205:2017-04-27T17:17:46+00:00 - Favorite - shp favorited something by dolus: Looks like Merry is pussing out and caving to pressure. Sad. <a href="https://gs.smuglo.li/file/23e37de3c321248d3f322d8ec042372914568ab4c9431a94e568a61b8146587f.png" title="https://gs.smuglo.li/file/23e37de3c321248d3f322d8ec042372914568ab4c9431a94e568a61b8146587f.png" rel="nofollow noreferrer" class="attachment">https://gs.smuglo.li/attachment/432294</a> <a href="https://gs.smuglo.li/file/e5a9549a19986d59d51750090910f47c186787adf02b2b6ac58df37556887297.png" title="https://gs.smuglo.li/file/e5a9549a19986d59d51750090910f47c186787adf02b2b6ac58df37556887297.png" rel="nofollow noreferrer" class="attachment">https://gs.smuglo.li/attachment/432295</a> <a href="https://gs.smuglo.li/file/2fdfabbc8ab0b8dc135903a8c48c29b440d1f97446b98ced4ad14a54d3b5d41f.png" title="https://gs.smuglo.li/file/2fdfabbc8ab0b8dc135903a8c48c29b440d1f97446b98ced4ad14a54d3b5d41f.png" rel="nofollow noreferrer" class="attachment">https://gs.smuglo.li/attachment/432296</a> <a href="https://gs.smuglo.li/file/af605d7c6fe3a8c26c6d334c2a8e0005f7e86a266f14a5b3755e7d3ac4e226de.png" title="https://gs.smuglo.li/file/af605d7c6fe3a8c26c6d334c2a8e0005f7e86a266f14a5b3755e7d3ac4e226de.png" rel="nofollow noreferrer" class="attachment">https://gs.smuglo.li/attachment/432297</a> - - http://activitystrea.ms/schema/1.0/favorite - 2017-04-27T17:17:46+00:00 - 2017-04-27T17:17:46+00:00 - - http://activitystrea.ms/schema/1.0/note - tag:gs.smuglo.li,2017-04-27:noticeId=2065465:objectType=note - New note by dolus - Looks like Merry is pussing out and caving to pressure. Sad. <a href="https://gs.smuglo.li/file/23e37de3c321248d3f322d8ec042372914568ab4c9431a94e568a61b8146587f.png" title="https://gs.smuglo.li/file/23e37de3c321248d3f322d8ec042372914568ab4c9431a94e568a61b8146587f.png" rel="nofollow noreferrer" class="attachment">https://gs.smuglo.li/attachment/432294</a> <a href="https://gs.smuglo.li/file/e5a9549a19986d59d51750090910f47c186787adf02b2b6ac58df37556887297.png" title="https://gs.smuglo.li/file/e5a9549a19986d59d51750090910f47c186787adf02b2b6ac58df37556887297.png" rel="nofollow noreferrer" class="attachment">https://gs.smuglo.li/attachment/432295</a> <a href="https://gs.smuglo.li/file/2fdfabbc8ab0b8dc135903a8c48c29b440d1f97446b98ced4ad14a54d3b5d41f.png" title="https://gs.smuglo.li/file/2fdfabbc8ab0b8dc135903a8c48c29b440d1f97446b98ced4ad14a54d3b5d41f.png" rel="nofollow noreferrer" class="attachment">https://gs.smuglo.li/attachment/432296</a> <a href="https://gs.smuglo.li/file/af605d7c6fe3a8c26c6d334c2a8e0005f7e86a266f14a5b3755e7d3ac4e226de.png" title="https://gs.smuglo.li/file/af605d7c6fe3a8c26c6d334c2a8e0005f7e86a266f14a5b3755e7d3ac4e226de.png" rel="nofollow noreferrer" class="attachment">https://gs.smuglo.li/attachment/432297</a> - - - - - - - https://gs.smuglo.li/conversation/927473 - - - - - - - tag:social.heldscal.la,2017-04-27:fave:29191:note:1932492:2017-04-27T17:13:55+00:00 - Favorite - shp favorited something by zemichi: <a href="https://gs.smuglo.li/file/1d45ea4ffc95f15037f361b56ad6b89f8451b70ad1ff7a03b7bb0345b8e2227c.jpg" title="https://gs.smuglo.li/file/1d45ea4ffc95f15037f361b56ad6b89f8451b70ad1ff7a03b7bb0345b8e2227c.jpg" rel="nofollow noreferrer" class="attachment">https://gs.smuglo.li/attachment/432344</a><br /> that's a lot of loli - - http://activitystrea.ms/schema/1.0/favorite - 2017-04-27T17:13:55+00:00 - 2017-04-27T17:13:55+00:00 - - http://activitystrea.ms/schema/1.0/note - tag:gs.smuglo.li,2017-04-27:noticeId=2065713:objectType=note - New note by zemichi - <a href="https://gs.smuglo.li/file/1d45ea4ffc95f15037f361b56ad6b89f8451b70ad1ff7a03b7bb0345b8e2227c.jpg" title="https://gs.smuglo.li/file/1d45ea4ffc95f15037f361b56ad6b89f8451b70ad1ff7a03b7bb0345b8e2227c.jpg" rel="nofollow noreferrer" class="attachment">https://gs.smuglo.li/attachment/432344</a><br /> that's a lot of loli - - - - - - - https://gs.smuglo.li/conversation/927673 - - - - - - - tag:social.heldscal.la,2017-04-27:fave:29191:note:1932559:2017-04-27T17:12:46+00:00 - Favorite - shp favorited something by gsimg: <a href="https://gs.kawa-kun.com/file/3435c5cafda46f31cad5abb5837b3521b7b458198507073a496f4d10bad3633b.jpg" title="https://gs.kawa-kun.com/file/3435c5cafda46f31cad5abb5837b3521b7b458198507073a496f4d10bad3633b.jpg" rel="nofollow noreferrer" class="attachment">https://gs.kawa-kun.com/file/3435c5cafda46f31cad5abb5837b3521b7b458198507073a496f4d10bad3633b.jpg</a> #<span class="tag"><a href="https://gs.kawa-kun.com/tag/nsfw" rel="tag">nsfw</a></span> - - http://activitystrea.ms/schema/1.0/favorite - 2017-04-27T17:12:46+00:00 - 2017-04-27T17:12:46+00:00 - - http://activitystrea.ms/schema/1.0/note - tag:gs.kawa-kun.com,2017-04-27:noticeId=1608309:objectType=note - New note by gsimg - <a href="https://gs.kawa-kun.com/file/3435c5cafda46f31cad5abb5837b3521b7b458198507073a496f4d10bad3633b.jpg" title="https://gs.kawa-kun.com/file/3435c5cafda46f31cad5abb5837b3521b7b458198507073a496f4d10bad3633b.jpg" rel="nofollow noreferrer" class="attachment">https://gs.kawa-kun.com/file/3435c5cafda46f31cad5abb5837b3521b7b458198507073a496f4d10bad3633b.jpg</a> #<span class="tag"><a href="https://gs.kawa-kun.com/tag/nsfw" rel="tag">nsfw</a></span> - - - - - - - https://gs.kawa-kun.com/conversation/690817 - - - - - - - tag:social.heldscal.la,2017-04-27:fave:29191:note:1932601:2017-04-27T17:12:28+00:00 - Favorite - shp favorited something by zemichi: <a href="https://gs.smuglo.li/file/5d9114fafea7b9866c9d852bcfeaf66aade65ae26149758346bc5ade7e3fa8f0.jpg" title="https://gs.smuglo.li/file/5d9114fafea7b9866c9d852bcfeaf66aade65ae26149758346bc5ade7e3fa8f0.jpg" rel="nofollow noreferrer" class="attachment">https://gs.smuglo.li/attachment/432372</a> - - http://activitystrea.ms/schema/1.0/favorite - 2017-04-27T17:12:28+00:00 - 2017-04-27T17:12:28+00:00 - - http://activitystrea.ms/schema/1.0/note - tag:gs.smuglo.li,2017-04-27:noticeId=2065821:objectType=note - New note by zemichi - <a href="https://gs.smuglo.li/file/5d9114fafea7b9866c9d852bcfeaf66aade65ae26149758346bc5ade7e3fa8f0.jpg" title="https://gs.smuglo.li/file/5d9114fafea7b9866c9d852bcfeaf66aade65ae26149758346bc5ade7e3fa8f0.jpg" rel="nofollow noreferrer" class="attachment">https://gs.smuglo.li/attachment/432372</a> - - - - - - - https://gs.smuglo.li/conversation/927760 - - - - - - - tag:social.heldscal.la,2017-04-27:noticeId=1932867:objectType=note - shp repeated a notice by shpbot - RT @<a href="https://gs.archae.me/user/4687" class="h-card u-url p-nickname mention" title="shpbot">shpbot</a> <a href="https://shitposter.club/file/cbf7fbbee1127a9870e871305ca7de70f1eb1bbb79ffe5b3b0f33e37514d14d8.jpg" title="https://shitposter.club/file/cbf7fbbee1127a9870e871305ca7de70f1eb1bbb79ffe5b3b0f33e37514d14d8.jpg" rel="nofollow external noreferrer" class="attachment" id="attachment-237676">https://shitposter.club/file/cbf7fbbee1127a9870e871305ca7de70f1eb1bbb79ffe5b3b0f33e37514d14d8.jpg</a> #<span class="tag"><a href="https://social.heldscal.la/tag/2hu" rel="tag">2hu</a></span> #<span class="tag"><a href="https://social.heldscal.la/tag/ordinarymagician" rel="tag">ordinarymagician</a></span> :thinking: <a href="https://shitposter.club/file/abf3f82d9ce28d2293d858af26c75bb5d4fdd091c0d90ca7bc72ea7efba220e4.jpg" title="https://shitposter.club/file/abf3f82d9ce28d2293d858af26c75bb5d4fdd091c0d90ca7bc72ea7efba220e4.jpg" rel="nofollow external noreferrer" class="attachment" id="attachment-312306">https://shitposter.club/file/abf3f82d9ce28d2293d858af26c75bb5d4fdd091c0d90ca7bc72ea7efba220e4.jpg</a> - - http://activitystrea.ms/schema/1.0/share - 2017-04-27T17:11:35+00:00 - 2017-04-27T17:11:35+00:00 - - http://activitystrea.ms/schema/1.0/activity - tag:gs.archae.me,2017-04-27:noticeId=760830:objectType=note - - <a href="https://shitposter.club/file/cbf7fbbee1127a9870e871305ca7de70f1eb1bbb79ffe5b3b0f33e37514d14d8.jpg" title="https://shitposter.club/file/cbf7fbbee1127a9870e871305ca7de70f1eb1bbb79ffe5b3b0f33e37514d14d8.jpg" rel="nofollow noreferrer" class="attachment">https://shitposter.club/file/cbf7fbbee1127a9870e871305ca7de70f1eb1bbb79ffe5b3b0f33e37514d14d8.jpg</a> #<span class="tag"><a href="https://gs.archae.me/tag/2hu" rel="tag">2hu</a></span> #<span class="tag"><a href="https://gs.archae.me/tag/ordinarymagician" rel="tag">ordinarymagician</a></span> :thinking: <a href="https://shitposter.club/file/abf3f82d9ce28d2293d858af26c75bb5d4fdd091c0d90ca7bc72ea7efba220e4.jpg" title="https://shitposter.club/file/abf3f82d9ce28d2293d858af26c75bb5d4fdd091c0d90ca7bc72ea7efba220e4.jpg" rel="nofollow noreferrer" class="attachment">https://shitposter.club/file/abf3f82d9ce28d2293d858af26c75bb5d4fdd091c0d90ca7bc72ea7efba220e4.jpg</a> - - http://activitystrea.ms/schema/1.0/post - 2017-04-27T17:00:08+00:00 - 2017-04-27T17:00:08+00:00 - - http://activitystrea.ms/schema/1.0/person - https://gs.archae.me/user/4687 - shpbot - - - - - - shpbot - shpbot - - - - http://activitystrea.ms/schema/1.0/note - tag:gs.archae.me,2017-04-27:noticeId=760830:objectType=note - New note by shpbot - <a href="https://shitposter.club/file/cbf7fbbee1127a9870e871305ca7de70f1eb1bbb79ffe5b3b0f33e37514d14d8.jpg" title="https://shitposter.club/file/cbf7fbbee1127a9870e871305ca7de70f1eb1bbb79ffe5b3b0f33e37514d14d8.jpg" rel="nofollow noreferrer" class="attachment">https://shitposter.club/file/cbf7fbbee1127a9870e871305ca7de70f1eb1bbb79ffe5b3b0f33e37514d14d8.jpg</a> #<span class="tag"><a href="https://gs.archae.me/tag/2hu" rel="tag">2hu</a></span> #<span class="tag"><a href="https://gs.archae.me/tag/ordinarymagician" rel="tag">ordinarymagician</a></span> :thinking: <a href="https://shitposter.club/file/abf3f82d9ce28d2293d858af26c75bb5d4fdd091c0d90ca7bc72ea7efba220e4.jpg" title="https://shitposter.club/file/abf3f82d9ce28d2293d858af26c75bb5d4fdd091c0d90ca7bc72ea7efba220e4.jpg" rel="nofollow noreferrer" class="attachment">https://shitposter.club/file/abf3f82d9ce28d2293d858af26c75bb5d4fdd091c0d90ca7bc72ea7efba220e4.jpg</a> - - - - - https://gs.archae.me/conversation/318317 - - - - - https://gs.archae.me/api/statuses/user_timeline/4687.atom - shpbot - - - https://social.heldscal.la/avatar/31581-original-20170405170019.jpeg - 2017-05-05T11:45:08+00:00 - - - - https://gs.archae.me/conversation/318317 - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:social.heldscal.la,2017-04-27:noticeId=1932815:objectType=note - New note by shp - federation issues with SPC atm it seems - - - http://activitystrea.ms/schema/1.0/post - 2017-04-27T17:08:55+00:00 - 2017-04-27T17:08:55+00:00 - - tag:social.heldscal.la,2017-04-27:objectType=thread:nonce=645a13c841f51769 - - - - - - - tag:social.heldscal.la,2017-04-26:fave:29191:note:1907285:2017-04-26T06:59:07+00:00 - Favorite - shp favorited something by lambadalambda: Is this the most offensive video on the net? <a href="https://social.heldscal.la/file/4c34bfb81a8155c265031bc48f7e69c29eb0d2941c57daf63f80e17b0e2e5f47.webm" title="https://social.heldscal.la/file/4c34bfb81a8155c265031bc48f7e69c29eb0d2941c57daf63f80e17b0e2e5f47.webm" rel="nofollow noreferrer" class="attachment">https://social.heldscal.la/attachment/402251</a> - - http://activitystrea.ms/schema/1.0/favorite - 2017-04-26T06:59:07+00:00 - 2017-04-26T06:59:07+00:00 - - http://activitystrea.ms/schema/1.0/note - tag:social.heldscal.la,2017-04-26:noticeId=1907285:objectType=note - New note by lambadalambda - Is this the most offensive video on the net? <a href="https://social.heldscal.la/file/4c34bfb81a8155c265031bc48f7e69c29eb0d2941c57daf63f80e17b0e2e5f47.webm" title="https://social.heldscal.la/file/4c34bfb81a8155c265031bc48f7e69c29eb0d2941c57daf63f80e17b0e2e5f47.webm" rel="nofollow external noreferrer" class="attachment" id="attachment-402251">https://social.heldscal.la/attachment/402251</a> - - - - - - - tag:social.heldscal.la,2017-04-26:objectType=thread:nonce=07b02e1328f456af - - - - - - - tag:social.heldscal.la,2017-04-26:noticeId=1907951:objectType=note - shp repeated a notice by shpbot - RT @<a href="https://gs.archae.me/user/4687" class="h-card u-url p-nickname mention" title="shpbot">shpbot</a> <a href="https://shitposter.club/file/718db06b564841331c72f9df767f8c9459e20c4dddbf0d4e61cd08ecbee7739d.jpg" title="https://shitposter.club/file/718db06b564841331c72f9df767f8c9459e20c4dddbf0d4e61cd08ecbee7739d.jpg" rel="nofollow external noreferrer" class="attachment" id="attachment-346198">https://shitposter.club/file/718db06b564841331c72f9df767f8c9459e20c4dddbf0d4e61cd08ecbee7739d.jpg</a> - - http://activitystrea.ms/schema/1.0/share - 2017-04-26T06:58:19+00:00 - 2017-04-26T06:58:19+00:00 - - http://activitystrea.ms/schema/1.0/activity - tag:gs.archae.me,2017-04-26:noticeId=752596:objectType=note - - <a href="https://shitposter.club/file/718db06b564841331c72f9df767f8c9459e20c4dddbf0d4e61cd08ecbee7739d.jpg" title="https://shitposter.club/file/718db06b564841331c72f9df767f8c9459e20c4dddbf0d4e61cd08ecbee7739d.jpg" rel="nofollow noreferrer" class="attachment">https://shitposter.club/file/718db06b564841331c72f9df767f8c9459e20c4dddbf0d4e61cd08ecbee7739d.jpg</a> - - http://activitystrea.ms/schema/1.0/post - 2017-04-26T06:15:07+00:00 - 2017-04-26T06:15:07+00:00 - - http://activitystrea.ms/schema/1.0/person - https://gs.archae.me/user/4687 - shpbot - - - - - - shpbot - shpbot - - - - http://activitystrea.ms/schema/1.0/note - tag:gs.archae.me,2017-04-26:noticeId=752596:objectType=note - New note by shpbot - <a href="https://shitposter.club/file/718db06b564841331c72f9df767f8c9459e20c4dddbf0d4e61cd08ecbee7739d.jpg" title="https://shitposter.club/file/718db06b564841331c72f9df767f8c9459e20c4dddbf0d4e61cd08ecbee7739d.jpg" rel="nofollow noreferrer" class="attachment">https://shitposter.club/file/718db06b564841331c72f9df767f8c9459e20c4dddbf0d4e61cd08ecbee7739d.jpg</a> - - - - - https://gs.archae.me/conversation/314010 - - - https://gs.archae.me/api/statuses/user_timeline/4687.atom - shpbot - - - https://social.heldscal.la/avatar/31581-original-20170405170019.jpeg - 2017-05-05T11:45:08+00:00 - - - - https://gs.archae.me/conversation/314010 - - - - - - - tag:social.heldscal.la,2017-04-26:fave:29191:note:1907341:2017-04-26T06:58:16+00:00 - Favorite - shp favorited something by moonman: <a href="https://shitposter.club/file/1377b0894e983599c11e739e406243cabed9f8af7961a2550ecaf97e32de8e60.jpg" title="https://shitposter.club/file/1377b0894e983599c11e739e406243cabed9f8af7961a2550ecaf97e32de8e60.jpg" class="attachment" rel="nofollow">https://shitposter.club/attachment/630989</a> - - http://activitystrea.ms/schema/1.0/favorite - 2017-04-26T06:58:16+00:00 - 2017-04-26T06:58:16+00:00 - - http://activitystrea.ms/schema/1.0/note - tag:shitposter.club,2017-04-26:noticeId=2681941:objectType=note - New note by moonman - <a href="https://shitposter.club/file/1377b0894e983599c11e739e406243cabed9f8af7961a2550ecaf97e32de8e60.jpg" title="https://shitposter.club/file/1377b0894e983599c11e739e406243cabed9f8af7961a2550ecaf97e32de8e60.jpg" class="attachment" rel="nofollow">https://shitposter.club/attachment/630989</a> - - - - - - - https://shitposter.club/conversation/1300990 - - - - - - - tag:social.heldscal.la,2017-04-26:fave:29191:comment:1907412:2017-04-26T06:57:56+00:00 - Favorite - shp favorited something by lambadalambda: @<a href="https://gs.smuglo.li/user/2" class="h-card u-url p-nickname mention" title="nepfag">nepfag</a> <a href="https://cherubini.casa/why-i-shut-down-wizards-town-and-left-mastodon-6d4e631346b3?source=linkShare-89c2f851e979-1493184822&amp;gi=a6a47c5466a0" title="https://cherubini.casa/why-i-shut-down-wizards-town-and-left-mastodon-6d4e631346b3?source=linkShare-89c2f851e979-1493184822&amp;gi=a6a47c5466a0" rel="nofollow noreferrer" class="attachment">https://social.heldscal.la/url/402273</a> - - http://activitystrea.ms/schema/1.0/favorite - 2017-04-26T06:57:56+00:00 - 2017-04-26T06:57:56+00:00 - - http://activitystrea.ms/schema/1.0/comment - tag:social.heldscal.la,2017-04-26:noticeId=1907412:objectType=comment - New comment by lambadalambda - @<a href="https://gs.smuglo.li/user/2" class="h-card u-url p-nickname mention" title="nepfag">nepfag</a> <a href="https://cherubini.casa/why-i-shut-down-wizards-town-and-left-mastodon-6d4e631346b3?source=linkShare-89c2f851e979-1493184822&amp;gi=a6a47c5466a0" title="https://cherubini.casa/why-i-shut-down-wizards-town-and-left-mastodon-6d4e631346b3?source=linkShare-89c2f851e979-1493184822&amp;gi=a6a47c5466a0" rel="nofollow external noreferrer" class="attachment" id="attachment-402273">https://social.heldscal.la/url/402273</a> - - - - - - - tag:social.heldscal.la,2017-04-26:objectType=thread:nonce=85c21eda7aaa7259 - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:social.heldscal.la,2017-04-26:noticeId=1907942:objectType=note - New note by shp - #<span class="tag"><a href="https://social.heldscal.la/tag/cofe" rel="tag">cofe</a></span> time my friends <a href="https://social.heldscal.la/file/ec254b45b3a86ff74bc08bc7e065cb681d77cf7d4cedc9cdcf59e16adf311da3.png" title="https://social.heldscal.la/file/ec254b45b3a86ff74bc08bc7e065cb681d77cf7d4cedc9cdcf59e16adf311da3.png" rel="nofollow external noreferrer" class="attachment" id="attachment-402381">https://social.heldscal.la/attachment/402381</a> - - - http://activitystrea.ms/schema/1.0/post - 2017-04-26T06:57:18+00:00 - 2017-04-26T06:57:18+00:00 - - tag:social.heldscal.la,2017-04-26:objectType=thread:nonce=9c9d9373bccfaf70 - - - - - - - - diff --git a/test/fixtures/tesla_mock/sakamoto.atom b/test/fixtures/tesla_mock/sakamoto.atom deleted file mode 100644 index 648946795..000000000 --- a/test/fixtures/tesla_mock/sakamoto.atom +++ /dev/null @@ -1 +0,0 @@ -http://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://social.sakamoto.gq/objects/0ccc1a2c-66b0-4305-b23a-7f7f2b040056New note by eal<a href='https://shitposter.club/user/5381'>@shpuld</a> <a href='https://pleroma.hjkos.com/users/hj'>@hj</a> IM NOT GAY DAD2017-08-04T12:51:26.130592Z2017-08-04T12:51:26.130592Zhttps://pleroma.hjkos.com/contexts/53093c74-2100-4bf4-aac6-66d1973d03efhttps://social.sakamoto.gq/users/ealhttp://activitystrea.ms/schema/1.0/personhttps://social.sakamoto.gq/users/ealeal坂本（・ヮ・）eal \ No newline at end of file diff --git a/test/fixtures/tesla_mock/sakamoto_eal_feed.atom b/test/fixtures/tesla_mock/sakamoto_eal_feed.atom deleted file mode 100644 index 9340d9038..000000000 --- a/test/fixtures/tesla_mock/sakamoto_eal_feed.atom +++ /dev/null @@ -1 +0,0 @@ -https://social.sakamoto.gq/users/eal/feed.atomeal's timeline2017-08-04T14:19:12.683854https://social.sakamoto.gq/users/ealhttp://activitystrea.ms/schema/1.0/personhttps://social.sakamoto.gq/users/ealeal坂本（・ヮ・）ealhttp://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://social.sakamoto.gq/objects/b79a1721-23f3-45a5-9610-adb08c2afae5New note by ealHonestly, I like all smileys that are not emoji.2017-08-04T14:19:12.675999Z2017-08-04T14:19:12.675999Zhttps://social.sakamoto.gq/contexts/e05ede92-8db9-4963-8b8e-e71a5797d68fhttp://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://social.sakamoto.gq/objects/45475bf3-2dfc-4d9e-8eae-1f4f86f48982New note by ealThen again, I like all smileys/emoticons that are not emoji.<br>2017-08-04T14:19:10.113373Z2017-08-04T14:19:10.113373Zhttps://social.sakamoto.gq/contexts/852d1605-4dcb-4ba7-9ba4-dfc37ed62fbchttp://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://social.sakamoto.gq/objects/8f8fd6d6-cc63-40c6-a5d0-1c0e4f919368New note by ealI love the russian-style smiley.2017-08-04T14:18:30.478552Z2017-08-04T14:18:30.478552Zhttps://social.sakamoto.gq/contexts/852d1605-4dcb-4ba7-9ba4-dfc37ed62fbchttp://activitystrea.ms/schema/1.0/activityhttp://activitystrea.ms/schema/1.0/followhttps://social.sakamoto.gq/activities/6e69df95-f2ad-4b8e-af4a-e93ff93d64e1eal started following https://cybre.space/users/0x3Feal started following https://cybre.space/users/0x3F2017-08-04T14:17:24.942193Z2017-08-04T14:17:24.942193Zhttp://activitystrea.ms/schema/1.0/personhttps://cybre.space/users/0x3Fhttps://cybre.space/users/0x3Fhttp://activitystrea.ms/schema/1.0/activityhttp://activitystrea.ms/schema/1.0/followhttps://social.sakamoto.gq/activities/54c5e260-0185-4267-a2a6-f5dd9c76c2c9eal started following https://niu.moe/users/ryeeal started following https://niu.moe/users/rye2017-08-04T14:16:35.604739Z2017-08-04T14:16:35.604739Zhttp://activitystrea.ms/schema/1.0/personhttps://niu.moe/users/ryehttps://niu.moe/users/ryehttp://activitystrea.ms/schema/1.0/activityhttp://activitystrea.ms/schema/1.0/followhttps://social.sakamoto.gq/activities/092ca863-19a8-416c-85d7-d3f23b3c0203eal started following https://mastodon.xyz/users/rafudesueal started following https://mastodon.xyz/users/rafudesu2017-08-04T14:16:10.993429Z2017-08-04T14:16:10.993429Zhttp://activitystrea.ms/schema/1.0/personhttps://mastodon.xyz/users/rafudesuhttps://mastodon.xyz/users/rafudesuhttp://activitystrea.ms/schema/1.0/activityhttp://activitystrea.ms/schema/1.0/followhttps://social.sakamoto.gq/activities/be5cf702-b127-423b-a6be-5f78f01a4289eal started following https://gs.kawa-kun.com/user/2eal started following https://gs.kawa-kun.com/user/22017-08-04T14:15:41.804611Z2017-08-04T14:15:41.804611Zhttp://activitystrea.ms/schema/1.0/personhttps://gs.kawa-kun.com/user/2https://gs.kawa-kun.com/user/2http://activitystrea.ms/schema/1.0/activityhttp://activitystrea.ms/schema/1.0/followhttps://social.sakamoto.gq/activities/4951e2a1-9bae-4e87-8e98-e6d2f8a52338eal started following https://gs.kawa-kun.com/user/4885eal started following https://gs.kawa-kun.com/user/48852017-08-04T14:15:00.135352Z2017-08-04T14:15:00.135352Zhttp://activitystrea.ms/schema/1.0/personhttps://gs.kawa-kun.com/user/4885https://gs.kawa-kun.com/user/4885http://activitystrea.ms/schema/1.0/activityhttp://activitystrea.ms/schema/1.0/followhttps://social.sakamoto.gq/activities/cadf8745-b9ee-4f6c-af32-bfddb70e4607eal started following https://mastodon.social/users/Murassaeal started following https://mastodon.social/users/Murassa2017-08-04T14:14:36.339560Z2017-08-04T14:14:36.339560Zhttp://activitystrea.ms/schema/1.0/personhttps://mastodon.social/users/Murassahttps://mastodon.social/users/Murassahttp://activitystrea.ms/schema/1.0/activityhttp://activitystrea.ms/schema/1.0/followhttps://social.sakamoto.gq/activities/a52c9aab-f0e6-4ccb-8dd3-9f417e72a41ceal started following https://mastodon.social/users/rysiekeal started following https://mastodon.social/users/rysiek2017-08-04T14:13:04.061572Z2017-08-04T14:13:04.061572Zhttp://activitystrea.ms/schema/1.0/personhttps://mastodon.social/users/rysiekhttps://mastodon.social/users/rysiekhttp://activitystrea.ms/schema/1.0/activityhttp://activitystrea.ms/schema/1.0/followhttps://social.sakamoto.gq/activities/738bc887-4cca-4b36-8c86-2b54d4c54732eal started following https://mastodon.hasameli.com/users/munineal started following https://mastodon.hasameli.com/users/munin2017-08-04T14:12:10.514155Z2017-08-04T14:12:10.514155Zhttp://activitystrea.ms/schema/1.0/personhttps://mastodon.hasameli.com/users/muninhttps://mastodon.hasameli.com/users/muninhttp://activitystrea.ms/schema/1.0/activityhttp://activitystrea.ms/schema/1.0/followhttps://social.sakamoto.gq/activities/dc66ad5a-b776-4180-a8aa-e4c1bf7cb703eal started following https://cybre.space/users/nightpooleal started following https://cybre.space/users/nightpool2017-08-04T14:11:16.046148Z2017-08-04T14:11:16.046148Zhttp://activitystrea.ms/schema/1.0/personhttps://cybre.space/users/nightpoolhttps://cybre.space/users/nightpoolhttp://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://social.sakamoto.gq/objects/9c5c00d7-3ce4-4c11-b965-dc5c2bda86c5New note by eal<a href='https://mastodon.zombocloud.com/users/staticsafe'>@staticsafe</a> privet )))2017-08-04T14:10:08.812247Z2017-08-04T14:10:08.812247Zhttps://social.sakamoto.gq/contexts/12a33823-0327-4c1c-a591-850ea79331b5http://activitystrea.ms/schema/1.0/activityhttp://activitystrea.ms/schema/1.0/followhttps://social.sakamoto.gq/activities/49798053-1f40-4a71-ad33-106e90630863eal started following https://social.homunyan.com/users/animeirleal started following https://social.homunyan.com/users/animeirl2017-08-04T14:09:44.904792Z2017-08-04T14:09:44.904792Zhttp://activitystrea.ms/schema/1.0/personhttps://social.homunyan.com/users/animeirlhttps://social.homunyan.com/users/animeirlhttp://activitystrea.ms/schema/1.0/favoritehttps://social.sakamoto.gq/activities/2d83a1c5-70a6-45d3-9b84-59d6a70fbb17New favorite by ealeal favorited something2017-08-04T14:07:27.210044Z2017-08-04T14:07:27.210044Zhttp://activitystrea.ms/schema/1.0/notehttps://pleroma.soykaf.com/objects/b831e52f-4ed4-438e-95b4-888897f64f09https://pleroma.hjkos.com/contexts/3ed48205-1e72-4e19-a618-89a0d2ca811ehttp://activitystrea.ms/schema/1.0/favoritehttps://social.sakamoto.gq/activities/06d28bed-544a-496b-8414-1c6d439273b5New favorite by ealeal favorited something2017-08-04T14:05:37.280200Z2017-08-04T14:05:37.280200Zhttp://activitystrea.ms/schema/1.0/notetag:toot-lab.reclaim.technology,2017-08-04:objectId=1166030:objectType=Statustag:p2px.me,2017-08-04:objectType=thread:nonce=f8bfc4d13db6ce91http://activitystrea.ms/schema/1.0/activityhttp://activitystrea.ms/schema/1.0/followhttps://social.sakamoto.gq/activities/72bf19d4-9ad4-4b2f-9cd0-f0d70f4e931beal started following https://mstdn.jp/users/nullkaleal started following https://mstdn.jp/users/nullkal2017-08-04T14:05:04.148904Z2017-08-04T14:05:04.148904Zhttp://activitystrea.ms/schema/1.0/personhttps://mstdn.jp/users/nullkalhttps://mstdn.jp/users/nullkalhttp://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://social.sakamoto.gq/objects/b0e89515-7621-4e09-b23d-83e192324107New note by eal<a href='https://p2px.me/user/1'>@stitchxd</a> test also2017-08-04T14:04:38.699051Z2017-08-04T14:04:38.699051Ztag:p2px.me,2017-08-04:objectType=thread:nonce=f8bfc4d13db6ce91http://activitystrea.ms/schema/1.0/favoritehttps://social.sakamoto.gq/activities/d8d2006b-6b23-45d6-ba27-39d27587777dNew favorite by ealeal favorited something2017-08-04T14:04:32.106626Z2017-08-04T14:04:32.106626Zhttp://activitystrea.ms/schema/1.0/notetag:p2px.me,2017-08-04:noticeId=222109:objectType=notetag:p2px.me,2017-08-04:objectType=thread:nonce=f8bfc4d13db6ce91http://activitystrea.ms/schema/1.0/activityhttp://activitystrea.ms/schema/1.0/followhttps://social.sakamoto.gq/activities/cb9db95d-ec27-41fa-bebd-5375fc13acb9eal started following https://mastodon.social/users/Gargroneal started following https://mastodon.social/users/Gargron2017-08-04T14:04:04.325531Z2017-08-04T14:04:04.325531Zhttp://activitystrea.ms/schema/1.0/personhttps://mastodon.social/users/Gargronhttps://mastodon.social/users/Gargron \ No newline at end of file diff --git a/test/fixtures/tesla_mock/shp@pleroma.soykaf.com.feed b/test/fixtures/tesla_mock/shp@pleroma.soykaf.com.feed deleted file mode 100644 index b24ef7ab6..000000000 --- a/test/fixtures/tesla_mock/shp@pleroma.soykaf.com.feed +++ /dev/null @@ -1 +0,0 @@ -https://pleroma.soykaf.com/users/shp/feed.atomshp's timeline2017-09-14T08:31:48.911686https://pleroma.soykaf.com/users/shphttp://activitystrea.ms/schema/1.0/personhttps://pleroma.soykaf.com/users/shpshpshpcofeshphttp://activitystrea.ms/schema/1.0/activityhttp://activitystrea.ms/schema/1.0/followhttps://pleroma.soykaf.com/activities/0b5f5ef2-020a-4f9e-a92b-a2bf21224644shp started following https://pleroma.soykaf.com/users/goozshp started following https://pleroma.soykaf.com/users/gooz2017-09-14T08:31:48.911226Z2017-09-14T08:31:48.911226Zhttp://activitystrea.ms/schema/1.0/personhttps://pleroma.soykaf.com/users/goozhttps://pleroma.soykaf.com/users/goozhttp://activitystrea.ms/schema/1.0/activityhttp://activitystrea.ms/schema/1.0/followhttps://pleroma.soykaf.com/activities/d928b7f7-dc10-478c-859b-cd604770da60shp started following https://niu.moe/users/xiaoyongmaoshp started following https://niu.moe/users/xiaoyongmao2017-09-14T08:16:52.674253Z2017-09-14T08:16:52.674253Zhttp://activitystrea.ms/schema/1.0/personhttps://niu.moe/users/xiaoyongmaohttps://niu.moe/users/xiaoyongmaohttp://activitystrea.ms/schema/1.0/favoritehttps://pleroma.soykaf.com/activities/3f5089b3-f1e5-47b6-8bfe-a9c4a860e724New favorite by shpshp favorited something2017-09-14T08:12:18.213055Z2017-09-14T08:12:18.213055Zhttp://activitystrea.ms/schema/1.0/notehttps://mastodon.xyz/users/Azurolu/statuses/8346804tag:mastodon.xyz,2017-09-14:objectId=3669709:objectType=Conversationhttp://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://pleroma.soykaf.com/objects/0def9b19-6b0f-44e0-96b3-543fa06a4010New note by shp<a href='https://niu.moe/users/Pasty'>@Pasty</a> I love the peach<br><a href="https://pleroma.soykaf.com/media/7e8bd209-dbd4-481a-a62c-d302d68df16d/__hinanawi_tenshi_touhou_drawn_by_e_o__8c6824f52dd494f6026607570179265f.jpg" class='attachment'>__hinanawi_tenshi_touhou_drawn_…</a>2017-09-14T08:12:04.367142Z2017-09-14T08:12:04.367142Ztag:niu.moe,2017-09-14:objectId=1660781:objectType=Conversationhttp://activitystrea.ms/schema/1.0/favoritehttps://pleroma.soykaf.com/activities/a4170edf-d273-4b82-931d-662aaf3872f3New favorite by shpshp favorited something2017-09-14T08:10:26.205104Z2017-09-14T08:10:26.205104Zhttp://activitystrea.ms/schema/1.0/notehttps://niu.moe/users/NekoiNemo/statuses/3210992tag:niu.moe,2017-09-14:objectId=1660761:objectType=Conversationhttp://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://pleroma.soykaf.com/objects/c50c47a0-fac5-4781-a7e6-f20e7226d5fcNew note by shp<a href='https://freezepeach.xyz/user/3458'>@hakui</a> <a href='https://pleroma.soykaf.com/users/lain'>@lain</a> you guys are forgetting the pancakes jeez2017-09-14T08:09:30.088418Z2017-09-14T08:09:30.088418Zhttps://pleroma.soykaf.com/contexts/ac9c98ee-3eca-4b4b-9620-64b5e85e2623http://activitystrea.ms/schema/1.0/favoritehttps://pleroma.soykaf.com/activities/2af9f622-5986-483c-83a1-ac59a9035b50New favorite by shpshp favorited something2017-09-14T08:09:16.346235Z2017-09-14T08:09:16.346235Zhttp://activitystrea.ms/schema/1.0/notetag:freezepeach.xyz,2017-09-14:noticeId=3926191:objectType=commenthttps://pleroma.soykaf.com/contexts/ac9c98ee-3eca-4b4b-9620-64b5e85e2623http://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://pleroma.soykaf.com/objects/f52aad69-5828-4e0e-bb7b-f2f0869d3ff0New note by shp<a href='https://gs.smuglo.li/user/253'>@kro</a> I'll probs try some of those 2hu mangos2017-09-14T08:09:13.262835Z2017-09-14T08:09:13.262835Ztag:gs.smuglo.li,2017-09-14:objectType=thread:nonce=c4ac2016e07c4123http://activitystrea.ms/schema/1.0/favoritehttps://pleroma.soykaf.com/activities/35743658-efee-46cf-9cdf-487b95709cd5New favorite by shpshp favorited something2017-09-14T08:09:00.517534Z2017-09-14T08:09:00.517534Zhttp://activitystrea.ms/schema/1.0/notetag:gs.smuglo.li,2017-09-14:noticeId=4113226:objectType=commenttag:gs.smuglo.li,2017-09-14:objectType=thread:nonce=c4ac2016e07c4123http://activitystrea.ms/schema/1.0/favoritehttps://pleroma.soykaf.com/activities/22258ba8-58dc-4e09-b476-fe28d3307377New favorite by shpshp favorited something2017-09-14T08:08:38.087136Z2017-09-14T08:08:38.087136Zhttp://activitystrea.ms/schema/1.0/notehttps://pleroma.soykaf.com/objects/13d7809e-5dca-4117-8738-887759392f2chttps://pleroma.soykaf.com/contexts/ac9c98ee-3eca-4b4b-9620-64b5e85e2623http://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://pleroma.soykaf.com/objects/f56d640a-0dbd-48af-80b1-06d0dbd26774New note by shp<a href='https://social.sakamoto.gq/users/eal'>@eal</a> ...but neither does my phone<br><br>low brightness, very dark wallpaper (pic related, but even darker, couldn't find the actual version)<br><a href="https://pleroma.soykaf.com/media/6d1b8d57-80ae-41d6-bdea-58fea09ecdf4/phonewallpaper.png" class='attachment'>phonewallpaper.png</a>2017-09-14T08:07:23.081214Z2017-09-14T08:07:23.081214Zhttps://pleroma.soykaf.com/contexts/f4c5d56e-fc58-467b-a8a5-10515c012355http://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://pleroma.soykaf.com/objects/d313df1d-121c-4ab8-abd1-e6aedcf55cbdNew note by shp<a href='https://niu.moe/users/Pasty'>@Pasty</a> y-you too2017-09-14T07:55:26.153486Z2017-09-14T07:55:26.153486Ztag:niu.moe,2017-09-14:objectId=1660616:objectType=Conversationhttp://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://pleroma.soykaf.com/objects/7b642424-4edb-48cc-8711-1eafb4745269New note by shp<a href='https://social.sakamoto.gq/users/eal'>@eal</a> bothers me more when sleeping, wore one for nearly 2 years2017-09-14T07:54:53.449227Z2017-09-14T07:54:53.449227Zhttps://pleroma.soykaf.com/contexts/f4c5d56e-fc58-467b-a8a5-10515c012355http://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://pleroma.soykaf.com/objects/5bc1bff1-88c3-489d-8efd-7e4755690a18New note by shpquick test2017-09-14T07:54:09.045525Z2017-09-14T07:54:09.045525Zhttps://pleroma.soykaf.com/contexts/cd770c2a-408e-4895-988c-60319298f219http://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://pleroma.soykaf.com/objects/956f1fb5-6f2f-433e-ab71-7f732b76f4beNew note by shphad some trouble getting sleep last night. only used phone to check the time a few times (v essential to have a near-black wallpaper to not blind yourself when you do that). can't rember the last time I rolled in the bed for longer than an hour like that2017-09-14T07:51:23.557775Z2017-09-14T07:51:23.557775Zhttps://pleroma.soykaf.com/contexts/f4c5d56e-fc58-467b-a8a5-10515c012355http://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://pleroma.soykaf.com/objects/d935d9f2-ebc7-4ff2-b65a-fbf418a60935New note by shp<a href='https://gs.smuglo.li/user/253'>@kro</a> doesn't sound like a bad idea at all2017-09-14T07:49:55.702555Z2017-09-14T07:49:55.702555Ztag:gs.smuglo.li,2017-09-14:objectType=thread:nonce=c4ac2016e07c4123http://activitystrea.ms/schema/1.0/favoritehttps://pleroma.soykaf.com/activities/342c8803-ee16-487d-9488-a39d763073f6New favorite by shpshp favorited something2017-09-14T07:49:41.875840Z2017-09-14T07:49:41.875840Zhttp://activitystrea.ms/schema/1.0/notetag:anticapitalist.party,2017-09-14:objectId=3322865:objectType=Statustag:anticapitalist.party,2017-09-14:objectId=1251751:objectType=Conversationhttp://activitystrea.ms/schema/1.0/favoritehttps://pleroma.soykaf.com/activities/5d98a19b-dd55-4077-9841-142937c613adNew favorite by shpshp favorited something2017-09-14T07:49:30.584265Z2017-09-14T07:49:30.584265Zhttp://activitystrea.ms/schema/1.0/notetag:gs.smuglo.li,2017-09-14:noticeId=4113170:objectType=commenttag:gs.smuglo.li,2017-09-14:objectType=thread:nonce=c4ac2016e07c4123http://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://pleroma.soykaf.com/objects/fdf3626a-50ba-458b-9bf7-b5f2cfa505fcNew note by shp<a href='https://pleroma.hjkos.com/users/hj'>@hj</a> c time2017-09-14T07:48:52.805422Z2017-09-14T07:48:52.805422Zhttps://pleroma.hjkos.com/contexts/dc4a3a3e-d366-4c0c-8789-8a9bee3537d9http://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://pleroma.soykaf.com/objects/c7c8eb17-b669-4827-9fbc-90f1fc54e4b1New note by shp<a href='https://sunshinegardens.org/users/tbny'>@tbny</a> err.. mediterranean from finnish*2017-09-14T07:46:52.764234Z2017-09-14T07:46:52.764234Zhttps://pleroma.soykaf.com/contexts/ac9c98ee-3eca-4b4b-9620-64b5e85e2623 \ No newline at end of file diff --git a/test/fixtures/tesla_mock/spc_5381.atom b/test/fixtures/tesla_mock/spc_5381.atom deleted file mode 100644 index c3288e97b..000000000 --- a/test/fixtures/tesla_mock/spc_5381.atom +++ /dev/null @@ -1,438 +0,0 @@ - - - GNU social - https://shitposter.club/api/statuses/user_timeline/5381.atom - shpuld timeline - Updates from shpuld on Shitposter Club! - https://shitposter.club/avatar/5381-96-20171230093854.png - 2018-02-23T13:42:22+00:00 - - http://activitystrea.ms/schema/1.0/person - https://shitposter.club/user/5381 - shpuld - - - - - - shpuld - shp - - - - - - - - - - - - - tag:shitposter.club,2018-02-23:fave:5381:comment:7387801:2018-02-23T13:39:40+00:00 - Favorite - shpuld favorited something by mayuutann: <p><span class="h-card"><a href="https://freezepeach.xyz/hakui" class="u-url mention">@<span>hakui</span></a></span> <span class="h-card"><a href="https://gs.smuglo.li/histoire" class="u-url mention">@<span>histoire</span></a></span> <span class="h-card"><a href="https://shitposter.club/shpuld" class="u-url mention">@<span>shpuld</span></a></span> <a href="https://mstdn.io/media/_Ee-x91XN0udpfZVO_U" rel="nofollow"><span class="invisible">https://</span><span class="ellipsis">mstdn.io/media/_Ee-x91XN0udpfZ</span><span class="invisible">VO_U</span></a></p> - - http://activitystrea.ms/schema/1.0/favorite - 2018-02-23T13:39:40+00:00 - 2018-02-23T13:39:40+00:00 - - http://activitystrea.ms/schema/1.0/comment - https://mstdn.io/users/mayuutann/statuses/99574950785668071 - New comment by mayuutann - <p><span class="h-card"><a href="https://freezepeach.xyz/hakui" class="u-url mention">@<span>hakui</span></a></span> <span class="h-card"><a href="https://gs.smuglo.li/histoire" class="u-url mention">@<span>histoire</span></a></span> <span class="h-card"><a href="https://shitposter.club/shpuld" class="u-url mention">@<span>shpuld</span></a></span> <a href="https://mstdn.io/media/_Ee-x91XN0udpfZVO_U" rel="nofollow"><span class="invisible">https://</span><span class="ellipsis">mstdn.io/media/_Ee-x91XN0udpfZ</span><span class="invisible">VO_U</span></a></p> - - - - - - - https://freezepeach.xyz/conversation/4182511 - - - - - - - http://activitystrea.ms/schema/1.0/comment - tag:shitposter.club,2018-02-23:noticeId=7387723:objectType=comment - New comment by shpuld - @<a href="https://freezepeach.xyz/user/3458" class="h-card mention" title="&#x5FA1;&#x5712;&#x306F;&#x304F;&#x3044;">hakui</a> @<a href="https://pleroma.soykaf.com/users/lain" class="h-card mention" title="&#x2468; lain &#x2468;">lain</a> how naive~ - - - http://activitystrea.ms/schema/1.0/post - 2018-02-23T13:30:15+00:00 - 2018-02-23T13:30:15+00:00 - - - - tag:shitposter.club,2018-02-23:objectType=thread:nonce=2f09acf104aebfe3 - - - - - - - - - http://activitystrea.ms/schema/1.0/comment - tag:shitposter.club,2018-02-23:noticeId=7387703:objectType=comment - New comment by shpuld - @<a href="https://freezepeach.xyz/user/3458" class="h-card mention" title="&#x5FA1;&#x5712;&#x306F;&#x304F;&#x3044;">hakui</a> @<a href="https://pleroma.soykaf.com/users/lain" class="h-card mention" title="&#x2468; lain &#x2468;">lain</a> you expect anyone to believe that?? - - - http://activitystrea.ms/schema/1.0/post - 2018-02-23T13:28:08+00:00 - 2018-02-23T13:28:08+00:00 - - - - tag:shitposter.club,2018-02-23:objectType=thread:nonce=2f09acf104aebfe3 - - - - - - - - - http://activitystrea.ms/schema/1.0/comment - tag:shitposter.club,2018-02-23:noticeId=7387639:objectType=comment - New comment by shpuld - @<a href="https://mstdn.io/users/mayuutann" class="h-card mention" title="Mayutan&#x2615;">mayuutann</a> @<a href="https://freezepeach.xyz/user/3458" class="h-card mention" title="&#x5FA1;&#x5712;&#x306F;&#x304F;&#x3044;">hakui</a> pacyuri!! <a href="https://shitposter.club/file/eea140be45df3f993c4533026bf9a78fe8facd296d2fa0c6d02b2e347c5dc30e.jpg" title="https://shitposter.club/file/eea140be45df3f993c4533026bf9a78fe8facd296d2fa0c6d02b2e347c5dc30e.jpg" class="attachment" id="attachment-1589462" rel="nofollow external">https://shitposter.club/attachment/1589462</a> - - - http://activitystrea.ms/schema/1.0/post - 2018-02-23T13:20:38+00:00 - 2018-02-23T13:20:38+00:00 - - - - https://freezepeach.xyz/conversation/4183220 - - - - - - - - - - http://activitystrea.ms/schema/1.0/comment - tag:shitposter.club,2018-02-23:noticeId=7387611:objectType=comment - New comment by shpuld - @<a href="https://freezepeach.xyz/user/3458" class="h-card mention" title="&#x5FA1;&#x5712;&#x306F;&#x304F;&#x3044;">hakui</a> why is pacyu eating a pizza so cute - - - http://activitystrea.ms/schema/1.0/post - 2018-02-23T13:18:07+00:00 - 2018-02-23T13:18:07+00:00 - - - - https://freezepeach.xyz/conversation/4183220 - - - - - - - - tag:shitposter.club,2018-02-23:fave:5381:comment:7387600:2018-02-23T13:17:52+00:00 - Favorite - shpuld favorited something by mayuutann: <p><span class="h-card"><a href="https://shitposter.club/shpuld" class="u-url mention">@<span>shpuld</span></a></span> <span class="h-card"><a href="https://gs.smuglo.li/histoire" class="u-url mention">@<span>histoire</span></a></span> <span class="h-card"><a href="https://freezepeach.xyz/hakui" class="u-url mention">@<span>hakui</span></a></span> pichu! <a href="https://mstdn.io/media/Crv5eubz1KO0dgBEulI" rel="nofollow"><span class="invisible">https://</span><span class="ellipsis">mstdn.io/media/Crv5eubz1KO0dgB</span><span class="invisible">EulI</span></a></p> - - http://activitystrea.ms/schema/1.0/favorite - 2018-02-23T13:17:52+00:00 - 2018-02-23T13:17:52+00:00 - - http://activitystrea.ms/schema/1.0/comment - https://mstdn.io/users/mayuutann/statuses/99574863865459283 - New comment by mayuutann - <p><span class="h-card"><a href="https://shitposter.club/shpuld" class="u-url mention">@<span>shpuld</span></a></span> <span class="h-card"><a href="https://gs.smuglo.li/histoire" class="u-url mention">@<span>histoire</span></a></span> <span class="h-card"><a href="https://freezepeach.xyz/hakui" class="u-url mention">@<span>hakui</span></a></span> pichu! <a href="https://mstdn.io/media/Crv5eubz1KO0dgBEulI" rel="nofollow"><span class="invisible">https://</span><span class="ellipsis">mstdn.io/media/Crv5eubz1KO0dgB</span><span class="invisible">EulI</span></a></p> - - - - - - - https://freezepeach.xyz/conversation/4182511 - - - - - - - tag:shitposter.club,2018-02-23:fave:5381:comment:7387544:2018-02-23T13:12:43+00:00 - Favorite - shpuld favorited something by mayuutann: <p><span class="h-card"><a href="https://shitposter.club/shpuld" class="u-url mention">@<span>shpuld</span></a></span> wa~~i!! :blobcheer:</p> - - http://activitystrea.ms/schema/1.0/favorite - 2018-02-23T13:12:43+00:00 - 2018-02-23T13:12:43+00:00 - - http://activitystrea.ms/schema/1.0/comment - https://mstdn.io/users/mayuutann/statuses/99574840290947233 - New comment by mayuutann - <p><span class="h-card"><a href="https://shitposter.club/shpuld" class="u-url mention">@<span>shpuld</span></a></span> wa~~i!! :blobcheer:</p> - - - - - - - tag:shitposter.club,2018-02-23:objectType=thread:nonce=d05e2b056274c5ab - - - - - - - http://activitystrea.ms/schema/1.0/comment - tag:shitposter.club,2018-02-23:noticeId=7387555:objectType=comment - New comment by shpuld - @<a href="https://freezepeach.xyz/user/3458" class="h-card mention" title="&#x5FA1;&#x5712;&#x306F;&#x304F;&#x3044;">hakui</a> more!! - - - http://activitystrea.ms/schema/1.0/post - 2018-02-23T13:12:23+00:00 - 2018-02-23T13:12:23+00:00 - - - - https://freezepeach.xyz/conversation/4183220 - - - - - - - - tag:shitposter.club,2018-02-23:fave:5381:note:7387537:2018-02-23T13:12:19+00:00 - Favorite - shpuld favorited something by hakui: you have pacyupacyu'd for: 45 minutes 03 seconds - - http://activitystrea.ms/schema/1.0/favorite - 2018-02-23T13:12:19+00:00 - 2018-02-23T13:12:19+00:00 - - http://activitystrea.ms/schema/1.0/note - tag:freezepeach.xyz,2018-02-23:noticeId=6451332:objectType=note - New note by hakui - you have pacyupacyu'd for: 45 minutes 03 seconds - - - - - - - https://freezepeach.xyz/conversation/4183220 - - - - - - - http://activitystrea.ms/schema/1.0/comment - tag:shitposter.club,2018-02-23:noticeId=7387539:objectType=comment - New comment by shpuld - @<a href="https://mstdn.io/users/mayuutann" class="h-card mention" title="Mayutan&#x2615;">mayuutann</a> ndndnd~ - - - http://activitystrea.ms/schema/1.0/post - 2018-02-23T13:11:04+00:00 - 2018-02-23T13:11:04+00:00 - - - - tag:shitposter.club,2018-02-23:objectType=thread:nonce=d05e2b056274c5ab - - - - - - - - http://activitystrea.ms/schema/1.0/comment - tag:shitposter.club,2018-02-23:noticeId=7387518:objectType=comment - New comment by shpuld - @<a href="https://mstdn.io/users/mayuutann" class="h-card mention" title="Mayutan&#x2615;">mayuutann</a> well done! mayumayu is so energetic - - - http://activitystrea.ms/schema/1.0/post - 2018-02-23T13:08:50+00:00 - 2018-02-23T13:08:50+00:00 - - - - tag:shitposter.club,2018-02-23:objectType=thread:nonce=d05e2b056274c5ab - - - - - - - - tag:shitposter.club,2018-02-23:fave:5381:note:7387503:2018-02-23T13:08:00+00:00 - Favorite - shpuld favorited something by mayuutann: <p>done with FIGURE MAT!!<br /> (Posted with IFTTT)</p> - - http://activitystrea.ms/schema/1.0/favorite - 2018-02-23T13:08:00+00:00 - 2018-02-23T13:08:00+00:00 - - http://activitystrea.ms/schema/1.0/note - https://mstdn.io/users/mayuutann/statuses/99574825526201897 - New note by mayuutann - <p>done with FIGURE MAT!!<br /> (Posted with IFTTT)</p> - - - - - - - tag:shitposter.club,2018-02-23:objectType=thread:nonce=c6aaa9b91e8d242f - - - - - - - http://activitystrea.ms/schema/1.0/comment - tag:shitposter.club,2018-02-23:noticeId=7387486:objectType=comment - New comment by shpuld - @<a href="https://freezepeach.xyz/user/3458" class="h-card mention" title="&#x5FA1;&#x5712;&#x306F;&#x304F;&#x3044;">hakui</a> @<a href="https://a.weirder.earth/users/mutstd" class="h-card mention" title="Mutant Standard">mutstd</a> @<a href="https://donphan.social/users/Siphonay" class="h-card mention" title="Siphonay">siphonay</a> jokes on you I'm oppressively shitposting myself - - - http://activitystrea.ms/schema/1.0/post - 2018-02-23T13:05:44+00:00 - 2018-02-23T13:05:44+00:00 - - - - tag:shitposter.club,2018-02-23:objectType=thread:nonce=5d306467336c9661 - - - - - - - - - - http://activitystrea.ms/schema/1.0/comment - tag:shitposter.club,2018-02-23:noticeId=7387466:objectType=comment - New comment by shpuld - @<a href="https://freezepeach.xyz/user/3458" class="h-card mention" title="&#x5FA1;&#x5712;&#x306F;&#x304F;&#x3044;">hakui</a> @<a href="https://a.weirder.earth/users/mutstd" class="h-card mention" title="Mutant Standard">mutstd</a> @<a href="https://donphan.social/users/Siphonay" class="h-card mention" title="Siphonay">siphonay</a> how does it feel being hostile - - - http://activitystrea.ms/schema/1.0/post - 2018-02-23T13:04:10+00:00 - 2018-02-23T13:04:10+00:00 - - - - tag:shitposter.club,2018-02-23:objectType=thread:nonce=5d306467336c9661 - - - - - - - - - - http://activitystrea.ms/schema/1.0/comment - tag:shitposter.club,2018-02-23:noticeId=7387459:objectType=comment - New comment by shpuld - @<a href="https://freezepeach.xyz/user/3458" class="h-card mention" title="&#x5FA1;&#x5712;&#x306F;&#x304F;&#x3044;">hakui</a> gorogoro - - - http://activitystrea.ms/schema/1.0/post - 2018-02-23T13:03:32+00:00 - 2018-02-23T13:03:32+00:00 - - - - https://freezepeach.xyz/conversation/4181784 - - - - - - - - http://activitystrea.ms/schema/1.0/comment - tag:shitposter.club,2018-02-23:noticeId=7387432:objectType=comment - New comment by shpuld - @<a href="https://freezepeach.xyz/user/3458" class="h-card mention" title="&#x5FA1;&#x5712;&#x306F;&#x304F;&#x3044;">hakui</a> ndnd - - - http://activitystrea.ms/schema/1.0/post - 2018-02-23T13:02:05+00:00 - 2018-02-23T13:02:05+00:00 - - - - https://freezepeach.xyz/conversation/4181784 - - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:shitposter.club,2018-02-23:noticeId=7387367:objectType=note - New note by shpuld - dear diary: I'm trying to do work but I can only think of tenshi eating a corndog - - - http://activitystrea.ms/schema/1.0/post - 2018-02-23T12:56:03+00:00 - 2018-02-23T12:56:03+00:00 - - tag:shitposter.club,2018-02-23:objectType=thread:nonce=57f316da416743fc - - - - - - - http://activitystrea.ms/schema/1.0/note - tag:shitposter.club,2018-02-23:noticeId=7387354:objectType=note - New note by shpuld - jesus christ it's such a fridey at work - - - http://activitystrea.ms/schema/1.0/post - 2018-02-23T12:53:50+00:00 - 2018-02-23T12:53:50+00:00 - - tag:shitposter.club,2018-02-23:objectType=thread:nonce=c05eb5e91bdcbdb7 - - - - - - - http://activitystrea.ms/schema/1.0/comment - tag:shitposter.club,2018-02-23:noticeId=7387343:objectType=comment - New comment by shpuld - @<a href="https://gs.smuglo.li/user/589" class="h-card mention" title="&#x16DE;&#x16A9;&#x16B3;&#x16C1;&#x16DE;&#x16A9;&#x16B3;&#x16C1;">dokidoki</a> give them free upgrades to krokodil - - - http://activitystrea.ms/schema/1.0/post - 2018-02-23T12:53:15+00:00 - 2018-02-23T12:53:15+00:00 - - - - https://gs.smuglo.li/conversation/3934774 - - - - - - - diff --git a/test/fixtures/unfollow.xml b/test/fixtures/unfollow.xml deleted file mode 100644 index 7a8f8fd2e..000000000 --- a/test/fixtures/unfollow.xml +++ /dev/null @@ -1,68 +0,0 @@ - - - GNU social - https://social.heldscal.la/api/statuses/user_timeline/23211.atom - lambadalambda timeline - Updates from lambadalambda on social.heldscal.la! - https://social.heldscal.la/avatar/23211-96-20170416114255.jpeg - 2017-05-07T09:54:49+00:00 - - http://activitystrea.ms/schema/1.0/person - https://social.heldscal.la/user/23211 - lambadalambda - Call me Deacon Blues. - - - - - - lambadalambda - Constance Variable - Call me Deacon Blues. - - Berlin - - - homepage - https://heldscal.la - true - - - - - - - - - - - - - undo:tag:social.heldscal.la,2017-05-07:subscription:23211:person:44803:2017-05-07T09:54:48+00:00 - Constance Variable (lambadalambda@social.heldscal.la)'s status on Sunday, 07-May-2017 09:54:49 UTC - <a href="https://social.heldscal.la/lambadalambda">Constance Variable</a> stopped following <a href="https://pawoo.net/@pekorino">ｍｏｎｏ</a>. - - http://activitystrea.ms/schema/1.0/unfollow - 2017-05-07T09:54:49+00:00 - 2017-05-07T09:54:49+00:00 - - http://activitystrea.ms/schema/1.0/person - https://pawoo.net/users/pekorino - ｍｏｎｏ - http://shitposter.club/mono 孤独のグルメ - - - - - pekorino - ｍｏｎｏ - http://shitposter.club/mono 孤独のグルメ - - - tag:social.heldscal.la,2017-05-07:objectType=thread:nonce=6e80caf94e03029f - - - - - - diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex index a0ebf65d9..344e27f13 100644 --- a/test/support/http_request_mock.ex +++ b/test/support/http_request_mock.ex @@ -103,14 +103,6 @@ def get( }} end - def get("https://mastodon.social/users/emelie.atom", _, _, _) do - {:ok, - %Tesla.Env{ - status: 200, - body: File.read!("test/fixtures/tesla_mock/emelie.atom") - }} - end - def get( "https://osada.macgirvin.com/.well-known/webfinger?resource=acct:mike@osada.macgirvin.com", _, @@ -137,14 +129,6 @@ def get( }} end - def get("https://pawoo.net/users/pekorino.atom", _, _, _) do - {:ok, - %Tesla.Env{ - status: 200, - body: File.read!("test/fixtures/tesla_mock/https___pawoo.net_users_pekorino.atom") - }} - end - def get( "https://pawoo.net/.well-known/webfinger?resource=acct:https://pawoo.net/users/pekorino", _, @@ -158,19 +142,6 @@ def get( }} end - def get( - "https://social.stopwatchingus-heidelberg.de/api/statuses/user_timeline/18330.atom", - _, - _, - _ - ) do - {:ok, - %Tesla.Env{ - status: 200, - body: File.read!("test/fixtures/tesla_mock/atarifrosch_feed.xml") - }} - end - def get( "https://social.stopwatchingus-heidelberg.de/.well-known/webfinger?resource=acct:https://social.stopwatchingus-heidelberg.de/user/18330", _, @@ -184,27 +155,6 @@ def get( }} end - def get("https://mamot.fr/users/Skruyb.atom", _, _, _) do - {:ok, - %Tesla.Env{ - status: 200, - body: File.read!("test/fixtures/tesla_mock/https___mamot.fr_users_Skruyb.atom") - }} - end - - def get( - "https://mamot.fr/.well-known/webfinger?resource=acct:https://mamot.fr/users/Skruyb", - _, - _, - [{"accept", "application/xrd+xml,application/jrd+json"}] - ) do - {:ok, - %Tesla.Env{ - status: 200, - body: File.read!("test/fixtures/tesla_mock/skruyb@mamot.fr.atom") - }} - end - def get( "https://social.heldscal.la/.well-known/webfinger?resource=nonexistant@social.heldscal.la", _, @@ -507,19 +457,6 @@ def get("https://mamot.fr/.well-known/host-meta", _, _, _) do }} end - def get( - "https://mamot.fr/.well-known/webfinger?resource=https://mamot.fr/users/Skruyb", - _, - _, - _ - ) do - {:ok, - %Tesla.Env{ - status: 200, - body: File.read!("test/fixtures/tesla_mock/skruyb@mamot.fr.atom") - }} - end - def get("http://pawoo.net/.well-known/host-meta", _, _, _) do {:ok, %Tesla.Env{ @@ -647,17 +584,6 @@ def get( }} end - def get("https://pleroma.soykaf.com/users/lain/feed.atom", _, _, _) do - {:ok, - %Tesla.Env{ - status: 200, - body: - File.read!( - "test/fixtures/tesla_mock/https___pleroma.soykaf.com_users_lain_feed.atom.xml" - ) - }} - end - def get(url, _, _, [{"accept", "application/xrd+xml,application/jrd+json"}]) when url in [ "https://pleroma.soykaf.com/.well-known/webfinger?resource=acct:https://pleroma.soykaf.com/users/lain", @@ -670,17 +596,6 @@ def get(url, _, _, [{"accept", "application/xrd+xml,application/jrd+json"}]) }} end - def get("https://shitposter.club/api/statuses/user_timeline/1.atom", _, _, _) do - {:ok, - %Tesla.Env{ - status: 200, - body: - File.read!( - "test/fixtures/tesla_mock/https___shitposter.club_api_statuses_user_timeline_1.atom.xml" - ) - }} - end - def get( "https://shitposter.club/.well-known/webfinger?resource=https://shitposter.club/user/1", _, @@ -694,37 +609,10 @@ def get( }} end - def get("https://shitposter.club/notice/2827873", _, _, _) do - {:ok, - %Tesla.Env{ - status: 200, - body: File.read!("test/fixtures/tesla_mock/https___shitposter.club_notice_2827873.json") - }} - end - - def get("https://shitposter.club/api/statuses/show/2827873.atom", _, _, _) do - {:ok, - %Tesla.Env{ - status: 200, - body: - File.read!( - "test/fixtures/tesla_mock/https___shitposter.club_api_statuses_show_2827873.atom.xml" - ) - }} - end - def get("https://testing.pleroma.lol/objects/b319022a-4946-44c5-9de9-34801f95507b", _, _, _) do {:ok, %Tesla.Env{status: 200}} end - def get("https://shitposter.club/api/statuses/user_timeline/5381.atom", _, _, _) do - {:ok, - %Tesla.Env{ - status: 200, - body: File.read!("test/fixtures/tesla_mock/spc_5381.atom") - }} - end - def get( "https://shitposter.club/.well-known/webfinger?resource=https://shitposter.club/user/5381", _, @@ -746,14 +634,6 @@ def get("http://shitposter.club/.well-known/host-meta", _, _, _) do }} end - def get("https://shitposter.club/api/statuses/show/7369654.atom", _, _, _) do - {:ok, - %Tesla.Env{ - status: 200, - body: File.read!("test/fixtures/tesla_mock/7369654.atom") - }} - end - def get("https://shitposter.club/notice/4027863", _, _, _) do {:ok, %Tesla.Env{ @@ -762,14 +642,6 @@ def get("https://shitposter.club/notice/4027863", _, _, _) do }} end - def get("https://social.sakamoto.gq/users/eal/feed.atom", _, _, _) do - {:ok, - %Tesla.Env{ - status: 200, - body: File.read!("test/fixtures/tesla_mock/sakamoto_eal_feed.atom") - }} - end - def get("http://social.sakamoto.gq/.well-known/host-meta", _, _, _) do {:ok, %Tesla.Env{ @@ -791,15 +663,6 @@ def get( }} end - def get( - "https://social.sakamoto.gq/objects/0ccc1a2c-66b0-4305-b23a-7f7f2b040056", - _, - _, - [{"accept", "application/atom+xml"}] - ) do - {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/sakamoto.atom")}} - end - def get("http://mastodon.social/.well-known/host-meta", _, _, _) do {:ok, %Tesla.Env{ @@ -853,28 +716,6 @@ def get( {:ok, %Tesla.Env{status: 406, body: ""}} end - def get("http://gs.example.org/index.php/api/statuses/user_timeline/1.atom", _, _, _) do - {:ok, - %Tesla.Env{ - status: 200, - body: - File.read!( - "test/fixtures/tesla_mock/http__gs.example.org_index.php_api_statuses_user_timeline_1.atom.xml" - ) - }} - end - - def get("https://social.heldscal.la/api/statuses/user_timeline/29191.atom", _, _, _) do - {:ok, - %Tesla.Env{ - status: 200, - body: - File.read!( - "test/fixtures/tesla_mock/https___social.heldscal.la_api_statuses_user_timeline_29191.atom.xml" - ) - }} - end - def get("http://squeet.me/.well-known/host-meta", _, _, _) do {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/squeet.me_host_meta")}} @@ -996,17 +837,6 @@ def get( }} end - def get("https://social.heldscal.la/api/statuses/user_timeline/23211.atom", _, _, _) do - {:ok, - %Tesla.Env{ - status: 200, - body: - File.read!( - "test/fixtures/tesla_mock/https___social.heldscal.la_api_statuses_user_timeline_23211.atom.xml" - ) - }} - end - def get( "https://social.heldscal.la/.well-known/webfinger?resource=https://social.heldscal.la/user/23211", _, @@ -1036,10 +866,6 @@ def get("https://social.heldscal.la/.well-known/host-meta", _, _, _) do }} end - def get("https://mastodon.social/users/lambadalambda.atom", _, _, _) do - {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/lambadalambda.atom")}} - end - def get("https://mastodon.social/users/lambadalambda", _, _, _) do {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/lambadalambda.json")}} end diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 6e096e9ec..cc55a7be7 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -105,7 +105,7 @@ test "it fetches reply-to activities if we don't have them" do object = data["object"] - |> Map.put("inReplyTo", "https://shitposter.club/notice/2827873") + |> Map.put("inReplyTo", "https://mstdn.io/users/mayuutann/statuses/99568293732299394") data = Map.put(data, "object", object) {:ok, returned_activity} = Transmogrifier.handle_incoming(data) @@ -113,11 +113,11 @@ test "it fetches reply-to activities if we don't have them" do assert activity = Activity.get_create_by_object_ap_id( - "tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment" + "https://mstdn.io/users/mayuutann/statuses/99568293732299394" ) assert returned_object.data["inReplyTo"] == - "tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment" + "https://mstdn.io/users/mayuutann/statuses/99568293732299394" end test "it does not fetch reply-to activities beyond max replies depth limit" do @@ -1104,17 +1104,17 @@ test "returns modified object when allowed incoming reply", %{data: data} do Map.put( data["object"], "inReplyTo", - "https://shitposter.club/notice/2827873" + "https://mstdn.io/users/mayuutann/statuses/99568293732299394" ) Pleroma.Config.put([:instance, :federation_incoming_replies_max_depth], 5) modified_object = Transmogrifier.fix_in_reply_to(object_with_reply) assert modified_object["inReplyTo"] == - "tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment" + "https://mstdn.io/users/mayuutann/statuses/99568293732299394" assert modified_object["context"] == - "tag:shitposter.club,2017-05-05:objectType=thread:nonce=3c16e9c2681f6d26" + "tag:shitposter.club,2018-02-22:objectType=thread:nonce=e5a7c72d60a9c0e4" end end @@ -1216,7 +1216,9 @@ test "returns nil when cannot normalize object" do @tag capture_log: true test "returns {:ok, %Object{}} for success case" do assert {:ok, %Object{}} = - Transmogrifier.get_obj_helper("https://shitposter.club/notice/2827873") + Transmogrifier.get_obj_helper( + "https://mstdn.io/users/mayuutann/statuses/99568293732299394" + ) end end diff --git a/test/web/mastodon_api/controllers/search_controller_test.exs b/test/web/mastodon_api/controllers/search_controller_test.exs index 24d1959f8..04dc6f445 100644 --- a/test/web/mastodon_api/controllers/search_controller_test.exs +++ b/test/web/mastodon_api/controllers/search_controller_test.exs @@ -282,18 +282,18 @@ test "search fetches remote statuses and prefers them over other results", %{con capture_log(fn -> {:ok, %{id: activity_id}} = CommonAPI.post(insert(:user), %{ - status: "check out https://shitposter.club/notice/2827873" + status: "check out http://mastodon.example.org/@admin/99541947525187367" }) results = conn - |> get("/api/v1/search?q=https://shitposter.club/notice/2827873") + |> get("/api/v1/search?q=http://mastodon.example.org/@admin/99541947525187367") |> json_response_and_validate_schema(200) - [status, %{"id" => ^activity_id}] = results["statuses"] - - assert status["uri"] == - "tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment" + assert [ + %{"url" => "http://mastodon.example.org/@admin/99541947525187367"}, + %{"id" => ^activity_id} + ] = results["statuses"] end) end From acf6393c87c776c0fab9fd41bbf5ba7078356be5 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 8 Sep 2020 20:13:00 +0300 Subject: [PATCH 078/128] SECURITY.md: we don't support 2.0 anymore, bump to 2.1 --- SECURITY.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SECURITY.md b/SECURITY.md index c212a2505..8617c1434 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -6,7 +6,7 @@ Currently, Pleroma offers bugfixes and security patches only for the latest mino | Version | Support |---------| -------- -| 2.0 | Bugfixes and security patches +| 2.1 | Bugfixes and security patches ## Reporting a vulnerability From a781ac6ca5b7ab23eea795331db0a3fff406630e Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 10 Jun 2020 15:37:43 +0400 Subject: [PATCH 079/128] Fix atom leak in AdminAPIController --- lib/pleroma/web/admin_api/controllers/admin_api_controller.ex | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex index aa2af1ab5..f5e4d49f9 100644 --- a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex @@ -379,8 +379,7 @@ defp maybe_parse_filters(filters) do filters |> String.split(",") |> Enum.filter(&Enum.member?(@filters, &1)) - |> Enum.map(&String.to_atom/1) - |> Map.new(&{&1, true}) + |> Map.new(&{String.to_existing_atom(&1), true}) end def right_add_multiple(%{assigns: %{user: admin}} = conn, %{ From 10ef532c63431811b3998ed7b14aea21755a2b57 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 7 Jul 2020 07:06:29 +0200 Subject: [PATCH 080/128] AP C2S: Restrict character limit on Note --- .../activity_pub/activity_pub_controller.ex | 35 ++++++++++++------- .../activity_pub_controller_test.exs | 16 +++++++++ 2 files changed, 38 insertions(+), 13 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index 220c4fe52..732c44271 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -399,21 +399,30 @@ def read_inbox(%{assigns: %{user: %User{nickname: as_nickname}}} = conn, %{ defp handle_user_activity( %User{} = user, - %{"type" => "Create", "object" => %{"type" => "Note"}} = params + %{"type" => "Create", "object" => %{"type" => "Note"} = object} = params ) do - object = - params["object"] - |> Map.merge(Map.take(params, ["to", "cc"])) - |> Map.put("attributedTo", user.ap_id()) - |> Transmogrifier.fix_object() + content = if is_binary(object["content"]), do: object["content"], else: "" + name = if is_binary(object["name"]), do: object["name"], else: "" + summary = if is_binary(object["summary"]), do: object["summary"], else: "" + length = String.length(content <> name <> summary) - ActivityPub.create(%{ - to: params["to"], - actor: user, - context: object["context"], - object: object, - additional: Map.take(params, ["cc"]) - }) + if length > Pleroma.Config.get([:instance, :limit]) do + {:error, dgettext("errors", "Note is over the character limit")} + else + object = + object + |> Map.merge(Map.take(params, ["to", "cc"])) + |> Map.put("attributedTo", user.ap_id()) + |> Transmogrifier.fix_object() + + ActivityPub.create(%{ + to: params["to"], + actor: user, + context: object["context"], + object: object, + additional: Map.take(params, ["cc"]) + }) + end end defp handle_user_activity(%User{} = user, %{"type" => "Delete"} = params) do diff --git a/test/web/activity_pub/activity_pub_controller_test.exs b/test/web/activity_pub/activity_pub_controller_test.exs index 57988dc1e..0517571f2 100644 --- a/test/web/activity_pub/activity_pub_controller_test.exs +++ b/test/web/activity_pub/activity_pub_controller_test.exs @@ -905,6 +905,8 @@ test "it requires authentication if instance is NOT federating", %{ end describe "POST /users/:nickname/outbox (C2S)" do + setup do: clear_config([:instance, :limit]) + setup do [ activity: %{ @@ -1121,6 +1123,20 @@ test "it doesn't spreads faulty attributedTo or actor fields", %{ assert cirno_object.data["actor"] == cirno.ap_id assert cirno_object.data["attributedTo"] == cirno.ap_id end + + test "Character limitation", %{conn: conn, activity: activity} do + Pleroma.Config.put([:instance, :limit], 5) + user = insert(:user) + + result = + conn + |> assign(:user, user) + |> put_req_header("content-type", "application/activity+json") + |> post("/users/#{user.nickname}/outbox", activity) + |> json_response(400) + + assert result == "Note is over the character limit" + end end describe "/relay/followers" do From 16c451f8f15b1b2907fb6fc40925b47821650f31 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Wed, 2 Sep 2020 20:11:24 +0200 Subject: [PATCH 081/128] search: Apply following filter only when user is usable --- lib/pleroma/user/search.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/user/search.ex b/lib/pleroma/user/search.ex index adbef7fb8..7babd47ea 100644 --- a/lib/pleroma/user/search.ex +++ b/lib/pleroma/user/search.ex @@ -115,8 +115,8 @@ defp trigram_rank(query, query_string) do ) end - defp base_query(_user, false), do: User - defp base_query(user, true), do: User.get_friends_query(user) + defp base_query(%User{} = user, true), do: User.get_friends_query(user) + defp base_query(_user, _following), do: User defp filter_invisible_users(query) do from(q in query, where: q.invisible == false) From 947ee55ae298a42c2667800c1aac96f637e31969 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Wed, 2 Sep 2020 20:24:03 +0200 Subject: [PATCH 082/128] user: harden get_friends_query(), get_followers_query() and their wrappers --- lib/pleroma/user.ex | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 94c96de8d..f323fc6ed 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1125,31 +1125,31 @@ def get_followers_query(%User{} = user, nil) do User.Query.build(%{followers: user, deactivated: false}) end - def get_followers_query(user, page) do + def get_followers_query(%User{} = user, page) do user |> get_followers_query(nil) |> User.Query.paginate(page, 20) end @spec get_followers_query(User.t()) :: Ecto.Query.t() - def get_followers_query(user), do: get_followers_query(user, nil) + def get_followers_query(%User{} = user), do: get_followers_query(user, nil) @spec get_followers(User.t(), pos_integer() | nil) :: {:ok, list(User.t())} - def get_followers(user, page \\ nil) do + def get_followers(%User{} = user, page \\ nil) do user |> get_followers_query(page) |> Repo.all() end @spec get_external_followers(User.t(), pos_integer() | nil) :: {:ok, list(User.t())} - def get_external_followers(user, page \\ nil) do + def get_external_followers(%User{} = user, page \\ nil) do user |> get_followers_query(page) |> User.Query.build(%{external: true}) |> Repo.all() end - def get_followers_ids(user, page \\ nil) do + def get_followers_ids(%User{} = user, page \\ nil) do user |> get_followers_query(page) |> select([u], u.id) @@ -1161,29 +1161,29 @@ def get_friends_query(%User{} = user, nil) do User.Query.build(%{friends: user, deactivated: false}) end - def get_friends_query(user, page) do + def get_friends_query(%User{} = user, page) do user |> get_friends_query(nil) |> User.Query.paginate(page, 20) end @spec get_friends_query(User.t()) :: Ecto.Query.t() - def get_friends_query(user), do: get_friends_query(user, nil) + def get_friends_query(%User{} = user), do: get_friends_query(user, nil) - def get_friends(user, page \\ nil) do + def get_friends(%User{} = user, page \\ nil) do user |> get_friends_query(page) |> Repo.all() end - def get_friends_ap_ids(user) do + def get_friends_ap_ids(%User{} = user) do user |> get_friends_query(nil) |> select([u], u.ap_id) |> Repo.all() end - def get_friends_ids(user, page \\ nil) do + def get_friends_ids(%User{} = user, page \\ nil) do user |> get_friends_query(page) |> select([u], u.id) From 630444ee0819ad5b58c5f9030758fe41e6fed530 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 31 Aug 2020 14:19:48 -0500 Subject: [PATCH 083/128] Do not make RelMe metadata provider optional. There's really no sound reason to turn this off anyway. --- config/config.exs | 1 - lib/pleroma/web/metadata.ex | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/config.exs b/config/config.exs index 2426fbd52..df61c069e 100644 --- a/config/config.exs +++ b/config/config.exs @@ -455,7 +455,6 @@ providers: [ Pleroma.Web.Metadata.Providers.OpenGraph, Pleroma.Web.Metadata.Providers.TwitterCard, - Pleroma.Web.Metadata.Providers.RelMe, Pleroma.Web.Metadata.Providers.Feed ], unfurl_nsfw: false diff --git a/lib/pleroma/web/metadata.ex b/lib/pleroma/web/metadata.ex index a9f70c43e..e45e74e7b 100644 --- a/lib/pleroma/web/metadata.ex +++ b/lib/pleroma/web/metadata.ex @@ -7,7 +7,8 @@ defmodule Pleroma.Web.Metadata do def build_tags(params) do providers = [ - Pleroma.Web.Metadata.Providers.RestrictIndexing + Pleroma.Web.Metadata.Providers.RestrictIndexing, + Pleroma.Web.Metadata.Providers.RelMe, | Pleroma.Config.get([__MODULE__, :providers], []) ] From ff07014b2657730101e826d7e82716989d43214c Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 31 Aug 2020 14:35:22 -0500 Subject: [PATCH 084/128] Disable providers of user and status metadata when instance is private --- CHANGELOG.md | 3 +++ lib/pleroma/web/metadata.ex | 12 ++++++++++-- test/web/metadata/metadata_test.exs | 9 +++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f57e191fa..496c78ffe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## unreleased-patch - ??? +### Security +- Fix metadata leak for accounts and statuses on private instances + ### Added - Rich media failure tracking (along with `:failure_backoff` option) diff --git a/lib/pleroma/web/metadata.ex b/lib/pleroma/web/metadata.ex index e45e74e7b..0f0b56321 100644 --- a/lib/pleroma/web/metadata.ex +++ b/lib/pleroma/web/metadata.ex @@ -8,8 +8,8 @@ defmodule Pleroma.Web.Metadata do def build_tags(params) do providers = [ Pleroma.Web.Metadata.Providers.RestrictIndexing, - Pleroma.Web.Metadata.Providers.RelMe, - | Pleroma.Config.get([__MODULE__, :providers], []) + Pleroma.Web.Metadata.Providers.RelMe + | activated_providers() ] Enum.reduce(providers, "", fn parser, acc -> @@ -43,4 +43,12 @@ def activity_nsfw?(%{data: %{"sensitive" => sensitive}}) do def activity_nsfw?(_) do false end + + defp activated_providers do + if Pleroma.Config.get!([:instance, :public]) do + Pleroma.Config.get([__MODULE__, :providers], []) + else + [] + end + end end diff --git a/test/web/metadata/metadata_test.exs b/test/web/metadata/metadata_test.exs index 3f8b29e58..4dd0d2f5c 100644 --- a/test/web/metadata/metadata_test.exs +++ b/test/web/metadata/metadata_test.exs @@ -22,4 +22,13 @@ test "for local user" do "" end end + + describe "no metadata for private instances" do + test "for local user" do + Pleroma.Config.put([:instance, :public], false) + user = insert(:user, bio: "This is my secret fedi account bio") + + assert "" = Pleroma.Web.Metadata.build_tags(%{user: user}) + end + end end From 14d07081fd82211071eafb3c31d8c756fe9af9f5 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 31 Aug 2020 14:48:22 -0500 Subject: [PATCH 085/128] Feed provider only generates a redirect, so always activate it. Making this configurable is misleading. --- config/config.exs | 3 +-- lib/pleroma/web/metadata.ex | 5 +++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config/config.exs b/config/config.exs index df61c069e..1a2b312b5 100644 --- a/config/config.exs +++ b/config/config.exs @@ -454,8 +454,7 @@ config :pleroma, Pleroma.Web.Metadata, providers: [ Pleroma.Web.Metadata.Providers.OpenGraph, - Pleroma.Web.Metadata.Providers.TwitterCard, - Pleroma.Web.Metadata.Providers.Feed + Pleroma.Web.Metadata.Providers.TwitterCard ], unfurl_nsfw: false diff --git a/lib/pleroma/web/metadata.ex b/lib/pleroma/web/metadata.ex index 0f0b56321..926b5b187 100644 --- a/lib/pleroma/web/metadata.ex +++ b/lib/pleroma/web/metadata.ex @@ -7,8 +7,9 @@ defmodule Pleroma.Web.Metadata do def build_tags(params) do providers = [ - Pleroma.Web.Metadata.Providers.RestrictIndexing, - Pleroma.Web.Metadata.Providers.RelMe + Pleroma.Web.Metadata.Providers.Feed, + Pleroma.Web.Metadata.Providers.RelMe, + Pleroma.Web.Metadata.Providers.RestrictIndexing | activated_providers() ] From 44ced17634a9c89b9ff4a47c64805356f7f6401c Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 31 Aug 2020 15:54:22 -0500 Subject: [PATCH 086/128] Fix test so setting doesn't leak --- test/web/metadata/metadata_test.exs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/web/metadata/metadata_test.exs b/test/web/metadata/metadata_test.exs index 4dd0d2f5c..f7371cae2 100644 --- a/test/web/metadata/metadata_test.exs +++ b/test/web/metadata/metadata_test.exs @@ -24,6 +24,8 @@ test "for local user" do end describe "no metadata for private instances" do + setup do: clear_config([:instance, :public]) + test "for local user" do Pleroma.Config.put([:instance, :public], false) user = insert(:user, bio: "This is my secret fedi account bio") From a85ed6defbd2cec71d9a5594ef1de18d5333c7c7 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 31 Aug 2020 15:58:21 -0500 Subject: [PATCH 087/128] Do not serve RSS/Atom feeds when instance is private --- lib/pleroma/web/feed/tag_controller.ex | 10 +++++++++- lib/pleroma/web/feed/user_controller.ex | 10 +++++++++- lib/pleroma/web/metadata/feed.ex | 20 ++++++++++++-------- test/web/feed/tag_controller_test.exs | 13 +++++++++++++ test/web/feed/user_controller_test.exs | 16 ++++++++++++++++ 5 files changed, 59 insertions(+), 10 deletions(-) diff --git a/lib/pleroma/web/feed/tag_controller.ex b/lib/pleroma/web/feed/tag_controller.ex index 39b2a766a..e090dd625 100644 --- a/lib/pleroma/web/feed/tag_controller.ex +++ b/lib/pleroma/web/feed/tag_controller.ex @@ -9,7 +9,15 @@ defmodule Pleroma.Web.Feed.TagController do alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.Feed.FeedView - def feed(conn, %{"tag" => raw_tag} = params) do + def feed(conn, params) do + if Pleroma.Config.get!([:instance, :public]) do + render_feed(conn, params) + else + render_error(conn, :not_found, "Not found") + end + end + + def render_feed(conn, %{"tag" => raw_tag} = params) do {format, tag} = parse_tag(raw_tag) activities = diff --git a/lib/pleroma/web/feed/user_controller.ex b/lib/pleroma/web/feed/user_controller.ex index 9cd334a33..595889b9d 100644 --- a/lib/pleroma/web/feed/user_controller.ex +++ b/lib/pleroma/web/feed/user_controller.ex @@ -37,7 +37,15 @@ def feed_redirect(conn, %{"nickname" => nickname}) do end end - def feed(conn, %{"nickname" => nickname} = params) do + def feed(conn, params) do + if Pleroma.Config.get!([:instance, :public]) do + render_feed(conn, params) + else + errors(conn, {:error, :not_found}) + end + end + + def render_feed(conn, %{"nickname" => nickname} = params) do format = get_format(conn) format = diff --git a/lib/pleroma/web/metadata/feed.ex b/lib/pleroma/web/metadata/feed.ex index bd1459a17..dfe391b56 100644 --- a/lib/pleroma/web/metadata/feed.ex +++ b/lib/pleroma/web/metadata/feed.ex @@ -11,13 +11,17 @@ defmodule Pleroma.Web.Metadata.Providers.Feed do @impl Provider def build_tags(%{user: user}) do - [ - {:link, - [ - rel: "alternate", - type: "application/atom+xml", - href: Helpers.user_feed_path(Endpoint, :feed, user.nickname) <> ".atom" - ], []} - ] + if Pleroma.Config.get!([:instance, :public]) do + [ + {:link, + [ + rel: "alternate", + type: "application/atom+xml", + href: Helpers.user_feed_path(Endpoint, :feed, user.nickname) <> ".atom" + ], []} + ] + else + [] + end end end diff --git a/test/web/feed/tag_controller_test.exs b/test/web/feed/tag_controller_test.exs index 3c29cd94f..868e40965 100644 --- a/test/web/feed/tag_controller_test.exs +++ b/test/web/feed/tag_controller_test.exs @@ -181,4 +181,17 @@ test "gets a feed (RSS)", %{conn: conn} do 'yeah #PleromaArt' ] end + + describe "private instance" do + setup do: clear_config([:instance, :public]) + + test "returns 404 for tags feed", %{conn: conn} do + Config.put([:instance, :public], false) + + conn + |> put_req_header("accept", "application/rss+xml") + |> get(tag_feed_path(conn, :feed, "pleromaart")) + |> response(404) + end + end end diff --git a/test/web/feed/user_controller_test.exs b/test/web/feed/user_controller_test.exs index 0d2a61967..9a5610baa 100644 --- a/test/web/feed/user_controller_test.exs +++ b/test/web/feed/user_controller_test.exs @@ -246,4 +246,20 @@ test "with non-html / non-json format, it returns error when user is not found", assert response == ~S({"error":"Not found"}) end end + + describe "private instance" do + setup do: clear_config([:instance, :public]) + + test "returns 404 for user feed", %{conn: conn} do + Config.put([:instance, :public], false) + user = insert(:user) + + {:ok, _} = CommonAPI.post(user, %{status: "test"}) + + assert conn + |> put_req_header("accept", "application/atom+xml") + |> get(user_feed_path(conn, :feed, user.nickname)) + |> response(404) + end + end end From 96697db3bc4d8fda90906cdeff77ae2668066bd4 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 31 Aug 2020 16:11:01 -0500 Subject: [PATCH 088/128] RelMe and Feed no longer configurable --- docs/configuration/cheatsheet.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index bffce95e7..ec59896ec 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -353,8 +353,6 @@ config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Http, * `providers`: a list of metadata providers to enable. Providers available: * `Pleroma.Web.Metadata.Providers.OpenGraph` * `Pleroma.Web.Metadata.Providers.TwitterCard` - * `Pleroma.Web.Metadata.Providers.RelMe` - add links from user bio with rel=me into the `
` as ``. - * `Pleroma.Web.Metadata.Providers.Feed` - add a link to a user's Atom feed into the `
` as ``. * `unfurl_nsfw`: If set to `true` nsfw attachments will be shown in previews. ### :rich_media (consumer) From 549c895d80f36109731565abf00303a7f80add21 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 31 Aug 2020 16:11:13 -0500 Subject: [PATCH 089/128] Document breaking change for metadata providers --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 496c78ffe..512547427 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## unreleased-patch - ??? +### Changed + +- **Breaking:** The metadata providers RelMe and Feed are no longer configurable. RelMe should always be activated and Feed only provides a header tag for the actual RSS/Atom feed when the instance is public. + ### Security - Fix metadata leak for accounts and statuses on private instances From 2011142ed9ae45f53496b3682da7114255c814a5 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 1 Sep 2020 10:43:44 -0500 Subject: [PATCH 090/128] Use :restrict_unauthenticated testing for more granular control --- lib/pleroma/web/feed/tag_controller.ex | 2 +- lib/pleroma/web/feed/user_controller.ex | 2 +- lib/pleroma/web/metadata.ex | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/feed/tag_controller.ex b/lib/pleroma/web/feed/tag_controller.ex index e090dd625..93a8294b7 100644 --- a/lib/pleroma/web/feed/tag_controller.ex +++ b/lib/pleroma/web/feed/tag_controller.ex @@ -10,7 +10,7 @@ defmodule Pleroma.Web.Feed.TagController do alias Pleroma.Web.Feed.FeedView def feed(conn, params) do - if Pleroma.Config.get!([:instance, :public]) do + unless Pleroma.Config.restrict_unauthenticated_access?(:activities, :local) do render_feed(conn, params) else render_error(conn, :not_found, "Not found") diff --git a/lib/pleroma/web/feed/user_controller.ex b/lib/pleroma/web/feed/user_controller.ex index 595889b9d..71eb1ea7e 100644 --- a/lib/pleroma/web/feed/user_controller.ex +++ b/lib/pleroma/web/feed/user_controller.ex @@ -38,7 +38,7 @@ def feed_redirect(conn, %{"nickname" => nickname}) do end def feed(conn, params) do - if Pleroma.Config.get!([:instance, :public]) do + unless Pleroma.Config.restrict_unauthenticated_access?(:profiles, :local) do render_feed(conn, params) else errors(conn, {:error, :not_found}) diff --git a/lib/pleroma/web/metadata.ex b/lib/pleroma/web/metadata.ex index 926b5b187..68835c826 100644 --- a/lib/pleroma/web/metadata.ex +++ b/lib/pleroma/web/metadata.ex @@ -46,7 +46,7 @@ def activity_nsfw?(_) do end defp activated_providers do - if Pleroma.Config.get!([:instance, :public]) do + unless Pleroma.Config.restrict_unauthenticated_access?(:activities, :local) do Pleroma.Config.get([__MODULE__, :providers], []) else [] From 0d2814ec8e41942cd5b056d9c1ed902be1e1773c Mon Sep 17 00:00:00 2001 From: rinpatch Date: Mon, 7 Sep 2020 15:06:06 +0300 Subject: [PATCH 091/128] Metadata: Move restriction check from Feed provider to activated_providers --- lib/pleroma/web/metadata.ex | 3 +-- lib/pleroma/web/metadata/feed.ex | 20 ++++++++------------ test/web/metadata/metadata_test.exs | 4 +--- 3 files changed, 10 insertions(+), 17 deletions(-) diff --git a/lib/pleroma/web/metadata.ex b/lib/pleroma/web/metadata.ex index 68835c826..0f2d8d1e7 100644 --- a/lib/pleroma/web/metadata.ex +++ b/lib/pleroma/web/metadata.ex @@ -7,7 +7,6 @@ defmodule Pleroma.Web.Metadata do def build_tags(params) do providers = [ - Pleroma.Web.Metadata.Providers.Feed, Pleroma.Web.Metadata.Providers.RelMe, Pleroma.Web.Metadata.Providers.RestrictIndexing | activated_providers() @@ -47,7 +46,7 @@ def activity_nsfw?(_) do defp activated_providers do unless Pleroma.Config.restrict_unauthenticated_access?(:activities, :local) do - Pleroma.Config.get([__MODULE__, :providers], []) + [Pleroma.Web.Metadata.Providers.Feed | Pleroma.Config.get([__MODULE__, :providers], [])] else [] end diff --git a/lib/pleroma/web/metadata/feed.ex b/lib/pleroma/web/metadata/feed.ex index dfe391b56..bd1459a17 100644 --- a/lib/pleroma/web/metadata/feed.ex +++ b/lib/pleroma/web/metadata/feed.ex @@ -11,17 +11,13 @@ defmodule Pleroma.Web.Metadata.Providers.Feed do @impl Provider def build_tags(%{user: user}) do - if Pleroma.Config.get!([:instance, :public]) do - [ - {:link, - [ - rel: "alternate", - type: "application/atom+xml", - href: Helpers.user_feed_path(Endpoint, :feed, user.nickname) <> ".atom" - ], []} - ] - else - [] - end + [ + {:link, + [ + rel: "alternate", + type: "application/atom+xml", + href: Helpers.user_feed_path(Endpoint, :feed, user.nickname) <> ".atom" + ], []} + ] end end diff --git a/test/web/metadata/metadata_test.exs b/test/web/metadata/metadata_test.exs index f7371cae2..9d3121b7b 100644 --- a/test/web/metadata/metadata_test.exs +++ b/test/web/metadata/metadata_test.exs @@ -24,10 +24,8 @@ test "for local user" do end describe "no metadata for private instances" do - setup do: clear_config([:instance, :public]) - test "for local user" do - Pleroma.Config.put([:instance, :public], false) + clear_config([:instance, :public], false) user = insert(:user, bio: "This is my secret fedi account bio") assert "" = Pleroma.Web.Metadata.build_tags(%{user: user}) From ace94bfcc8a57a633a0bdf5746eefae32e6fdad9 Mon Sep 17 00:00:00 2001 From: tarteka Date: Wed, 9 Sep 2020 09:49:26 +0000 Subject: [PATCH 092/128] Added translation using Weblate (Spanish) --- priv/gettext/es/LC_MESSAGES/errors.po | 584 ++++++++++++++++++++++++++ 1 file changed, 584 insertions(+) create mode 100644 priv/gettext/es/LC_MESSAGES/errors.po diff --git a/priv/gettext/es/LC_MESSAGES/errors.po b/priv/gettext/es/LC_MESSAGES/errors.po new file mode 100644 index 000000000..4b0972e88 --- /dev/null +++ b/priv/gettext/es/LC_MESSAGES/errors.po @@ -0,0 +1,584 @@ +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2020-09-09 09:49+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: Automatically generated\n" +"Language-Team: none\n" +"Language: es\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"X-Generator: Translate Toolkit 2.5.1\n" + +## This file is a PO Template file. +## +## `msgid`s here are often extracted from source code. +## Add new translations manually only if they're dynamic +## translations that can't be statically extracted. +## +## Run `mix gettext.extract` to bring this file up to +## date. Leave `msgstr`s empty as changing them here as no +## effect: edit them in PO (`.po`) files instead. +## From Ecto.Changeset.cast/4 +msgid "can't be blank" +msgstr "" + +## From Ecto.Changeset.unique_constraint/3 +msgid "has already been taken" +msgstr "" + +## From Ecto.Changeset.put_change/3 +msgid "is invalid" +msgstr "" + +## From Ecto.Changeset.validate_format/3 +msgid "has invalid format" +msgstr "" + +## From Ecto.Changeset.validate_subset/3 +msgid "has an invalid entry" +msgstr "" + +## From Ecto.Changeset.validate_exclusion/3 +msgid "is reserved" +msgstr "" + +## From Ecto.Changeset.validate_confirmation/3 +msgid "does not match confirmation" +msgstr "" + +## From Ecto.Changeset.no_assoc_constraint/3 +msgid "is still associated with this entry" +msgstr "" + +msgid "are still associated with this entry" +msgstr "" + +## From Ecto.Changeset.validate_length/3 +msgid "should be %{count} character(s)" +msgid_plural "should be %{count} character(s)" +msgstr[0] "" +msgstr[1] "" + +msgid "should have %{count} item(s)" +msgid_plural "should have %{count} item(s)" +msgstr[0] "" +msgstr[1] "" + +msgid "should be at least %{count} character(s)" +msgid_plural "should be at least %{count} character(s)" +msgstr[0] "" +msgstr[1] "" + +msgid "should have at least %{count} item(s)" +msgid_plural "should have at least %{count} item(s)" +msgstr[0] "" +msgstr[1] "" + +msgid "should be at most %{count} character(s)" +msgid_plural "should be at most %{count} character(s)" +msgstr[0] "" +msgstr[1] "" + +msgid "should have at most %{count} item(s)" +msgid_plural "should have at most %{count} item(s)" +msgstr[0] "" +msgstr[1] "" + +## From Ecto.Changeset.validate_number/3 +msgid "must be less than %{number}" +msgstr "" + +msgid "must be greater than %{number}" +msgstr "" + +msgid "must be less than or equal to %{number}" +msgstr "" + +msgid "must be greater than or equal to %{number}" +msgstr "" + +msgid "must be equal to %{number}" +msgstr "" + +#: lib/pleroma/web/common_api/common_api.ex:505 +#, elixir-format +msgid "Account not found" +msgstr "" + +#: lib/pleroma/web/common_api/common_api.ex:339 +#, elixir-format +msgid "Already voted" +msgstr "" + +#: lib/pleroma/web/oauth/oauth_controller.ex:359 +#, elixir-format +msgid "Bad request" +msgstr "" + +#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:426 +#, elixir-format +msgid "Can't delete object" +msgstr "" + +#: lib/pleroma/web/controller_helper.ex:105 +#: lib/pleroma/web/controller_helper.ex:111 +#, elixir-format +msgid "Can't display this activity" +msgstr "" + +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:285 +#, elixir-format +msgid "Can't find user" +msgstr "" + +#: lib/pleroma/web/pleroma_api/controllers/account_controller.ex:61 +#, elixir-format +msgid "Can't get favorites" +msgstr "" + +#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:438 +#, elixir-format +msgid "Can't like object" +msgstr "" + +#: lib/pleroma/web/common_api/utils.ex:563 +#, elixir-format +msgid "Cannot post an empty status without attachments" +msgstr "" + +#: lib/pleroma/web/common_api/utils.ex:511 +#, elixir-format +msgid "Comment must be up to %{max_size} characters" +msgstr "" + +#: lib/pleroma/config/config_db.ex:191 +#, elixir-format +msgid "Config with params %{params} not found" +msgstr "" + +#: lib/pleroma/web/common_api/common_api.ex:181 +#: lib/pleroma/web/common_api/common_api.ex:185 +#, elixir-format +msgid "Could not delete" +msgstr "" + +#: lib/pleroma/web/common_api/common_api.ex:231 +#, elixir-format +msgid "Could not favorite" +msgstr "" + +#: lib/pleroma/web/common_api/common_api.ex:453 +#, elixir-format +msgid "Could not pin" +msgstr "" + +#: lib/pleroma/web/common_api/common_api.ex:278 +#, elixir-format +msgid "Could not unfavorite" +msgstr "" + +#: lib/pleroma/web/common_api/common_api.ex:463 +#, elixir-format +msgid "Could not unpin" +msgstr "" + +#: lib/pleroma/web/common_api/common_api.ex:216 +#, elixir-format +msgid "Could not unrepeat" +msgstr "" + +#: lib/pleroma/web/common_api/common_api.ex:512 +#: lib/pleroma/web/common_api/common_api.ex:521 +#, elixir-format +msgid "Could not update state" +msgstr "" + +#: lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex:207 +#, elixir-format +msgid "Error." +msgstr "" + +#: lib/pleroma/web/twitter_api/twitter_api.ex:106 +#, elixir-format +msgid "Invalid CAPTCHA" +msgstr "" + +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:116 +#: lib/pleroma/web/oauth/oauth_controller.ex:568 +#, elixir-format +msgid "Invalid credentials" +msgstr "" + +#: lib/pleroma/plugs/ensure_authenticated_plug.ex:38 +#, elixir-format +msgid "Invalid credentials." +msgstr "" + +#: lib/pleroma/web/common_api/common_api.ex:355 +#, elixir-format +msgid "Invalid indices" +msgstr "" + +#: lib/pleroma/web/admin_api/controllers/fallback_controller.ex:29 +#, elixir-format +msgid "Invalid parameters" +msgstr "" + +#: lib/pleroma/web/common_api/utils.ex:414 +#, elixir-format +msgid "Invalid password." +msgstr "" + +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:220 +#, elixir-format +msgid "Invalid request" +msgstr "" + +#: lib/pleroma/web/twitter_api/twitter_api.ex:109 +#, elixir-format +msgid "Kocaptcha service unavailable" +msgstr "" + +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:112 +#, elixir-format +msgid "Missing parameters" +msgstr "" + +#: lib/pleroma/web/common_api/utils.ex:547 +#, elixir-format +msgid "No such conversation" +msgstr "" + +#: lib/pleroma/web/admin_api/controllers/admin_api_controller.ex:388 +#: lib/pleroma/web/admin_api/controllers/admin_api_controller.ex:414 lib/pleroma/web/admin_api/controllers/admin_api_controller.ex:456 +#, elixir-format +msgid "No such permission_group" +msgstr "" + +#: lib/pleroma/plugs/uploaded_media.ex:84 +#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:486 lib/pleroma/web/admin_api/controllers/fallback_controller.ex:11 +#: lib/pleroma/web/feed/user_controller.ex:71 lib/pleroma/web/ostatus/ostatus_controller.ex:143 +#, elixir-format +msgid "Not found" +msgstr "" + +#: lib/pleroma/web/common_api/common_api.ex:331 +#, elixir-format +msgid "Poll's author can't vote" +msgstr "" + +#: lib/pleroma/web/mastodon_api/controllers/fallback_controller.ex:20 +#: lib/pleroma/web/mastodon_api/controllers/poll_controller.ex:37 lib/pleroma/web/mastodon_api/controllers/poll_controller.ex:49 +#: lib/pleroma/web/mastodon_api/controllers/poll_controller.ex:50 lib/pleroma/web/mastodon_api/controllers/status_controller.ex:306 +#: lib/pleroma/web/mastodon_api/controllers/subscription_controller.ex:71 +#, elixir-format +msgid "Record not found" +msgstr "" + +#: lib/pleroma/web/admin_api/controllers/fallback_controller.ex:35 +#: lib/pleroma/web/feed/user_controller.ex:77 lib/pleroma/web/mastodon_api/controllers/fallback_controller.ex:36 +#: lib/pleroma/web/ostatus/ostatus_controller.ex:149 +#, elixir-format +msgid "Something went wrong" +msgstr "" + +#: lib/pleroma/web/common_api/activity_draft.ex:107 +#, elixir-format +msgid "The message visibility must be direct" +msgstr "" + +#: lib/pleroma/web/common_api/utils.ex:573 +#, elixir-format +msgid "The status is over the character limit" +msgstr "" + +#: lib/pleroma/plugs/ensure_public_or_authenticated_plug.ex:31 +#, elixir-format +msgid "This resource requires authentication." +msgstr "" + +#: lib/pleroma/plugs/rate_limiter/rate_limiter.ex:206 +#, elixir-format +msgid "Throttled" +msgstr "" + +#: lib/pleroma/web/common_api/common_api.ex:356 +#, elixir-format +msgid "Too many choices" +msgstr "" + +#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:443 +#, elixir-format +msgid "Unhandled activity type" +msgstr "" + +#: lib/pleroma/web/admin_api/controllers/admin_api_controller.ex:485 +#, elixir-format +msgid "You can't revoke your own admin status." +msgstr "" + +#: lib/pleroma/web/oauth/oauth_controller.ex:221 +#: lib/pleroma/web/oauth/oauth_controller.ex:308 +#, elixir-format +msgid "Your account is currently disabled" +msgstr "" + +#: lib/pleroma/web/oauth/oauth_controller.ex:183 +#: lib/pleroma/web/oauth/oauth_controller.ex:331 +#, elixir-format +msgid "Your login is missing a confirmed e-mail address" +msgstr "" + +#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:390 +#, elixir-format +msgid "can't read inbox of %{nickname} as %{as_nickname}" +msgstr "" + +#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:473 +#, elixir-format +msgid "can't update outbox of %{nickname} as %{as_nickname}" +msgstr "" + +#: lib/pleroma/web/common_api/common_api.ex:471 +#, elixir-format +msgid "conversation is already muted" +msgstr "" + +#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:314 +#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:492 +#, elixir-format +msgid "error" +msgstr "" + +#: lib/pleroma/web/pleroma_api/controllers/mascot_controller.ex:32 +#, elixir-format +msgid "mascots can only be images" +msgstr "" + +#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:62 +#, elixir-format +msgid "not found" +msgstr "" + +#: lib/pleroma/web/oauth/oauth_controller.ex:394 +#, elixir-format +msgid "Bad OAuth request." +msgstr "" + +#: lib/pleroma/web/twitter_api/twitter_api.ex:115 +#, elixir-format +msgid "CAPTCHA already used" +msgstr "" + +#: lib/pleroma/web/twitter_api/twitter_api.ex:112 +#, elixir-format +msgid "CAPTCHA expired" +msgstr "" + +#: lib/pleroma/plugs/uploaded_media.ex:57 +#, elixir-format +msgid "Failed" +msgstr "" + +#: lib/pleroma/web/oauth/oauth_controller.ex:410 +#, elixir-format +msgid "Failed to authenticate: %{message}." +msgstr "" + +#: lib/pleroma/web/oauth/oauth_controller.ex:441 +#, elixir-format +msgid "Failed to set up user account." +msgstr "" + +#: lib/pleroma/plugs/oauth_scopes_plug.ex:38 +#, elixir-format +msgid "Insufficient permissions: %{permissions}." +msgstr "" + +#: lib/pleroma/plugs/uploaded_media.ex:104 +#, elixir-format +msgid "Internal Error" +msgstr "" + +#: lib/pleroma/web/oauth/fallback_controller.ex:22 +#: lib/pleroma/web/oauth/fallback_controller.ex:29 +#, elixir-format +msgid "Invalid Username/Password" +msgstr "" + +#: lib/pleroma/web/twitter_api/twitter_api.ex:118 +#, elixir-format +msgid "Invalid answer data" +msgstr "" + +#: lib/pleroma/web/nodeinfo/nodeinfo_controller.ex:33 +#, elixir-format +msgid "Nodeinfo schema version not handled" +msgstr "" + +#: lib/pleroma/web/oauth/oauth_controller.ex:172 +#, elixir-format +msgid "This action is outside the authorized scopes" +msgstr "" + +#: lib/pleroma/web/oauth/fallback_controller.ex:14 +#, elixir-format +msgid "Unknown error, please check the details and try again." +msgstr "" + +#: lib/pleroma/web/oauth/oauth_controller.ex:119 +#: lib/pleroma/web/oauth/oauth_controller.ex:158 +#, elixir-format +msgid "Unlisted redirect_uri." +msgstr "" + +#: lib/pleroma/web/oauth/oauth_controller.ex:390 +#, elixir-format +msgid "Unsupported OAuth provider: %{provider}." +msgstr "" + +#: lib/pleroma/uploaders/uploader.ex:72 +#, elixir-format +msgid "Uploader callback timeout" +msgstr "" + +#: lib/pleroma/web/uploader_controller.ex:23 +#, elixir-format +msgid "bad request" +msgstr "" + +#: lib/pleroma/web/twitter_api/twitter_api.ex:103 +#, elixir-format +msgid "CAPTCHA Error" +msgstr "" + +#: lib/pleroma/web/common_api/common_api.ex:290 +#, elixir-format +msgid "Could not add reaction emoji" +msgstr "" + +#: lib/pleroma/web/common_api/common_api.ex:301 +#, elixir-format +msgid "Could not remove reaction emoji" +msgstr "" + +#: lib/pleroma/web/twitter_api/twitter_api.ex:129 +#, elixir-format +msgid "Invalid CAPTCHA (Missing parameter: %{name})" +msgstr "" + +#: lib/pleroma/web/mastodon_api/controllers/list_controller.ex:92 +#, elixir-format +msgid "List not found" +msgstr "" + +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:123 +#, elixir-format +msgid "Missing parameter: %{name}" +msgstr "" + +#: lib/pleroma/web/oauth/oauth_controller.ex:210 +#: lib/pleroma/web/oauth/oauth_controller.ex:321 +#, elixir-format +msgid "Password reset is required" +msgstr "" + +#: lib/pleroma/tests/auth_test_controller.ex:9 +#: lib/pleroma/web/activity_pub/activity_pub_controller.ex:6 lib/pleroma/web/admin_api/controllers/admin_api_controller.ex:6 +#: lib/pleroma/web/admin_api/controllers/config_controller.ex:6 lib/pleroma/web/admin_api/controllers/fallback_controller.ex:6 +#: lib/pleroma/web/admin_api/controllers/invite_controller.ex:6 lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex:6 +#: lib/pleroma/web/admin_api/controllers/oauth_app_controller.ex:6 lib/pleroma/web/admin_api/controllers/relay_controller.ex:6 +#: lib/pleroma/web/admin_api/controllers/report_controller.ex:6 lib/pleroma/web/admin_api/controllers/status_controller.ex:6 +#: lib/pleroma/web/controller_helper.ex:6 lib/pleroma/web/embed_controller.ex:6 +#: lib/pleroma/web/fallback_redirect_controller.ex:6 lib/pleroma/web/feed/tag_controller.ex:6 +#: lib/pleroma/web/feed/user_controller.ex:6 lib/pleroma/web/mailer/subscription_controller.ex:2 +#: lib/pleroma/web/masto_fe_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/account_controller.ex:6 +#: lib/pleroma/web/mastodon_api/controllers/app_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/auth_controller.ex:6 +#: lib/pleroma/web/mastodon_api/controllers/conversation_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/custom_emoji_controller.ex:6 +#: lib/pleroma/web/mastodon_api/controllers/domain_block_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/fallback_controller.ex:6 +#: lib/pleroma/web/mastodon_api/controllers/filter_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/follow_request_controller.ex:6 +#: lib/pleroma/web/mastodon_api/controllers/instance_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/list_controller.ex:6 +#: lib/pleroma/web/mastodon_api/controllers/marker_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex:14 +#: lib/pleroma/web/mastodon_api/controllers/media_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/notification_controller.ex:6 +#: lib/pleroma/web/mastodon_api/controllers/poll_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/report_controller.ex:8 +#: lib/pleroma/web/mastodon_api/controllers/scheduled_activity_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/search_controller.ex:6 +#: lib/pleroma/web/mastodon_api/controllers/status_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/subscription_controller.ex:7 +#: lib/pleroma/web/mastodon_api/controllers/suggestion_controller.ex:6 lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex:6 +#: lib/pleroma/web/media_proxy/media_proxy_controller.ex:6 lib/pleroma/web/mongooseim/mongoose_im_controller.ex:6 +#: lib/pleroma/web/nodeinfo/nodeinfo_controller.ex:6 lib/pleroma/web/oauth/fallback_controller.ex:6 +#: lib/pleroma/web/oauth/mfa_controller.ex:10 lib/pleroma/web/oauth/oauth_controller.ex:6 +#: lib/pleroma/web/ostatus/ostatus_controller.ex:6 lib/pleroma/web/pleroma_api/controllers/account_controller.ex:6 +#: lib/pleroma/web/pleroma_api/controllers/chat_controller.ex:5 lib/pleroma/web/pleroma_api/controllers/conversation_controller.ex:6 +#: lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex:2 lib/pleroma/web/pleroma_api/controllers/emoji_reaction_controller.ex:6 +#: lib/pleroma/web/pleroma_api/controllers/mascot_controller.ex:6 lib/pleroma/web/pleroma_api/controllers/notification_controller.ex:6 +#: lib/pleroma/web/pleroma_api/controllers/scrobble_controller.ex:6 +#: lib/pleroma/web/pleroma_api/controllers/two_factor_authentication_controller.ex:7 lib/pleroma/web/static_fe/static_fe_controller.ex:6 +#: lib/pleroma/web/twitter_api/controllers/password_controller.ex:10 lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex:6 +#: lib/pleroma/web/twitter_api/controllers/util_controller.ex:6 lib/pleroma/web/twitter_api/twitter_api_controller.ex:6 +#: lib/pleroma/web/uploader_controller.ex:6 lib/pleroma/web/web_finger/web_finger_controller.ex:6 +#, elixir-format +msgid "Security violation: OAuth scopes check was neither handled nor explicitly skipped." +msgstr "" + +#: lib/pleroma/plugs/ensure_authenticated_plug.ex:28 +#, elixir-format +msgid "Two-factor authentication enabled, you must use a access token." +msgstr "" + +#: lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex:210 +#, elixir-format +msgid "Unexpected error occurred while adding file to pack." +msgstr "" + +#: lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex:138 +#, elixir-format +msgid "Unexpected error occurred while creating pack." +msgstr "" + +#: lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex:278 +#, elixir-format +msgid "Unexpected error occurred while removing file from pack." +msgstr "" + +#: lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex:250 +#, elixir-format +msgid "Unexpected error occurred while updating file in pack." +msgstr "" + +#: lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex:179 +#, elixir-format +msgid "Unexpected error occurred while updating pack metadata." +msgstr "" + +#: lib/pleroma/web/mastodon_api/controllers/subscription_controller.ex:61 +#, elixir-format +msgid "Web push subscription is disabled on this Pleroma instance" +msgstr "" + +#: lib/pleroma/web/admin_api/controllers/admin_api_controller.ex:451 +#, elixir-format +msgid "You can't revoke your own admin/moderator status." +msgstr "" + +#: lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex:126 +#, elixir-format +msgid "authorization required for timeline view" +msgstr "" + +#: lib/pleroma/web/mastodon_api/controllers/fallback_controller.ex:24 +#, elixir-format +msgid "Access denied" +msgstr "" + +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:282 +#, elixir-format +msgid "This API requires an authenticated user" +msgstr "" + +#: lib/pleroma/plugs/user_is_admin_plug.ex:21 +#, elixir-format +msgid "User is not an admin." +msgstr "" From 6ec1a9ded164df2872ad4ec99ca673e7618c9253 Mon Sep 17 00:00:00 2001 From: tarteka Date: Wed, 9 Sep 2020 09:52:08 +0000 Subject: [PATCH 093/128] Translated using Weblate (Spanish) Currently translated at 9.4% (10 of 106 strings) Translation: Pleroma/Pleroma backend Translate-URL: https://translate.pleroma.social/projects/pleroma/pleroma/es/ --- priv/gettext/es/LC_MESSAGES/errors.po | 32 ++++++++++++++------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/priv/gettext/es/LC_MESSAGES/errors.po b/priv/gettext/es/LC_MESSAGES/errors.po index 4b0972e88..ba75936a9 100644 --- a/priv/gettext/es/LC_MESSAGES/errors.po +++ b/priv/gettext/es/LC_MESSAGES/errors.po @@ -3,14 +3,16 @@ msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: 2020-09-09 09:49+0000\n" -"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" -"Last-Translator: Automatically generated\n" -"Language-Team: none\n" +"PO-Revision-Date: 2020-09-09 10:52+0000\n" +"Last-Translator: tarteka \n" +"Language-Team: Spanish \n" "Language: es\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" -"X-Generator: Translate Toolkit 2.5.1\n" +"Plural-Forms: nplurals=2; plural=n != 1;\n" +"X-Generator: Weblate 4.0.4\n" ## This file is a PO Template file. ## @@ -23,44 +25,44 @@ msgstr "" ## effect: edit them in PO (`.po`) files instead. ## From Ecto.Changeset.cast/4 msgid "can't be blank" -msgstr "" +msgstr "no puede estar en blanco" ## From Ecto.Changeset.unique_constraint/3 msgid "has already been taken" -msgstr "" +msgstr "ya está en uso" ## From Ecto.Changeset.put_change/3 msgid "is invalid" -msgstr "" +msgstr "es inválido" ## From Ecto.Changeset.validate_format/3 msgid "has invalid format" -msgstr "" +msgstr "el formato no es válido" ## From Ecto.Changeset.validate_subset/3 msgid "has an invalid entry" -msgstr "" +msgstr "tiene una entrada inválida" ## From Ecto.Changeset.validate_exclusion/3 msgid "is reserved" -msgstr "" +msgstr "está reservado" ## From Ecto.Changeset.validate_confirmation/3 msgid "does not match confirmation" -msgstr "" +msgstr "la confirmación no coincide" ## From Ecto.Changeset.no_assoc_constraint/3 msgid "is still associated with this entry" -msgstr "" +msgstr "todavía está asociado con esta entrada" msgid "are still associated with this entry" -msgstr "" +msgstr "todavía están asociados con esta entrada" ## From Ecto.Changeset.validate_length/3 msgid "should be %{count} character(s)" msgid_plural "should be %{count} character(s)" -msgstr[0] "" -msgstr[1] "" +msgstr[0] "debe tener %{count} carácter" +msgstr[1] "debe tener %{count} caracteres" msgid "should have %{count} item(s)" msgid_plural "should have %{count} item(s)" From 0c1d24318565ec5bca0dfbe749f09d7acc9f7e42 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 9 Sep 2020 18:34:07 +0300 Subject: [PATCH 094/128] bump concurrent_limiter Should fix gun deadlocks --- mix.exs | 2 +- mix.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index 9499aab2d..18f748672 100644 --- a/mix.exs +++ b/mix.exs @@ -180,7 +180,7 @@ defp deps do {:flake_id, "~> 0.1.0"}, {:concurrent_limiter, git: "https://git.pleroma.social/pleroma/elixir-libraries/concurrent_limiter.git", - ref: "55e92f84b4ed531bd487952a71040a9c69dc2807"}, + ref: "d81be41024569330f296fc472e24198d7499ba78"}, {:remote_ip, git: "https://git.pleroma.social/pleroma/remote_ip.git", ref: "b647d0deecaa3acb140854fe4bda5b7e1dc6d1c8"}, diff --git a/mix.lock b/mix.lock index c4f9cd28c..a28c47017 100644 --- a/mix.lock +++ b/mix.lock @@ -14,7 +14,7 @@ "certifi": {:hex, :certifi, "2.5.1", "867ce347f7c7d78563450a18a6a28a8090331e77fa02380b4a21962a65d36ee5", [:rebar3], [{:parse_trans, "~>3.3", [hex: :parse_trans, repo: "hexpm", optional: false]}], "hexpm", "805abd97539caf89ec6d4732c91e62ba9da0cda51ac462380bbd28ee697a8c42"}, "combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"}, "comeonin": {:hex, :comeonin, "5.3.1", "7fe612b739c78c9c1a75186ef2d322ce4d25032d119823269d0aa1e2f1e20025", [:mix], [], "hexpm", "d6222483060c17f0977fad1b7401ef0c5863c985a64352755f366aee3799c245"}, - "concurrent_limiter": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/concurrent_limiter.git", "55e92f84b4ed531bd487952a71040a9c69dc2807", [ref: "55e92f84b4ed531bd487952a71040a9c69dc2807"]}, + "concurrent_limiter": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/concurrent_limiter.git", "d81be41024569330f296fc472e24198d7499ba78", [ref: "d81be41024569330f296fc472e24198d7499ba78"]}, "connection": {:hex, :connection, "1.0.4", "a1cae72211f0eef17705aaededacac3eb30e6625b04a6117c1b2db6ace7d5976", [:mix], [], "hexpm", "4a0850c9be22a43af9920a71ab17c051f5f7d45c209e40269a1938832510e4d9"}, "cors_plug": {:hex, :cors_plug, "2.0.2", "2b46083af45e4bc79632bd951550509395935d3e7973275b2b743bd63cc942ce", [:mix], [{:plug, "~> 1.8", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "f0d0e13f71c51fd4ef8b2c7e051388e4dfb267522a83a22392c856de7e46465f"}, "cowboy": {:hex, :cowboy, "2.8.0", "f3dc62e35797ecd9ac1b50db74611193c29815401e53bac9a5c0577bd7bc667d", [:rebar3], [{:cowlib, "~> 2.9.1", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "~> 1.7.1", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "4643e4fba74ac96d4d152c75803de6fad0b3fa5df354c71afdd6cbeeb15fac8a"}, From cad69669fc692da360929a5961e96550de1f1fe1 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Wed, 9 Sep 2020 22:44:38 +0300 Subject: [PATCH 095/128] [#2130] Fixed OAuth OOB authentication for users with enabled MFA. --- lib/pleroma/web/oauth/oauth_controller.ex | 5 ++++- .../o_auth/o_auth/oob_authorization_created.html.eex | 2 +- .../web/templates/o_auth/o_auth/oob_token_exists.html.eex | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/oauth/oauth_controller.ex b/lib/pleroma/web/oauth/oauth_controller.ex index dd00600ea..06b116368 100644 --- a/lib/pleroma/web/oauth/oauth_controller.ex +++ b/lib/pleroma/web/oauth/oauth_controller.ex @@ -145,7 +145,10 @@ def create_authorization( def after_create_authorization(%Plug.Conn{} = conn, %Authorization{} = auth, %{ "authorization" => %{"redirect_uri" => @oob_token_redirect_uri} }) do - render(conn, "oob_authorization_created.html", %{auth: auth}) + # Enforcing the view to reuse the template when calling from other controllers + conn + |> put_view(OAuthView) + |> render("oob_authorization_created.html", %{auth: auth}) end def after_create_authorization(%Plug.Conn{} = conn, %Authorization{} = auth, %{ diff --git a/lib/pleroma/web/templates/o_auth/o_auth/oob_authorization_created.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/oob_authorization_created.html.eex index 8443d906b..ffabe29a6 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/oob_authorization_created.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/oob_authorization_created.html.eex @@ -1,2 +1,2 @@

Successfully authorized

-

Token code is <%= @auth.token %>

+

Token code is
<%= @auth.token %>

diff --git a/lib/pleroma/web/templates/o_auth/o_auth/oob_token_exists.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/oob_token_exists.html.eex index 961aad976..82785c4b9 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/oob_token_exists.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/oob_token_exists.html.eex @@ -1,2 +1,2 @@

Authorization exists

-

Access token is <%= @token.token %>

+

Access token is
<%= @token.token %>

From ab56dd54e787eae82cf00fddc90eab4c5cbac4a9 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 10 Sep 2020 11:23:39 +0300 Subject: [PATCH 096/128] use Pleroma.HTTP in emoji packs tasks --- lib/mix/tasks/pleroma/emoji.ex | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/lib/mix/tasks/pleroma/emoji.ex b/lib/mix/tasks/pleroma/emoji.ex index 8f52ee98d..1750373f9 100644 --- a/lib/mix/tasks/pleroma/emoji.ex +++ b/lib/mix/tasks/pleroma/emoji.ex @@ -183,7 +183,7 @@ def run(["gen-pack" | args]) do IO.puts("Downloading the pack and generating SHA256") - binary_archive = Tesla.get!(client(), src).body + {:ok, %{body: binary_archive}} = Pleroma.HTTP.get(src) archive_sha = :crypto.hash(:sha256, binary_archive) |> Base.encode16() IO.puts("SHA256 is #{archive_sha}") @@ -252,7 +252,7 @@ defp fetch_and_decode!(from) do end defp fetch("http" <> _ = from) do - with {:ok, %{body: body}} <- Tesla.get(client(), from) do + with {:ok, %{body: body}} <- Pleroma.HTTP.get(from) do {:ok, body} end end @@ -271,13 +271,5 @@ defp parse_global_opts(args) do ) end - defp client do - middleware = [ - {Tesla.Middleware.FollowRedirects, [max_redirects: 3]} - ] - - Tesla.client(middleware) - end - defp default_manifest, do: Pleroma.Config.get!([:emoji, :default_manifest]) end From 3ce658b93098551792a69f2455e6e9339a1722e2 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 25 Aug 2020 19:17:51 +0300 Subject: [PATCH 097/128] schedule expired oauth tokens deletion with Oban --- config/config.exs | 2 +- config/description.exs | 1 - lib/pleroma/web/oauth/token.ex | 24 ++++++++++------ lib/pleroma/web/oauth/token/clean_worker.ex | 2 -- lib/pleroma/web/oauth/token/query.ex | 6 ---- .../web/oauth/token/strategy/refresh_token.ex | 2 +- .../workers/cron/clear_oauth_token_worker.ex | 23 --------------- lib/pleroma/workers/purge_expired_token.ex | 28 +++++++++++++++++++ test/plugs/oauth_plug_test.exs | 2 +- test/web/oauth/token_test.exs | 13 --------- .../twitter_api/password_controller_test.exs | 4 +-- .../cron/clear_oauth_token_worker_test.exs | 22 --------------- .../purge_expired_oauth_token_test.exs | 27 ++++++++++++++++++ 13 files changed, 76 insertions(+), 80 deletions(-) delete mode 100644 lib/pleroma/workers/cron/clear_oauth_token_worker.ex create mode 100644 lib/pleroma/workers/purge_expired_token.ex delete mode 100644 test/workers/cron/clear_oauth_token_worker_test.exs create mode 100644 test/workers/purge_expired_oauth_token_test.exs diff --git a/config/config.exs b/config/config.exs index 1a2b312b5..fa4c96b79 100644 --- a/config/config.exs +++ b/config/config.exs @@ -530,6 +530,7 @@ log: false, queues: [ activity_expiration: 10, + oauth_token_expiration: 1, federator_incoming: 50, federator_outgoing: 50, web_push: 50, @@ -543,7 +544,6 @@ ], plugins: [Oban.Plugins.Pruner], crontab: [ - {"0 0 * * *", Pleroma.Workers.Cron.ClearOauthTokenWorker}, {"* * * * *", Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker}, {"0 0 * * 0", Pleroma.Workers.Cron.DigestEmailsWorker}, {"0 0 * * *", Pleroma.Workers.Cron.NewUsersDigestWorker} diff --git a/config/description.exs b/config/description.exs index eac97ad64..4c4deed30 100644 --- a/config/description.exs +++ b/config/description.exs @@ -2290,7 +2290,6 @@ type: {:list, :tuple}, description: "Settings for cron background jobs", suggestions: [ - {"0 0 * * *", Pleroma.Workers.Cron.ClearOauthTokenWorker}, {"* * * * *", Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker}, {"0 0 * * 0", Pleroma.Workers.Cron.DigestEmailsWorker}, {"0 0 * * *", Pleroma.Workers.Cron.NewUsersDigestWorker} diff --git a/lib/pleroma/web/oauth/token.ex b/lib/pleroma/web/oauth/token.ex index 08bb7326d..4d00fcb1c 100644 --- a/lib/pleroma/web/oauth/token.ex +++ b/lib/pleroma/web/oauth/token.ex @@ -50,7 +50,7 @@ def exchange_token(app, auth) do true <- auth.app_id == app.id do user = if auth.user_id, do: User.get_cached_by_id(auth.user_id), else: %User{} - create_token( + create( app, user, %{scopes: auth.scopes} @@ -83,8 +83,21 @@ defp put_valid_until(changeset, attrs) do |> validate_required([:valid_until]) end - @spec create_token(App.t(), User.t(), map()) :: {:ok, Token} | {:error, Changeset.t()} - def create_token(%App{} = app, %User{} = user, attrs \\ %{}) do + @spec create(App.t(), User.t(), map()) :: {:ok, Token} | {:error, Changeset.t()} + def create(%App{} = app, %User{} = user, attrs \\ %{}) do + with {:ok, token} <- do_create(app, user, attrs) do + if Pleroma.Config.get([:oauth2, :clean_expired_tokens]) do + Pleroma.Workers.PurgeExpiredOAuthToken.enqueue(%{ + token_id: token.id, + valid_until: DateTime.from_naive!(token.valid_until, "Etc/UTC") + }) + end + + {:ok, token} + end + end + + defp do_create(app, user, attrs) do %__MODULE__{user_id: user.id, app_id: app.id} |> cast(%{scopes: attrs[:scopes] || app.scopes}, [:scopes]) |> validate_required([:scopes, :app_id]) @@ -105,11 +118,6 @@ def delete_user_token(%User{id: user_id}, token_id) do |> Repo.delete_all() end - def delete_expired_tokens do - Query.get_expired_tokens() - |> Repo.delete_all() - end - def get_user_tokens(%User{id: user_id}) do Query.get_by_user(user_id) |> Query.preload([:app]) diff --git a/lib/pleroma/web/oauth/token/clean_worker.ex b/lib/pleroma/web/oauth/token/clean_worker.ex index e3aa4eb7e..2f51bdb75 100644 --- a/lib/pleroma/web/oauth/token/clean_worker.ex +++ b/lib/pleroma/web/oauth/token/clean_worker.ex @@ -12,7 +12,6 @@ defmodule Pleroma.Web.OAuth.Token.CleanWorker do @one_day 86_400_000 alias Pleroma.MFA - alias Pleroma.Web.OAuth alias Pleroma.Workers.BackgroundWorker def start_link(_), do: GenServer.start_link(__MODULE__, %{}) @@ -32,7 +31,6 @@ def handle_info(:perform, state) do end def perform(:clean) do - OAuth.Token.delete_expired_tokens() MFA.Token.delete_expired_tokens() end end diff --git a/lib/pleroma/web/oauth/token/query.ex b/lib/pleroma/web/oauth/token/query.ex index 93d6e26ed..fd6d9b112 100644 --- a/lib/pleroma/web/oauth/token/query.ex +++ b/lib/pleroma/web/oauth/token/query.ex @@ -33,12 +33,6 @@ def get_by_id(query \\ Token, id) do from(q in query, where: q.id == ^id) end - @spec get_expired_tokens(query, DateTime.t() | nil) :: query - def get_expired_tokens(query \\ Token, date \\ nil) do - expired_date = date || Timex.now() - from(q in query, where: fragment("?", q.valid_until) < ^expired_date) - end - @spec get_by_user(query, String.t()) :: query def get_by_user(query \\ Token, user_id) do from(q in query, where: q.user_id == ^user_id) diff --git a/lib/pleroma/web/oauth/token/strategy/refresh_token.ex b/lib/pleroma/web/oauth/token/strategy/refresh_token.ex index debc29b0b..625b0fde2 100644 --- a/lib/pleroma/web/oauth/token/strategy/refresh_token.ex +++ b/lib/pleroma/web/oauth/token/strategy/refresh_token.ex @@ -46,7 +46,7 @@ defp revoke_access_token(token) do defp create_access_token({:error, error}, _), do: {:error, error} defp create_access_token({:ok, token}, %{app: app, user: user} = token_params) do - Token.create_token(app, user, add_refresh_token(token_params, token.refresh_token)) + Token.create(app, user, add_refresh_token(token_params, token.refresh_token)) end defp add_refresh_token(params, token) do diff --git a/lib/pleroma/workers/cron/clear_oauth_token_worker.ex b/lib/pleroma/workers/cron/clear_oauth_token_worker.ex deleted file mode 100644 index 276f47efc..000000000 --- a/lib/pleroma/workers/cron/clear_oauth_token_worker.ex +++ /dev/null @@ -1,23 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Workers.Cron.ClearOauthTokenWorker do - @moduledoc """ - The worker to cleanup expired oAuth tokens. - """ - - use Oban.Worker, queue: "background" - - alias Pleroma.Config - alias Pleroma.Web.OAuth.Token - - @impl Oban.Worker - def perform(_job) do - if Config.get([:oauth2, :clean_expired_tokens], false) do - Token.delete_expired_tokens() - end - - :ok - end -end diff --git a/lib/pleroma/workers/purge_expired_token.ex b/lib/pleroma/workers/purge_expired_token.ex new file mode 100644 index 000000000..6068e43bf --- /dev/null +++ b/lib/pleroma/workers/purge_expired_token.ex @@ -0,0 +1,28 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.PurgeExpiredOAuthToken do + @moduledoc """ + Worker which purges expired OAuth tokens + """ + + use Oban.Worker, queue: :oauth_token_expiration, max_attempts: 1 + + @spec enqueue(%{token_id: integer(), valid_until: DateTime.t()}) :: + {:ok, Oban.Job.t()} | {:error, Ecto.Changeset.t()} + def enqueue(args) do + {scheduled_at, args} = Map.pop(args, :valid_until) + + args + |> __MODULE__.new(scheduled_at: scheduled_at) + |> Oban.insert() + end + + @impl true + def perform(%Oban.Job{args: %{"token_id" => id}}) do + Pleroma.Web.OAuth.Token + |> Pleroma.Repo.get(id) + |> Pleroma.Repo.delete() + end +end diff --git a/test/plugs/oauth_plug_test.exs b/test/plugs/oauth_plug_test.exs index f74c068cd..9d39d3153 100644 --- a/test/plugs/oauth_plug_test.exs +++ b/test/plugs/oauth_plug_test.exs @@ -16,7 +16,7 @@ defmodule Pleroma.Plugs.OAuthPlugTest do setup %{conn: conn} do user = insert(:user) - {:ok, %{token: token}} = Pleroma.Web.OAuth.Token.create_token(insert(:oauth_app), user) + {:ok, %{token: token}} = Pleroma.Web.OAuth.Token.create(insert(:oauth_app), user) %{user: user, token: token, conn: conn} end diff --git a/test/web/oauth/token_test.exs b/test/web/oauth/token_test.exs index 40d71eb59..c88b9cc98 100644 --- a/test/web/oauth/token_test.exs +++ b/test/web/oauth/token_test.exs @@ -69,17 +69,4 @@ test "deletes all tokens of a user" do assert tokens == 2 end - - test "deletes expired tokens" do - insert(:oauth_token, valid_until: Timex.shift(Timex.now(), days: -3)) - insert(:oauth_token, valid_until: Timex.shift(Timex.now(), days: -3)) - t3 = insert(:oauth_token) - t4 = insert(:oauth_token, valid_until: Timex.shift(Timex.now(), minutes: 10)) - {tokens, _} = Token.delete_expired_tokens() - assert tokens == 2 - available_tokens = Pleroma.Repo.all(Token) - - token_ids = available_tokens |> Enum.map(& &1.id) - assert token_ids == [t3.id, t4.id] - end end diff --git a/test/web/twitter_api/password_controller_test.exs b/test/web/twitter_api/password_controller_test.exs index 231a46c67..a5e9e2178 100644 --- a/test/web/twitter_api/password_controller_test.exs +++ b/test/web/twitter_api/password_controller_test.exs @@ -37,7 +37,7 @@ test "it shows password reset form", %{conn: conn} do test "it returns HTTP 200", %{conn: conn} do user = insert(:user) {:ok, token} = PasswordResetToken.create_token(user) - {:ok, _access_token} = Token.create_token(insert(:oauth_app), user, %{}) + {:ok, _access_token} = Token.create(insert(:oauth_app), user, %{}) params = %{ "password" => "test", @@ -62,7 +62,7 @@ test "it sets password_reset_pending to false", %{conn: conn} do user = insert(:user, password_reset_pending: true) {:ok, token} = PasswordResetToken.create_token(user) - {:ok, _access_token} = Token.create_token(insert(:oauth_app), user, %{}) + {:ok, _access_token} = Token.create(insert(:oauth_app), user, %{}) params = %{ "password" => "test", diff --git a/test/workers/cron/clear_oauth_token_worker_test.exs b/test/workers/cron/clear_oauth_token_worker_test.exs deleted file mode 100644 index 67836f34f..000000000 --- a/test/workers/cron/clear_oauth_token_worker_test.exs +++ /dev/null @@ -1,22 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Workers.Cron.ClearOauthTokenWorkerTest do - use Pleroma.DataCase - - import Pleroma.Factory - alias Pleroma.Workers.Cron.ClearOauthTokenWorker - - setup do: clear_config([:oauth2, :clean_expired_tokens]) - - test "deletes expired tokens" do - insert(:oauth_token, - valid_until: NaiveDateTime.add(NaiveDateTime.utc_now(), -60 * 10) - ) - - Pleroma.Config.put([:oauth2, :clean_expired_tokens], true) - ClearOauthTokenWorker.perform(%Oban.Job{}) - assert Pleroma.Repo.all(Pleroma.Web.OAuth.Token) == [] - end -end diff --git a/test/workers/purge_expired_oauth_token_test.exs b/test/workers/purge_expired_oauth_token_test.exs new file mode 100644 index 000000000..3bd650d89 --- /dev/null +++ b/test/workers/purge_expired_oauth_token_test.exs @@ -0,0 +1,27 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.PurgeExpiredOAuthTokenTest do + use Pleroma.DataCase, async: true + use Oban.Testing, repo: Pleroma.Repo + + import Pleroma.Factory + + setup do: clear_config([:oauth2, :clean_expired_tokens], true) + + test "purges expired token" do + user = insert(:user) + app = insert(:oauth_app) + + {:ok, %{id: id}} = Pleroma.Web.OAuth.Token.create(app, user) + + assert_enqueued( + worker: Pleroma.Workers.PurgeExpiredOAuthToken, + args: %{token_id: id} + ) + + assert {:ok, %{id: ^id}} = + perform_job(Pleroma.Workers.PurgeExpiredOAuthToken, %{token_id: id}) + end +end From 7dd986a563545cb63e8404d9b107f1d29c499940 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Sat, 5 Sep 2020 18:35:01 +0300 Subject: [PATCH 098/128] expire mfa tokens through Oban --- config/config.exs | 2 +- docs/configuration/cheatsheet.md | 10 +-- lib/pleroma/mfa/token.ex | 71 +++++++++---------- lib/pleroma/web/oauth/oauth_controller.ex | 4 +- lib/pleroma/web/oauth/token.ex | 5 +- lib/pleroma/web/oauth/token/clean_worker.ex | 36 ---------- .../controllers/remote_follow_controller.ex | 2 +- lib/pleroma/workers/purge_expired_token.ex | 11 +-- .../remote_follow_controller_test.exs | 4 +- .../purge_expired_oauth_token_test.exs | 27 ------- test/workers/purge_expired_token_test.exs | 51 +++++++++++++ 11 files changed, 106 insertions(+), 117 deletions(-) delete mode 100644 lib/pleroma/web/oauth/token/clean_worker.ex delete mode 100644 test/workers/purge_expired_oauth_token_test.exs create mode 100644 test/workers/purge_expired_token_test.exs diff --git a/config/config.exs b/config/config.exs index fa4c96b79..95a6ea9db 100644 --- a/config/config.exs +++ b/config/config.exs @@ -530,7 +530,7 @@ log: false, queues: [ activity_expiration: 10, - oauth_token_expiration: 1, + token_expiration: 5, federator_incoming: 50, federator_outgoing: 50, web_push: 50, diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index ec59896ec..d0bebbd45 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -691,9 +691,8 @@ Pleroma has the following queues: Pleroma has these periodic job workers: -`Pleroma.Workers.Cron.ClearOauthTokenWorker` - a job worker to cleanup expired oauth tokens. - -Example: +* `Pleroma.Workers.Cron.DigestEmailsWorker` - digest emails for users with new mentions and follows +* `Pleroma.Workers.Cron.NewUsersDigestWorker` - digest emails for admins with new registrations ```elixir config :pleroma, Oban, @@ -705,7 +704,8 @@ config :pleroma, Oban, federator_outgoing: 50 ], crontab: [ - {"0 0 * * *", Pleroma.Workers.Cron.ClearOauthTokenWorker} + {"0 0 * * 0", Pleroma.Workers.Cron.DigestEmailsWorker}, + {"0 0 * * *", Pleroma.Workers.Cron.NewUsersDigestWorker} ] ``` @@ -972,7 +972,7 @@ Configure OAuth 2 provider capabilities: * `token_expires_in` - The lifetime in seconds of the access token. * `issue_new_refresh_token` - Keeps old refresh token or generate new refresh token when to obtain an access token. -* `clean_expired_tokens` - Enable a background job to clean expired oauth tokens. Defaults to `false`. Interval settings sets in configuration periodic jobs [`Oban.Cron`](#obancron) +* `clean_expired_tokens` - Enable a background job to clean expired oauth tokens. Defaults to `false`. ## Link parsing diff --git a/lib/pleroma/mfa/token.ex b/lib/pleroma/mfa/token.ex index 0b2449971..69b64c0e8 100644 --- a/lib/pleroma/mfa/token.ex +++ b/lib/pleroma/mfa/token.ex @@ -10,10 +10,11 @@ defmodule Pleroma.MFA.Token do alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.OAuth.Authorization - alias Pleroma.Web.OAuth.Token, as: OAuthToken @expires 300 + @type t() :: %__MODULE__{} + schema "mfa_tokens" do field(:token, :string) field(:valid_until, :naive_datetime_usec) @@ -24,6 +25,7 @@ defmodule Pleroma.MFA.Token do timestamps() end + @spec get_by_token(String.t()) :: {:ok, t()} | {:error, :not_found} def get_by_token(token) do from( t in __MODULE__, @@ -33,33 +35,40 @@ def get_by_token(token) do |> Repo.find_resource() end - def validate(token) do - with {:fetch_token, {:ok, token}} <- {:fetch_token, get_by_token(token)}, - {:expired, false} <- {:expired, is_expired?(token)} do + @spec validate(String.t()) :: {:ok, t()} | {:error, :not_found} | {:error, :expired_token} + def validate(token_str) do + with {:ok, token} <- get_by_token(token_str), + false <- expired?(token) do {:ok, token} - else - {:expired, _} -> {:error, :expired_token} - {:fetch_token, _} -> {:error, :not_found} - error -> {:error, error} end end - def create_token(%User{} = user) do - %__MODULE__{} - |> change - |> assign_user(user) - |> put_token - |> put_valid_until - |> Repo.insert() + defp expired?(%__MODULE__{valid_until: valid_until}) do + with true <- NaiveDateTime.diff(NaiveDateTime.utc_now(), valid_until) > 0 do + {:error, :expired_token} + end end - def create_token(user, authorization) do + @spec create(User.t(), Authorization.t() | nil) :: {:ok, t()} | {:error, Ecto.Changeset.t()} + def create(user, authorization \\ nil) do + with {:ok, token} <- do_create(user, authorization) do + Pleroma.Workers.PurgeExpiredToken.enqueue(%{ + token_id: token.id, + valid_until: DateTime.from_naive!(token.valid_until, "Etc/UTC"), + mod: __MODULE__ + }) + + {:ok, token} + end + end + + defp do_create(user, authorization) do %__MODULE__{} - |> change + |> change() |> assign_user(user) - |> assign_authorization(authorization) - |> put_token - |> put_valid_until + |> maybe_assign_authorization(authorization) + |> put_token() + |> put_valid_until() |> Repo.insert() end @@ -69,15 +78,19 @@ defp assign_user(changeset, user) do |> validate_required([:user]) end - defp assign_authorization(changeset, authorization) do + defp maybe_assign_authorization(changeset, %Authorization{} = authorization) do changeset |> put_assoc(:authorization, authorization) |> validate_required([:authorization]) end + defp maybe_assign_authorization(changeset, _), do: changeset + defp put_token(changeset) do + token = Pleroma.Web.OAuth.Token.Utils.generate_token() + changeset - |> change(%{token: OAuthToken.Utils.generate_token()}) + |> change(%{token: token}) |> validate_required([:token]) |> unique_constraint(:token) end @@ -89,18 +102,4 @@ defp put_valid_until(changeset) do |> change(%{valid_until: expires_in}) |> validate_required([:valid_until]) end - - def is_expired?(%__MODULE__{valid_until: valid_until}) do - NaiveDateTime.diff(NaiveDateTime.utc_now(), valid_until) > 0 - end - - def is_expired?(_), do: false - - def delete_expired_tokens do - from( - q in __MODULE__, - where: fragment("?", q.valid_until) < ^Timex.now() - ) - |> Repo.delete_all() - end end diff --git a/lib/pleroma/web/oauth/oauth_controller.ex b/lib/pleroma/web/oauth/oauth_controller.ex index dd00600ea..bbe7aa8a0 100644 --- a/lib/pleroma/web/oauth/oauth_controller.ex +++ b/lib/pleroma/web/oauth/oauth_controller.ex @@ -197,7 +197,7 @@ defp handle_create_authorization_error( {:mfa_required, user, auth, _}, params ) do - {:ok, token} = MFA.Token.create_token(user, auth) + {:ok, token} = MFA.Token.create(user, auth) data = %{ "mfa_token" => token.token, @@ -579,7 +579,7 @@ defp put_session_registration_id(%Plug.Conn{} = conn, registration_id), do: put_session(conn, :registration_id, registration_id) defp build_and_response_mfa_token(user, auth) do - with {:ok, token} <- MFA.Token.create_token(user, auth) do + with {:ok, token} <- MFA.Token.create(user, auth) do MFAView.render("mfa_response.json", %{token: token, user: user}) end end diff --git a/lib/pleroma/web/oauth/token.ex b/lib/pleroma/web/oauth/token.ex index 4d00fcb1c..de37998f2 100644 --- a/lib/pleroma/web/oauth/token.ex +++ b/lib/pleroma/web/oauth/token.ex @@ -87,9 +87,10 @@ defp put_valid_until(changeset, attrs) do def create(%App{} = app, %User{} = user, attrs \\ %{}) do with {:ok, token} <- do_create(app, user, attrs) do if Pleroma.Config.get([:oauth2, :clean_expired_tokens]) do - Pleroma.Workers.PurgeExpiredOAuthToken.enqueue(%{ + Pleroma.Workers.PurgeExpiredToken.enqueue(%{ token_id: token.id, - valid_until: DateTime.from_naive!(token.valid_until, "Etc/UTC") + valid_until: DateTime.from_naive!(token.valid_until, "Etc/UTC"), + mod: __MODULE__ }) end diff --git a/lib/pleroma/web/oauth/token/clean_worker.ex b/lib/pleroma/web/oauth/token/clean_worker.ex deleted file mode 100644 index 2f51bdb75..000000000 --- a/lib/pleroma/web/oauth/token/clean_worker.ex +++ /dev/null @@ -1,36 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.OAuth.Token.CleanWorker do - @moduledoc """ - The module represents functions to clean an expired OAuth and MFA tokens. - """ - use GenServer - - @ten_seconds 10_000 - @one_day 86_400_000 - - alias Pleroma.MFA - alias Pleroma.Workers.BackgroundWorker - - def start_link(_), do: GenServer.start_link(__MODULE__, %{}) - - def init(_) do - Process.send_after(self(), :perform, @ten_seconds) - {:ok, nil} - end - - @doc false - def handle_info(:perform, state) do - BackgroundWorker.enqueue("clean_expired_tokens", %{}) - interval = Pleroma.Config.get([:oauth2, :clean_expired_tokens_interval], @one_day) - - Process.send_after(self(), :perform, interval) - {:noreply, state} - end - - def perform(:clean) do - MFA.Token.delete_expired_tokens() - end -end diff --git a/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex b/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex index 521dc9322..072d889e2 100644 --- a/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex @@ -135,7 +135,7 @@ defp handle_follow_error(conn, {:verify_mfa_code, followee, token, _} = _) do end defp handle_follow_error(conn, {:mfa_required, followee, user, _} = _) do - {:ok, %{token: token}} = MFA.Token.create_token(user) + {:ok, %{token: token}} = MFA.Token.create(user) render(conn, "follow_mfa.html", %{followee: followee, mfa_token: token, error: false}) end diff --git a/lib/pleroma/workers/purge_expired_token.ex b/lib/pleroma/workers/purge_expired_token.ex index 6068e43bf..a81e0cd28 100644 --- a/lib/pleroma/workers/purge_expired_token.ex +++ b/lib/pleroma/workers/purge_expired_token.ex @@ -2,14 +2,14 @@ # Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Workers.PurgeExpiredOAuthToken do +defmodule Pleroma.Workers.PurgeExpiredToken do @moduledoc """ Worker which purges expired OAuth tokens """ - use Oban.Worker, queue: :oauth_token_expiration, max_attempts: 1 + use Oban.Worker, queue: :token_expiration, max_attempts: 1 - @spec enqueue(%{token_id: integer(), valid_until: DateTime.t()}) :: + @spec enqueue(%{token_id: integer(), valid_until: DateTime.t(), mod: module()}) :: {:ok, Oban.Job.t()} | {:error, Ecto.Changeset.t()} def enqueue(args) do {scheduled_at, args} = Map.pop(args, :valid_until) @@ -20,8 +20,9 @@ def enqueue(args) do end @impl true - def perform(%Oban.Job{args: %{"token_id" => id}}) do - Pleroma.Web.OAuth.Token + def perform(%Oban.Job{args: %{"token_id" => id, "mod" => module}}) do + module + |> String.to_existing_atom() |> Pleroma.Repo.get(id) |> Pleroma.Repo.delete() end diff --git a/test/web/twitter_api/remote_follow_controller_test.exs b/test/web/twitter_api/remote_follow_controller_test.exs index f7e54c26a..3852c7ce9 100644 --- a/test/web/twitter_api/remote_follow_controller_test.exs +++ b/test/web/twitter_api/remote_follow_controller_test.exs @@ -227,7 +227,7 @@ test "follows", %{conn: conn} do } ) - {:ok, %{token: token}} = MFA.Token.create_token(user) + {:ok, %{token: token}} = MFA.Token.create(user) user2 = insert(:user) otp_token = TOTP.generate_token(otp_secret) @@ -256,7 +256,7 @@ test "returns error when auth code is incorrect", %{conn: conn} do } ) - {:ok, %{token: token}} = MFA.Token.create_token(user) + {:ok, %{token: token}} = MFA.Token.create(user) user2 = insert(:user) otp_token = TOTP.generate_token(TOTP.generate_secret()) diff --git a/test/workers/purge_expired_oauth_token_test.exs b/test/workers/purge_expired_oauth_token_test.exs deleted file mode 100644 index 3bd650d89..000000000 --- a/test/workers/purge_expired_oauth_token_test.exs +++ /dev/null @@ -1,27 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Workers.PurgeExpiredOAuthTokenTest do - use Pleroma.DataCase, async: true - use Oban.Testing, repo: Pleroma.Repo - - import Pleroma.Factory - - setup do: clear_config([:oauth2, :clean_expired_tokens], true) - - test "purges expired token" do - user = insert(:user) - app = insert(:oauth_app) - - {:ok, %{id: id}} = Pleroma.Web.OAuth.Token.create(app, user) - - assert_enqueued( - worker: Pleroma.Workers.PurgeExpiredOAuthToken, - args: %{token_id: id} - ) - - assert {:ok, %{id: ^id}} = - perform_job(Pleroma.Workers.PurgeExpiredOAuthToken, %{token_id: id}) - end -end diff --git a/test/workers/purge_expired_token_test.exs b/test/workers/purge_expired_token_test.exs new file mode 100644 index 000000000..fb7708c3f --- /dev/null +++ b/test/workers/purge_expired_token_test.exs @@ -0,0 +1,51 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.PurgeExpiredTokenTest do + use Pleroma.DataCase, async: true + use Oban.Testing, repo: Pleroma.Repo + + import Pleroma.Factory + + setup do: clear_config([:oauth2, :clean_expired_tokens], true) + + test "purges expired oauth token" do + user = insert(:user) + app = insert(:oauth_app) + + {:ok, %{id: id}} = Pleroma.Web.OAuth.Token.create(app, user) + + assert_enqueued( + worker: Pleroma.Workers.PurgeExpiredToken, + args: %{token_id: id, mod: Pleroma.Web.OAuth.Token} + ) + + assert {:ok, %{id: ^id}} = + perform_job(Pleroma.Workers.PurgeExpiredToken, %{ + token_id: id, + mod: Pleroma.Web.OAuth.Token + }) + + assert Repo.aggregate(Pleroma.Web.OAuth.Token, :count, :id) == 0 + end + + test "purges expired mfa token" do + authorization = insert(:oauth_authorization) + + {:ok, %{id: id}} = Pleroma.MFA.Token.create(authorization.user, authorization) + + assert_enqueued( + worker: Pleroma.Workers.PurgeExpiredToken, + args: %{token_id: id, mod: Pleroma.MFA.Token} + ) + + assert {:ok, %{id: ^id}} = + perform_job(Pleroma.Workers.PurgeExpiredToken, %{ + token_id: id, + mod: Pleroma.MFA.Token + }) + + assert Repo.aggregate(Pleroma.MFA.Token, :count, :id) == 0 + end +end From c6647c08e10a45aedcd77258a0e71c41d213eaa6 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Mon, 7 Sep 2020 11:54:10 +0300 Subject: [PATCH 099/128] migration and changelog --- CHANGELOG.md | 1 + ...ar_oauth_token_worker_from_oban_config.exs | 20 +++++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 priv/repo/migrations/20200907084956_remove_cron_clear_oauth_token_worker_from_oban_config.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index 19b2596cc..14c0252f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Removed - **Breaking:** Removed `Pleroma.Workers.Cron.StatsWorker` setting from Oban `:crontab`. +- **Breaking:** Removed `Pleroma.Workers.Cron.ClearOauthTokenWorker` setting from Oban `:crontab` config. ## [2.1.1] - 2020-09-08 diff --git a/priv/repo/migrations/20200907084956_remove_cron_clear_oauth_token_worker_from_oban_config.exs b/priv/repo/migrations/20200907084956_remove_cron_clear_oauth_token_worker_from_oban_config.exs new file mode 100644 index 000000000..d9c972563 --- /dev/null +++ b/priv/repo/migrations/20200907084956_remove_cron_clear_oauth_token_worker_from_oban_config.exs @@ -0,0 +1,20 @@ +defmodule Pleroma.Repo.Migrations.RemoveCronClearOauthTokenWorkerFromObanConfig do + use Ecto.Migration + + def change do + with %Pleroma.ConfigDB{} = config <- + Pleroma.ConfigDB.get_by_params(%{group: :pleroma, key: Oban}), + crontab when is_list(crontab) <- config.value[:crontab], + index when is_integer(index) <- + Enum.find_index(crontab, fn {_, worker} -> + worker == Pleroma.Workers.Cron.ClearOauthTokenWorker + end) do + updated_value = Keyword.put(config.value, :crontab, List.delete_at(crontab, index)) + + config + |> Ecto.Changeset.change(value: updated_value) + |> Pleroma.Repo.update() + end + + end +end From e11fca88d424b394359f50646e4b4ec9b3ae1a8b Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Mon, 7 Sep 2020 13:44:42 +0300 Subject: [PATCH 100/128] migration to move tokens expiration into Oban --- ...92050_move_tokens_expiration_into_oban.exs | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 priv/repo/migrations/20200907092050_move_tokens_expiration_into_oban.exs diff --git a/priv/repo/migrations/20200907092050_move_tokens_expiration_into_oban.exs b/priv/repo/migrations/20200907092050_move_tokens_expiration_into_oban.exs new file mode 100644 index 000000000..832bd02a7 --- /dev/null +++ b/priv/repo/migrations/20200907092050_move_tokens_expiration_into_oban.exs @@ -0,0 +1,36 @@ +defmodule Pleroma.Repo.Migrations.MoveTokensExpirationIntoOban do + use Ecto.Migration + + import Ecto.Query, only: [from: 2] + + def change do + Supervisor.start_link([{Oban, Pleroma.Config.get(Oban)}], + strategy: :one_for_one, + name: Pleroma.Supervisor + ) + + if Pleroma.Config.get([:oauth2, :clean_expired_tokens]) do + from(t in Pleroma.Web.OAuth.Token, where: t.valid_until > ^NaiveDateTime.utc_now()) + |> Pleroma.Repo.stream() + |> Stream.each(fn token -> + Pleroma.Workers.PurgeExpiredToken.enqueue(%{ + token_id: token.id, + valid_until: DateTime.from_naive!(token.valid_until, "Etc/UTC"), + mod: Pleroma.Web.OAuth.Token + }) + end) + |> Stream.run() + end + + from(t in Pleroma.MFA.Token, where: t.valid_until > ^NaiveDateTime.utc_now()) + |> Pleroma.Repo.stream() + |> Stream.each(fn token -> + Pleroma.Workers.PurgeExpiredToken.enqueue(%{ + token_id: token.id, + valid_until: DateTime.from_naive!(token.valid_until, "Etc/UTC"), + mod: Pleroma.MFA.Token + }) + end) + |> Stream.run() + end +end From eca42566ba62ece75b79915f4af0c8a0f0c48a17 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Mon, 7 Sep 2020 13:54:28 +0300 Subject: [PATCH 101/128] formatting --- ...6_remove_cron_clear_oauth_token_worker_from_oban_config.exs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/priv/repo/migrations/20200907084956_remove_cron_clear_oauth_token_worker_from_oban_config.exs b/priv/repo/migrations/20200907084956_remove_cron_clear_oauth_token_worker_from_oban_config.exs index d9c972563..b5a0a0ff6 100644 --- a/priv/repo/migrations/20200907084956_remove_cron_clear_oauth_token_worker_from_oban_config.exs +++ b/priv/repo/migrations/20200907084956_remove_cron_clear_oauth_token_worker_from_oban_config.exs @@ -2,7 +2,7 @@ defmodule Pleroma.Repo.Migrations.RemoveCronClearOauthTokenWorkerFromObanConfig use Ecto.Migration def change do - with %Pleroma.ConfigDB{} = config <- + with %Pleroma.ConfigDB{} = config <- Pleroma.ConfigDB.get_by_params(%{group: :pleroma, key: Oban}), crontab when is_list(crontab) <- config.value[:crontab], index when is_integer(index) <- @@ -15,6 +15,5 @@ def change do |> Ecto.Changeset.change(value: updated_value) |> Pleroma.Repo.update() end - end end From 8af1fd32234df7d0cdb74d78bcca9f68587b70f2 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Mon, 7 Sep 2020 20:06:28 +0300 Subject: [PATCH 102/128] oban warning --- lib/pleroma/config/oban.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/config/oban.ex b/lib/pleroma/config/oban.ex index c2d56ebab..81758c93d 100644 --- a/lib/pleroma/config/oban.ex +++ b/lib/pleroma/config/oban.ex @@ -5,7 +5,7 @@ def warn do oban_config = Pleroma.Config.get(Oban) crontab = - [Pleroma.Workers.Cron.StatsWorker] + [Pleroma.Workers.Cron.StatsWorker, Pleroma.Workers.Cron.ClearOauthTokenWorker] |> Enum.reduce(oban_config[:crontab], fn removed_worker, acc -> with acc when is_list(acc) <- acc, setting when is_tuple(setting) <- From e8bfb50fa3c16f98845e326b153c8a89505e8a55 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 10 Sep 2020 20:09:44 +0300 Subject: [PATCH 103/128] pass options without adapter key --- lib/pleroma/reverse_proxy/client/tesla.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/reverse_proxy/client/tesla.ex b/lib/pleroma/reverse_proxy/client/tesla.ex index d5a339681..4b118eec2 100644 --- a/lib/pleroma/reverse_proxy/client/tesla.ex +++ b/lib/pleroma/reverse_proxy/client/tesla.ex @@ -28,7 +28,7 @@ def request(method, url, headers, body, opts \\ []) do url, body, headers, - Keyword.put(opts, :adapter, opts) + opts ) do if is_map(response.body) and method != :head do {:ok, response.status, response.headers, response.body} From cb06e98da27994ac8034f3ba387b6eeaf8a2c48f Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 10 Sep 2020 13:47:53 +0300 Subject: [PATCH 104/128] websocket handler: Do not log client ping frames as errors --- lib/pleroma/web/mastodon_api/websocket_handler.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/websocket_handler.ex b/lib/pleroma/web/mastodon_api/websocket_handler.ex index 94e4595d8..e6010bb4a 100644 --- a/lib/pleroma/web/mastodon_api/websocket_handler.ex +++ b/lib/pleroma/web/mastodon_api/websocket_handler.ex @@ -64,7 +64,9 @@ def websocket_handle(:pong, state) do {:ok, %{state | timer: timer()}} end - # We never receive messages. + # We only receive pings for now + def websocket_handle(:ping, state), do: {:ok, state} + def websocket_handle(frame, state) do Logger.error("#{__MODULE__} received frame: #{inspect(frame)}") {:ok, state} From e16e8f98169f822416c18778abfa8495a486c8f2 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 10 Sep 2020 13:48:24 +0300 Subject: [PATCH 105/128] Websocket handler: do not raise if handler is terminated before switching protocols Closes #2131 --- lib/pleroma/web/mastodon_api/websocket_handler.ex | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/pleroma/web/mastodon_api/websocket_handler.ex b/lib/pleroma/web/mastodon_api/websocket_handler.ex index e6010bb4a..5090d9622 100644 --- a/lib/pleroma/web/mastodon_api/websocket_handler.ex +++ b/lib/pleroma/web/mastodon_api/websocket_handler.ex @@ -100,6 +100,10 @@ def websocket_info(:tick, state) do {:reply, :ping, %{state | timer: nil, count: 0}, :hibernate} end + # State can be `[]` only in case we terminate before switching to websocket, + # we already log errors for these cases in `init/1`, so just do nothing here + def terminate(_reason, _req, []), do: :ok + def terminate(reason, _req, state) do Logger.debug( "#{__MODULE__} terminating websocket connection for user #{ From 01fa68fe4542286519e3520793c6b59103b050ff Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 10 Sep 2020 21:26:52 +0300 Subject: [PATCH 106/128] Websocket handler: fix never matching code on failed auth `:cowboy_req.reply` does not return tuples since 2.0, see https://ninenines.eu/docs/en/cowboy/2.4/manual/cowboy_req.reply/ --- lib/pleroma/web/mastodon_api/websocket_handler.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/websocket_handler.ex b/lib/pleroma/web/mastodon_api/websocket_handler.ex index 5090d9622..cf923ded8 100644 --- a/lib/pleroma/web/mastodon_api/websocket_handler.ex +++ b/lib/pleroma/web/mastodon_api/websocket_handler.ex @@ -37,12 +37,12 @@ def init(%{qs: qs} = req, state) do else {:error, :bad_topic} -> Logger.debug("#{__MODULE__} bad topic #{inspect(req)}") - {:ok, req} = :cowboy_req.reply(404, req) + req = :cowboy_req.reply(404, req) {:ok, req, state} {:error, :unauthorized} -> Logger.debug("#{__MODULE__} authentication error: #{inspect(req)}") - {:ok, req} = :cowboy_req.reply(401, req) + req = :cowboy_req.reply(401, req) {:ok, req, state} end end From 275602daa7c4a01dffb83759012554da2d9335fe Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 10 Sep 2020 21:28:47 +0300 Subject: [PATCH 107/128] Streaming integration tests: remove unexpected error assumption For some reason instead of fixing unexpected errors, we made tests assert they indeed trigger... Now that the errors are fixed these were failing --- test/integration/mastodon_websocket_test.exs | 26 ++++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/test/integration/mastodon_websocket_test.exs b/test/integration/mastodon_websocket_test.exs index ea17e9feb..76fbc8bda 100644 --- a/test/integration/mastodon_websocket_test.exs +++ b/test/integration/mastodon_websocket_test.exs @@ -99,30 +99,30 @@ test "accepts valid tokens", state do test "accepts the 'user' stream", %{token: token} = _state do assert {:ok, _} = start_socket("?stream=user&access_token=#{token.token}") - assert capture_log(fn -> - assert {:error, {401, _}} = start_socket("?stream=user") - Process.sleep(30) - end) =~ ":badarg" + capture_log(fn -> + assert {:error, {401, _}} = start_socket("?stream=user") + Process.sleep(30) + end) end test "accepts the 'user:notification' stream", %{token: token} = _state do assert {:ok, _} = start_socket("?stream=user:notification&access_token=#{token.token}") - assert capture_log(fn -> - assert {:error, {401, _}} = start_socket("?stream=user:notification") - Process.sleep(30) - end) =~ ":badarg" + capture_log(fn -> + assert {:error, {401, _}} = start_socket("?stream=user:notification") + Process.sleep(30) + end) end test "accepts valid token on Sec-WebSocket-Protocol header", %{token: token} do assert {:ok, _} = start_socket("?stream=user", [{"Sec-WebSocket-Protocol", token.token}]) - assert capture_log(fn -> - assert {:error, {401, _}} = - start_socket("?stream=user", [{"Sec-WebSocket-Protocol", "I am a friend"}]) + capture_log(fn -> + assert {:error, {401, _}} = + start_socket("?stream=user", [{"Sec-WebSocket-Protocol", "I am a friend"}]) - Process.sleep(30) - end) =~ ":badarg" + Process.sleep(30) + end) end end end From 9bf1065a06837b4c753549d89afe23a636a20972 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Sat, 22 Aug 2020 20:46:01 +0300 Subject: [PATCH 108/128] schedule activity expiration in Oban --- config/config.exs | 3 +- config/description.exs | 1 - lib/mix/tasks/pleroma/database.ex | 18 ++-- lib/pleroma/activity.ex | 3 - lib/pleroma/activity_expiration.ex | 74 ---------------- lib/pleroma/web/activity_pub/activity_pub.ex | 7 +- .../mrf/activity_expiration_policy.ex | 4 +- lib/pleroma/web/activity_pub/side_effects.ex | 6 +- lib/pleroma/web/common_api/activity_draft.ex | 2 +- lib/pleroma/web/common_api/common_api.ex | 5 +- .../web/mastodon_api/views/status_view.ex | 5 +- .../cron/purge_expired_activities_worker.ex | 48 ----------- lib/pleroma/workers/purge_expired_activity.ex | 72 ++++++++++++++++ test/activity_expiration_test.exs | 55 ------------ test/activity_test.exs | 9 -- test/support/factory.ex | 19 ----- test/tasks/database_test.exs | 62 +++++++------- test/web/activity_pub/activity_pub_test.exs | 25 ++++-- .../mrf/activity_expiration_policy_test.exs | 8 +- test/web/common_api/common_api_test.exs | 14 ++-- .../controllers/status_controller_test.exs | 44 +++++----- .../purge_expired_activities_worker_test.exs | 84 ------------------- test/workers/purge_expired_activity_test.exs | 47 +++++++++++ 23 files changed, 229 insertions(+), 386 deletions(-) delete mode 100644 lib/pleroma/activity_expiration.ex delete mode 100644 lib/pleroma/workers/cron/purge_expired_activities_worker.ex create mode 100644 lib/pleroma/workers/purge_expired_activity.ex delete mode 100644 test/activity_expiration_test.exs delete mode 100644 test/workers/cron/purge_expired_activities_worker_test.exs create mode 100644 test/workers/purge_expired_activity_test.exs diff --git a/config/config.exs b/config/config.exs index 95a6ea9db..d975db31e 100644 --- a/config/config.exs +++ b/config/config.exs @@ -544,7 +544,6 @@ ], plugins: [Oban.Plugins.Pruner], crontab: [ - {"* * * * *", Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker}, {"0 0 * * 0", Pleroma.Workers.Cron.DigestEmailsWorker}, {"0 0 * * *", Pleroma.Workers.Cron.NewUsersDigestWorker} ] @@ -655,7 +654,7 @@ account_confirmation_resend: {8_640_000, 5}, ap_routes: {60_000, 15} -config :pleroma, Pleroma.ActivityExpiration, enabled: true +config :pleroma, Pleroma.Workers.PurgeExpiredActivity, enabled: true config :pleroma, Pleroma.Plugs.RemoteIp, enabled: true diff --git a/config/description.exs b/config/description.exs index 4c4deed30..6ce27278c 100644 --- a/config/description.exs +++ b/config/description.exs @@ -2290,7 +2290,6 @@ type: {:list, :tuple}, description: "Settings for cron background jobs", suggestions: [ - {"* * * * *", Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker}, {"0 0 * * 0", Pleroma.Workers.Cron.DigestEmailsWorker}, {"0 0 * * *", Pleroma.Workers.Cron.NewUsersDigestWorker} ] diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index 7d8f00b08..aab4b5e9a 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -133,8 +133,7 @@ def run(["ensure_expiration"]) do days = Pleroma.Config.get([:mrf_activity_expiration, :days], 365) Pleroma.Activity - |> join(:left, [a], u in assoc(a, :expiration)) - |> join(:inner, [a, _u], o in Object, + |> join(:inner, [a], o in Object, on: fragment( "(?->>'id') = COALESCE((?)->'object'->> 'id', (?)->>'object')", @@ -144,14 +143,21 @@ def run(["ensure_expiration"]) do ) ) |> where(local: true) - |> where([a, u], is_nil(u)) |> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data)) - |> where([_a, _u, o], fragment("?->>'type' = 'Note'", o.data)) + |> where([_a, o], fragment("?->>'type' = 'Note'", o.data)) |> Pleroma.RepoStreamer.chunk_stream(100) |> Stream.each(fn activities -> Enum.each(activities, fn activity -> - expires_at = Timex.shift(activity.inserted_at, days: days) - Pleroma.ActivityExpiration.create(activity, expires_at, false) + expires_at = + activity.inserted_at + |> DateTime.from_naive!("Etc/UTC") + |> Timex.shift(days: days) + + Pleroma.Workers.PurgeExpiredActivity.enqueue(%{ + activity_id: activity.id, + expires_at: expires_at, + validate: false + }) end) end) |> Stream.run() diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 97feebeaa..03cd3b8c0 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -7,7 +7,6 @@ defmodule Pleroma.Activity do alias Pleroma.Activity alias Pleroma.Activity.Queries - alias Pleroma.ActivityExpiration alias Pleroma.Bookmark alias Pleroma.Notification alias Pleroma.Object @@ -60,8 +59,6 @@ defmodule Pleroma.Activity do # typical case. has_one(:object, Object, on_delete: :nothing, foreign_key: :id) - has_one(:expiration, ActivityExpiration, on_delete: :delete_all) - timestamps() end diff --git a/lib/pleroma/activity_expiration.ex b/lib/pleroma/activity_expiration.ex deleted file mode 100644 index 955f0578e..000000000 --- a/lib/pleroma/activity_expiration.ex +++ /dev/null @@ -1,74 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.ActivityExpiration do - use Ecto.Schema - - alias Pleroma.Activity - alias Pleroma.ActivityExpiration - alias Pleroma.Repo - - import Ecto.Changeset - import Ecto.Query - - @type t :: %__MODULE__{} - @min_activity_lifetime :timer.hours(1) - - schema "activity_expirations" do - belongs_to(:activity, Activity, type: FlakeId.Ecto.CompatType) - field(:scheduled_at, :naive_datetime) - end - - def changeset(%ActivityExpiration{} = expiration, attrs, validate_scheduled_at) do - expiration - |> cast(attrs, [:scheduled_at]) - |> validate_required([:scheduled_at]) - |> validate_scheduled_at(validate_scheduled_at) - end - - def get_by_activity_id(activity_id) do - ActivityExpiration - |> where([exp], exp.activity_id == ^activity_id) - |> Repo.one() - end - - def create(%Activity{} = activity, scheduled_at, validate_scheduled_at \\ true) do - %ActivityExpiration{activity_id: activity.id} - |> changeset(%{scheduled_at: scheduled_at}, validate_scheduled_at) - |> Repo.insert() - end - - def due_expirations(offset \\ 0) do - naive_datetime = - NaiveDateTime.utc_now() - |> NaiveDateTime.add(offset, :millisecond) - - ActivityExpiration - |> where([exp], exp.scheduled_at < ^naive_datetime) - |> limit(50) - |> preload(:activity) - |> Repo.all() - |> Enum.reject(fn %{activity: activity} -> - Activity.pinned_by_actor?(activity) - end) - end - - def validate_scheduled_at(changeset, false), do: changeset - - def validate_scheduled_at(changeset, true) do - validate_change(changeset, :scheduled_at, fn _, scheduled_at -> - if not expires_late_enough?(scheduled_at) do - [scheduled_at: "an ephemeral activity must live for at least one hour"] - else - [] - end - end) - end - - def expires_late_enough?(scheduled_at) do - now = NaiveDateTime.utc_now() - diff = NaiveDateTime.diff(scheduled_at, now, :millisecond) - diff > @min_activity_lifetime - end -end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 333621413..c33848277 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -5,7 +5,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do alias Pleroma.Activity alias Pleroma.Activity.Ir.Topics - alias Pleroma.ActivityExpiration alias Pleroma.Config alias Pleroma.Constants alias Pleroma.Conversation @@ -165,7 +164,11 @@ def notify_and_stream(activity) do end defp maybe_create_activity_expiration({:ok, %{data: %{"expires_at" => expires_at}} = activity}) do - with {:ok, _} <- ActivityExpiration.create(activity, expires_at) do + with {:ok, _job} <- + Pleroma.Workers.PurgeExpiredActivity.enqueue(%{ + activity_id: activity.id, + expires_at: expires_at + }) do {:ok, activity} end end diff --git a/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex b/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex index 7b4c78e0f..bee47b4ed 100644 --- a/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex @@ -31,10 +31,10 @@ defp note?(activity) do defp maybe_add_expiration(activity) do days = Pleroma.Config.get([:mrf_activity_expiration, :days], 365) - expires_at = NaiveDateTime.utc_now() |> Timex.shift(days: days) + expires_at = DateTime.utc_now() |> Timex.shift(days: days) with %{"expires_at" => existing_expires_at} <- activity, - :lt <- NaiveDateTime.compare(existing_expires_at, expires_at) do + :lt <- DateTime.compare(existing_expires_at, expires_at) do activity else _ -> Map.put(activity, "expires_at", expires_at) diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index a5e2323bd..b30ca1bd7 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -7,7 +7,6 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do """ alias Pleroma.Activity alias Pleroma.Activity.Ir.Topics - alias Pleroma.ActivityExpiration alias Pleroma.Chat alias Pleroma.Chat.MessageReference alias Pleroma.FollowingRelationship @@ -189,7 +188,10 @@ def handle(%{data: %{"type" => "Create"}} = activity, meta) do end if expires_at = activity.data["expires_at"] do - ActivityExpiration.create(activity, expires_at) + Pleroma.Workers.PurgeExpiredActivity.enqueue(%{ + activity_id: activity.id, + expires_at: expires_at + }) end BackgroundWorker.enqueue("fetch_data_for_activity", %{"activity_id" => activity.id}) diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex index f849b2e01..548f76609 100644 --- a/lib/pleroma/web/common_api/activity_draft.ex +++ b/lib/pleroma/web/common_api/activity_draft.ex @@ -202,7 +202,7 @@ defp changes(draft) do additional = case draft.expires_at do - %NaiveDateTime{} = expires_at -> Map.put(additional, "expires_at", expires_at) + %DateTime{} = expires_at -> Map.put(additional, "expires_at", expires_at) _ -> additional end diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 4ab533658..500c3883e 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -4,7 +4,6 @@ defmodule Pleroma.Web.CommonAPI do alias Pleroma.Activity - alias Pleroma.ActivityExpiration alias Pleroma.Conversation.Participation alias Pleroma.Formatter alias Pleroma.Object @@ -381,9 +380,9 @@ def get_replied_to_visibility(activity) do def check_expiry_date({:ok, nil} = res), do: res def check_expiry_date({:ok, in_seconds}) do - expiry = NaiveDateTime.utc_now() |> NaiveDateTime.add(in_seconds) + expiry = DateTime.add(DateTime.utc_now(), in_seconds) - if ActivityExpiration.expires_late_enough?(expiry) do + if Pleroma.Workers.PurgeExpiredActivity.expires_late_enough?(expiry) do {:ok, expiry} else {:error, "Expiry date is too soon"} diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 3fe1967be..ca42917fc 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -8,7 +8,6 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do require Pleroma.Constants alias Pleroma.Activity - alias Pleroma.ActivityExpiration alias Pleroma.HTML alias Pleroma.Object alias Pleroma.Repo @@ -245,8 +244,8 @@ def render("show.json", %{activity: %{data: %{"object" => _object}} = activity} expires_at = with true <- client_posted_this_activity, - %ActivityExpiration{scheduled_at: scheduled_at} <- - ActivityExpiration.get_by_activity_id(activity.id) do + %Oban.Job{scheduled_at: scheduled_at} <- + Pleroma.Workers.PurgeExpiredActivity.get_expiration(activity.id) do scheduled_at else _ -> nil diff --git a/lib/pleroma/workers/cron/purge_expired_activities_worker.ex b/lib/pleroma/workers/cron/purge_expired_activities_worker.ex deleted file mode 100644 index 6549207fc..000000000 --- a/lib/pleroma/workers/cron/purge_expired_activities_worker.ex +++ /dev/null @@ -1,48 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker do - @moduledoc """ - The worker to purge expired activities. - """ - - use Oban.Worker, queue: "activity_expiration" - - alias Pleroma.Activity - alias Pleroma.ActivityExpiration - alias Pleroma.Config - alias Pleroma.User - alias Pleroma.Web.CommonAPI - - require Logger - - @interval :timer.minutes(1) - - @impl Oban.Worker - def perform(_job) do - if Config.get([ActivityExpiration, :enabled]) do - Enum.each(ActivityExpiration.due_expirations(@interval), &delete_activity/1) - end - after - :ok - end - - def delete_activity(%ActivityExpiration{activity_id: activity_id}) do - with {:activity, %Activity{} = activity} <- - {:activity, Activity.get_by_id_with_object(activity_id)}, - {:user, %User{} = user} <- {:user, User.get_by_ap_id(activity.object.data["actor"])} do - CommonAPI.delete(activity.id, user) - else - {:activity, _} -> - Logger.error( - "#{__MODULE__} Couldn't delete expired activity: not found activity ##{activity_id}" - ) - - {:user, _} -> - Logger.error( - "#{__MODULE__} Couldn't delete expired activity: not found actor of ##{activity_id}" - ) - end - end -end diff --git a/lib/pleroma/workers/purge_expired_activity.ex b/lib/pleroma/workers/purge_expired_activity.ex new file mode 100644 index 000000000..016b000c1 --- /dev/null +++ b/lib/pleroma/workers/purge_expired_activity.ex @@ -0,0 +1,72 @@ +defmodule Pleroma.Workers.PurgeExpiredActivity do + @moduledoc """ + Worker which purges expired activity. + """ + + use Oban.Worker, queue: :activity_expiration, max_attempts: 1 + + import Ecto.Query + + def enqueue(args) do + with true <- enabled?(), + args when is_map(args) <- validate_expires_at(args) do + {scheduled_at, args} = Map.pop(args, :expires_at) + + args + |> __MODULE__.new(scheduled_at: scheduled_at) + |> Oban.insert() + end + end + + @impl true + def perform(%Oban.Job{args: %{"activity_id" => id}}) do + with %Pleroma.Activity{} = activity <- find_activity(id), + %Pleroma.User{} = user <- find_user(activity.object.data["actor"]) do + Pleroma.Web.CommonAPI.delete(activity.id, user) + end + end + + defp enabled? do + with false <- Pleroma.Config.get([__MODULE__, :enabled], false) do + {:error, :expired_activities_disabled} + end + end + + defp validate_expires_at(%{validate: false} = args), do: Map.delete(args, :validate) + + defp validate_expires_at(args) do + if expires_late_enough?(args[:expires_at]) do + args + else + {:error, :expiration_too_close} + end + end + + defp find_activity(id) do + with nil <- Pleroma.Activity.get_by_id_with_object(id) do + {:error, :activity_not_found} + end + end + + defp find_user(ap_id) do + with nil <- Pleroma.User.get_by_ap_id(ap_id) do + {:error, :user_not_found} + end + end + + def get_expiration(id) do + from(j in Oban.Job, + where: j.state == "scheduled", + where: j.queue == "activity_expiration", + where: fragment("?->>'activity_id' = ?", j.args, ^id) + ) + |> Pleroma.Repo.one() + end + + @spec expires_late_enough?(DateTime.t()) :: boolean() + def expires_late_enough?(scheduled_at) do + now = DateTime.utc_now() + diff = DateTime.diff(scheduled_at, now, :millisecond) + diff > :timer.hours(1) + end +end diff --git a/test/activity_expiration_test.exs b/test/activity_expiration_test.exs deleted file mode 100644 index f86d79826..000000000 --- a/test/activity_expiration_test.exs +++ /dev/null @@ -1,55 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.ActivityExpirationTest do - use Pleroma.DataCase - alias Pleroma.ActivityExpiration - import Pleroma.Factory - - setup do: clear_config([ActivityExpiration, :enabled]) - - test "finds activities due to be deleted only" do - activity = insert(:note_activity) - - expiration_due = - insert(:expiration_in_the_past, %{activity_id: activity.id}) |> Repo.preload(:activity) - - activity2 = insert(:note_activity) - insert(:expiration_in_the_future, %{activity_id: activity2.id}) - - expirations = ActivityExpiration.due_expirations() - - assert length(expirations) == 1 - assert hd(expirations) == expiration_due - end - - test "denies expirations that don't live long enough" do - activity = insert(:note_activity) - now = NaiveDateTime.utc_now() - assert {:error, _} = ActivityExpiration.create(activity, now) - end - - test "deletes an expiration activity" do - Pleroma.Config.put([ActivityExpiration, :enabled], true) - activity = insert(:note_activity) - - naive_datetime = - NaiveDateTime.add( - NaiveDateTime.utc_now(), - -:timer.minutes(2), - :millisecond - ) - - expiration = - insert( - :expiration_in_the_past, - %{activity_id: activity.id, scheduled_at: naive_datetime} - ) - - Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker.perform(%Oban.Job{}) - - refute Pleroma.Repo.get(Pleroma.Activity, activity.id) - refute Pleroma.Repo.get(Pleroma.ActivityExpiration, expiration.id) - end -end diff --git a/test/activity_test.exs b/test/activity_test.exs index 2a92327d1..ee6a99cc3 100644 --- a/test/activity_test.exs +++ b/test/activity_test.exs @@ -185,15 +185,6 @@ test "find all statuses for unauthenticated users when `limit_to_local_content` end end - test "add an activity with an expiration" do - activity = insert(:note_activity) - insert(:expiration_in_the_future, %{activity_id: activity.id}) - - Pleroma.ActivityExpiration - |> where([a], a.activity_id == ^activity.id) - |> Repo.one!() - end - test "all_by_ids_with_object/1" do %{id: id1} = insert(:note_activity) %{id: id2} = insert(:note_activity) diff --git a/test/support/factory.ex b/test/support/factory.ex index 486eda8da..2fdfabbc5 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -200,25 +200,6 @@ def note_activity_factory(attrs \\ %{}) do |> Map.merge(attrs) end - defp expiration_offset_by_minutes(attrs, minutes) do - scheduled_at = - NaiveDateTime.utc_now() - |> NaiveDateTime.add(:timer.minutes(minutes), :millisecond) - |> NaiveDateTime.truncate(:second) - - %Pleroma.ActivityExpiration{} - |> Map.merge(attrs) - |> Map.put(:scheduled_at, scheduled_at) - end - - def expiration_in_the_past_factory(attrs \\ %{}) do - expiration_offset_by_minutes(attrs, -60) - end - - def expiration_in_the_future_factory(attrs \\ %{}) do - expiration_offset_by_minutes(attrs, 61) - end - def article_activity_factory do article = insert(:article) diff --git a/test/tasks/database_test.exs b/test/tasks/database_test.exs index 3a28aa133..292a5ef5f 100644 --- a/test/tasks/database_test.exs +++ b/test/tasks/database_test.exs @@ -3,14 +3,15 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Mix.Tasks.Pleroma.DatabaseTest do + use Pleroma.DataCase + use Oban.Testing, repo: Pleroma.Repo + alias Pleroma.Activity alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.CommonAPI - use Pleroma.DataCase - import Pleroma.Factory setup_all do @@ -130,40 +131,45 @@ test "it turns OrderedCollection likes into empty arrays" do describe "ensure_expiration" do test "it adds to expiration old statuses" do - %{id: activity_id1} = insert(:note_activity) + activity1 = insert(:note_activity) - %{id: activity_id2} = - insert(:note_activity, %{inserted_at: NaiveDateTime.from_iso8601!("2015-01-23 23:50:07")}) + {:ok, inserted_at, 0} = DateTime.from_iso8601("2015-01-23T23:50:07Z") + activity2 = insert(:note_activity, %{inserted_at: inserted_at}) - %{id: activity_id3} = activity3 = insert(:note_activity) + %{id: activity_id3} = insert(:note_activity) - expires_at = - NaiveDateTime.utc_now() - |> NaiveDateTime.add(60 * 61, :second) - |> NaiveDateTime.truncate(:second) + expires_at = DateTime.add(DateTime.utc_now(), 60 * 61) - Pleroma.ActivityExpiration.create(activity3, expires_at) + Pleroma.Workers.PurgeExpiredActivity.enqueue(%{ + activity_id: activity_id3, + expires_at: expires_at + }) Mix.Tasks.Pleroma.Database.run(["ensure_expiration"]) - expirations = - Pleroma.ActivityExpiration - |> order_by(:activity_id) - |> Repo.all() + assert_enqueued( + worker: Pleroma.Workers.PurgeExpiredActivity, + args: %{activity_id: activity1.id}, + scheduled_at: + activity1.inserted_at + |> DateTime.from_naive!("Etc/UTC") + |> Timex.shift(days: 365) + ) - assert [ - %Pleroma.ActivityExpiration{ - activity_id: ^activity_id1 - }, - %Pleroma.ActivityExpiration{ - activity_id: ^activity_id2, - scheduled_at: ~N[2016-01-23 23:50:07] - }, - %Pleroma.ActivityExpiration{ - activity_id: ^activity_id3, - scheduled_at: ^expires_at - } - ] = expirations + assert_enqueued( + worker: Pleroma.Workers.PurgeExpiredActivity, + args: %{activity_id: activity2.id}, + scheduled_at: + activity2.inserted_at + |> DateTime.from_naive!("Etc/UTC") + |> Timex.shift(days: 365) + ) + + assert_enqueued( + worker: Pleroma.Workers.PurgeExpiredActivity, + args: %{activity_id: activity_id3}, + scheduled_at: expires_at + ) end end end diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 03f968aaf..9af573924 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -2069,18 +2069,25 @@ test "it just returns the input if the user has no following/follower addresses" end describe "global activity expiration" do - setup do: clear_config([:mrf, :policies]) - test "creates an activity expiration for local Create activities" do - Pleroma.Config.put( - [:mrf, :policies], - Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy + clear_config([:mrf, :policies], Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy) + + {:ok, activity} = ActivityBuilder.insert(%{"type" => "Create", "context" => "3hu"}) + {:ok, follow} = ActivityBuilder.insert(%{"type" => "Follow", "context" => "3hu"}) + + assert_enqueued( + worker: Pleroma.Workers.PurgeExpiredActivity, + args: %{activity_id: activity.id}, + scheduled_at: + activity.inserted_at + |> DateTime.from_naive!("Etc/UTC") + |> Timex.shift(days: 365) ) - {:ok, %{id: id_create}} = ActivityBuilder.insert(%{"type" => "Create", "context" => "3hu"}) - {:ok, _follow} = ActivityBuilder.insert(%{"type" => "Follow", "context" => "3hu"}) - - assert [%{activity_id: ^id_create}] = Pleroma.ActivityExpiration |> Repo.all() + refute_enqueued( + worker: Pleroma.Workers.PurgeExpiredActivity, + args: %{activity_id: follow.id} + ) end end diff --git a/test/web/activity_pub/mrf/activity_expiration_policy_test.exs b/test/web/activity_pub/mrf/activity_expiration_policy_test.exs index f25cf8b12..e7370d4ef 100644 --- a/test/web/activity_pub/mrf/activity_expiration_policy_test.exs +++ b/test/web/activity_pub/mrf/activity_expiration_policy_test.exs @@ -18,11 +18,11 @@ test "adds `expires_at` property" do "object" => %{"type" => "Note"} }) - assert Timex.diff(expires_at, NaiveDateTime.utc_now(), :days) == 364 + assert Timex.diff(expires_at, DateTime.utc_now(), :days) == 364 end test "keeps existing `expires_at` if it less than the config setting" do - expires_at = NaiveDateTime.utc_now() |> Timex.shift(days: 1) + expires_at = DateTime.utc_now() |> Timex.shift(days: 1) assert {:ok, %{"type" => "Create", "expires_at" => ^expires_at}} = ActivityExpirationPolicy.filter(%{ @@ -35,7 +35,7 @@ test "keeps existing `expires_at` if it less than the config setting" do end test "overwrites existing `expires_at` if it greater than the config setting" do - too_distant_future = NaiveDateTime.utc_now() |> Timex.shift(years: 2) + too_distant_future = DateTime.utc_now() |> Timex.shift(years: 2) assert {:ok, %{"type" => "Create", "expires_at" => expires_at}} = ActivityExpirationPolicy.filter(%{ @@ -46,7 +46,7 @@ test "overwrites existing `expires_at` if it greater than the config setting" do "object" => %{"type" => "Note"} }) - assert Timex.diff(expires_at, NaiveDateTime.utc_now(), :days) == 364 + assert Timex.diff(expires_at, DateTime.utc_now(), :days) == 364 end test "ignores remote activities" do diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs index 800db9a20..5afb0a6dc 100644 --- a/test/web/common_api/common_api_test.exs +++ b/test/web/common_api/common_api_test.exs @@ -4,6 +4,8 @@ defmodule Pleroma.Web.CommonAPITest do use Pleroma.DataCase + use Oban.Testing, repo: Pleroma.Repo + alias Pleroma.Activity alias Pleroma.Chat alias Pleroma.Conversation.Participation @@ -598,15 +600,15 @@ test "it validates character limits are correctly enforced" do test "it can handle activities that expire" do user = insert(:user) - expires_at = - NaiveDateTime.utc_now() - |> NaiveDateTime.truncate(:second) - |> NaiveDateTime.add(1_000_000, :second) + expires_at = DateTime.add(DateTime.utc_now(), 1_000_000) assert {:ok, activity} = CommonAPI.post(user, %{status: "chai", expires_in: 1_000_000}) - assert expiration = Pleroma.ActivityExpiration.get_by_activity_id(activity.id) - assert expiration.scheduled_at == expires_at + assert_enqueued( + worker: Pleroma.Workers.PurgeExpiredActivity, + args: %{activity_id: activity.id}, + scheduled_at: expires_at + ) end end diff --git a/test/web/mastodon_api/controllers/status_controller_test.exs b/test/web/mastodon_api/controllers/status_controller_test.exs index f221884e7..17a156be8 100644 --- a/test/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/web/mastodon_api/controllers/status_controller_test.exs @@ -4,9 +4,9 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do use Pleroma.Web.ConnCase + use Oban.Testing, repo: Pleroma.Repo alias Pleroma.Activity - alias Pleroma.ActivityExpiration alias Pleroma.Config alias Pleroma.Conversation.Participation alias Pleroma.Object @@ -29,8 +29,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusControllerTest do setup do: oauth_access(["write:statuses"]) test "posting a status does not increment reblog_count when relaying", %{conn: conn} do - Pleroma.Config.put([:instance, :federating], true) - Pleroma.Config.get([:instance, :allow_relay], true) + Config.put([:instance, :federating], true) + Config.get([:instance, :allow_relay], true) response = conn @@ -103,7 +103,9 @@ test "posting a status", %{conn: conn} do # An activity that will expire: # 2 hours - expires_in = 120 * 60 + expires_in = 2 * 60 * 60 + + expires_at = DateTime.add(DateTime.utc_now(), expires_in) conn_four = conn @@ -116,19 +118,13 @@ test "posting a status", %{conn: conn} do assert fourth_response = %{"id" => fourth_id} = json_response_and_validate_schema(conn_four, 200) - assert activity = Activity.get_by_id(fourth_id) - assert expiration = ActivityExpiration.get_by_activity_id(fourth_id) + assert Activity.get_by_id(fourth_id) - estimated_expires_at = - NaiveDateTime.utc_now() - |> NaiveDateTime.add(expires_in) - |> NaiveDateTime.truncate(:second) - - # This assert will fail if the test takes longer than a minute. I sure hope it never does: - assert abs(NaiveDateTime.diff(expiration.scheduled_at, estimated_expires_at, :second)) < 60 - - assert fourth_response["pleroma"]["expires_at"] == - NaiveDateTime.to_iso8601(expiration.scheduled_at) + assert_enqueued( + worker: Pleroma.Workers.PurgeExpiredActivity, + args: %{activity_id: fourth_id}, + scheduled_at: expires_at + ) end test "it fails to create a status if `expires_in` is less or equal than an hour", %{ @@ -160,8 +156,8 @@ test "it fails to create a status if `expires_in` is less or equal than an hour" end test "Get MRF reason when posting a status is rejected by one", %{conn: conn} do - Pleroma.Config.put([:mrf_keyword, :reject], ["GNO"]) - Pleroma.Config.put([:mrf, :policies], [Pleroma.Web.ActivityPub.MRF.KeywordPolicy]) + Config.put([:mrf_keyword, :reject], ["GNO"]) + Config.put([:mrf, :policies], [Pleroma.Web.ActivityPub.MRF.KeywordPolicy]) assert %{"error" => "[KeywordPolicy] Matches with rejected keyword"} = conn @@ -1681,19 +1677,17 @@ test "returns the favorites of a user" do test "expires_at is nil for another user" do %{conn: conn, user: user} = oauth_access(["read:statuses"]) + expires_at = DateTime.add(DateTime.utc_now(), 1_000_000) {:ok, activity} = CommonAPI.post(user, %{status: "foobar", expires_in: 1_000_000}) - expires_at = - activity.id - |> ActivityExpiration.get_by_activity_id() - |> Map.get(:scheduled_at) - |> NaiveDateTime.to_iso8601() - - assert %{"pleroma" => %{"expires_at" => ^expires_at}} = + assert %{"pleroma" => %{"expires_at" => a_expires_at}} = conn |> get("/api/v1/statuses/#{activity.id}") |> json_response_and_validate_schema(:ok) + {:ok, a_expires_at, 0} = DateTime.from_iso8601(a_expires_at) + assert DateTime.diff(expires_at, a_expires_at) == 0 + %{conn: conn} = oauth_access(["read:statuses"]) assert %{"pleroma" => %{"expires_at" => nil}} = diff --git a/test/workers/cron/purge_expired_activities_worker_test.exs b/test/workers/cron/purge_expired_activities_worker_test.exs deleted file mode 100644 index d1acd9ae6..000000000 --- a/test/workers/cron/purge_expired_activities_worker_test.exs +++ /dev/null @@ -1,84 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Workers.Cron.PurgeExpiredActivitiesWorkerTest do - use Pleroma.DataCase - - alias Pleroma.ActivityExpiration - alias Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker - - import Pleroma.Factory - import ExUnit.CaptureLog - - setup do - clear_config([ActivityExpiration, :enabled]) - end - - test "deletes an expiration activity" do - Pleroma.Config.put([ActivityExpiration, :enabled], true) - activity = insert(:note_activity) - - naive_datetime = - NaiveDateTime.add( - NaiveDateTime.utc_now(), - -:timer.minutes(2), - :millisecond - ) - - expiration = - insert( - :expiration_in_the_past, - %{activity_id: activity.id, scheduled_at: naive_datetime} - ) - - Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker.perform(%Oban.Job{}) - - refute Pleroma.Repo.get(Pleroma.Activity, activity.id) - refute Pleroma.Repo.get(Pleroma.ActivityExpiration, expiration.id) - end - - test "works with ActivityExpirationPolicy" do - Pleroma.Config.put([ActivityExpiration, :enabled], true) - - clear_config([:mrf, :policies], Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy) - - user = insert(:user) - - days = Pleroma.Config.get([:mrf_activity_expiration, :days], 365) - - {:ok, %{id: id} = activity} = Pleroma.Web.CommonAPI.post(user, %{status: "cofe"}) - - past_date = - NaiveDateTime.utc_now() |> Timex.shift(days: -days) |> NaiveDateTime.truncate(:second) - - activity - |> Repo.preload(:expiration) - |> Map.get(:expiration) - |> Ecto.Changeset.change(%{scheduled_at: past_date}) - |> Repo.update!() - - Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker.perform(%Oban.Job{}) - - assert [%{data: %{"type" => "Delete", "deleted_activity_id" => ^id}}] = - Pleroma.Repo.all(Pleroma.Activity) - end - - describe "delete_activity/1" do - test "adds log message if activity isn't find" do - assert capture_log([level: :error], fn -> - PurgeExpiredActivitiesWorker.delete_activity(%ActivityExpiration{ - activity_id: "test-activity" - }) - end) =~ "Couldn't delete expired activity: not found activity" - end - - test "adds log message if actor isn't find" do - assert capture_log([level: :error], fn -> - PurgeExpiredActivitiesWorker.delete_activity(%ActivityExpiration{ - activity_id: "test-activity" - }) - end) =~ "Couldn't delete expired activity: not found activity" - end - end -end diff --git a/test/workers/purge_expired_activity_test.exs b/test/workers/purge_expired_activity_test.exs new file mode 100644 index 000000000..8b5dc9fd2 --- /dev/null +++ b/test/workers/purge_expired_activity_test.exs @@ -0,0 +1,47 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.PurgeExpiredActivityTest do + use Pleroma.DataCase, async: true + use Oban.Testing, repo: Pleroma.Repo + + import Pleroma.Factory + + alias Pleroma.Workers.PurgeExpiredActivity + + test "denies expirations that don't live long enough" do + activity = insert(:note_activity) + + assert {:error, :expiration_too_close} = + PurgeExpiredActivity.enqueue(%{ + activity_id: activity.id, + expires_at: DateTime.utc_now() + }) + + refute_enqueued( + worker: Pleroma.Workers.PurgeExpiredActivity, + args: %{activity_id: activity.id} + ) + end + + test "enqueue job" do + activity = insert(:note_activity) + + assert {:ok, _} = + PurgeExpiredActivity.enqueue(%{ + activity_id: activity.id, + expires_at: DateTime.add(DateTime.utc_now(), 3601) + }) + + assert_enqueued( + worker: Pleroma.Workers.PurgeExpiredActivity, + args: %{activity_id: activity.id} + ) + + assert {:ok, _} = + perform_job(Pleroma.Workers.PurgeExpiredActivity, %{activity_id: activity.id}) + + assert %Oban.Job{} = Pleroma.Workers.PurgeExpiredActivity.get_expiration(activity.id) + end +end From de4c935071a47c78d873484b202e09dce5399570 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Mon, 24 Aug 2020 13:43:02 +0300 Subject: [PATCH 109/128] don't expire pinned posts --- lib/pleroma/activity.ex | 9 ++++++-- lib/pleroma/workers/purge_expired_activity.ex | 18 +++++++++++++++- test/workers/purge_expired_activity_test.exs | 21 +++++++++++++++++++ 3 files changed, 45 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 03cd3b8c0..84aba9572 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -301,14 +301,14 @@ def all_by_actor_and_id(actor, status_ids) do |> Repo.all() end - def follow_requests_for_actor(%Pleroma.User{ap_id: ap_id}) do + def follow_requests_for_actor(%User{ap_id: ap_id}) do ap_id |> Queries.by_object_id() |> Queries.by_type("Follow") |> where([a], fragment("? ->> 'state' = 'pending'", a.data)) end - def following_requests_for_actor(%Pleroma.User{ap_id: ap_id}) do + def following_requests_for_actor(%User{ap_id: ap_id}) do Queries.by_type("Follow") |> where([a], fragment("?->>'state' = 'pending'", a.data)) |> where([a], a.actor == ^ap_id) @@ -343,4 +343,9 @@ def pinned_by_actor?(%Activity{} = activity) do actor = user_actor(activity) activity.id in actor.pinned_activities end + + @spec pinned_by_actor?(Activity.t(), User.t()) :: boolean() + def pinned_by_actor?(%Activity{id: id}, %User{} = user) do + id in user.pinned_activities + end end diff --git a/lib/pleroma/workers/purge_expired_activity.ex b/lib/pleroma/workers/purge_expired_activity.ex index 016b000c1..ba0053008 100644 --- a/lib/pleroma/workers/purge_expired_activity.ex +++ b/lib/pleroma/workers/purge_expired_activity.ex @@ -21,8 +21,18 @@ def enqueue(args) do @impl true def perform(%Oban.Job{args: %{"activity_id" => id}}) do with %Pleroma.Activity{} = activity <- find_activity(id), - %Pleroma.User{} = user <- find_user(activity.object.data["actor"]) do + %Pleroma.User{} = user <- find_user(activity.object.data["actor"]), + false <- pinned_by_actor?(activity, user) do Pleroma.Web.CommonAPI.delete(activity.id, user) + else + :pinned_by_actor -> + # if activity is pinned, schedule deletion on next day + enqueue(%{activity_id: id, expires_at: DateTime.add(DateTime.utc_now(), 24 * 3600)}) + + :ok + + error -> + error end end @@ -54,6 +64,12 @@ defp find_user(ap_id) do end end + defp pinned_by_actor?(activity, user) do + with true <- Pleroma.Activity.pinned_by_actor?(activity, user) do + :pinned_by_actor + end + end + def get_expiration(id) do from(j in Oban.Job, where: j.state == "scheduled", diff --git a/test/workers/purge_expired_activity_test.exs b/test/workers/purge_expired_activity_test.exs index 8b5dc9fd2..736d7d567 100644 --- a/test/workers/purge_expired_activity_test.exs +++ b/test/workers/purge_expired_activity_test.exs @@ -44,4 +44,25 @@ test "enqueue job" do assert %Oban.Job{} = Pleroma.Workers.PurgeExpiredActivity.get_expiration(activity.id) end + + test "don't delete pinned posts, schedule deletion on next day" do + activity = insert(:note_activity) + + assert {:ok, _} = + PurgeExpiredActivity.enqueue(%{ + activity_id: activity.id, + expires_at: DateTime.utc_now(), + validate: false + }) + + user = Pleroma.User.get_by_ap_id(activity.actor) + {:ok, activity} = Pleroma.Web.CommonAPI.pin(activity.id, user) + + assert %{success: 1, failure: 0} == + Oban.drain_queue(queue: :activity_expiration, with_scheduled: true) + + job = Pleroma.Workers.PurgeExpiredActivity.get_expiration(activity.id) + + assert DateTime.diff(job.scheduled_at, DateTime.add(DateTime.utc_now(), 24 * 3600)) in [0, 1] + end end From 629a8de9cb2ba2cc2d09679862a24031f34abc2f Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 25 Aug 2020 09:10:45 +0300 Subject: [PATCH 110/128] deprecation warning changed namespace for activity expiration configuration --- config/description.exs | 6 +++--- lib/pleroma/config/deprecation_warnings.ex | 19 ++++++++++++++++++- lib/pleroma/workers/purge_expired_activity.ex | 8 +++++--- ...541_rename_activity_expiration_setting.exs | 13 +++++++++++++ 4 files changed, 39 insertions(+), 7 deletions(-) create mode 100644 priv/repo/migrations/20200824115541_rename_activity_expiration_setting.exs diff --git a/config/description.exs b/config/description.exs index 6ce27278c..1253944de 100644 --- a/config/description.exs +++ b/config/description.exs @@ -2472,14 +2472,14 @@ }, %{ group: :pleroma, - key: Pleroma.ActivityExpiration, + key: Pleroma.Workers.PurgeExpiredActivity, type: :group, - description: "Expired activity settings", + description: "Expired activities settings", children: [ %{ key: :enabled, type: :boolean, - description: "Whether expired activities will be sent to the job queue to be deleted" + description: "Enables expired activities addition & deletion" } ] }, diff --git a/lib/pleroma/config/deprecation_warnings.ex b/lib/pleroma/config/deprecation_warnings.ex index 2bfe4ddba..412d55a77 100644 --- a/lib/pleroma/config/deprecation_warnings.ex +++ b/lib/pleroma/config/deprecation_warnings.ex @@ -8,7 +8,7 @@ defmodule Pleroma.Config.DeprecationWarnings do require Logger alias Pleroma.Config - @type config_namespace() :: [atom()] + @type config_namespace() :: atom() | [atom()] @type config_map() :: {config_namespace(), config_namespace(), String.t()} @mrf_config_map [ @@ -57,6 +57,7 @@ def warn do check_media_proxy_whitelist_config() check_welcome_message_config() check_gun_pool_options() + check_activity_expiration_config() end def check_welcome_message_config do @@ -158,4 +159,20 @@ def check_gun_pool_options do Config.put(:pools, updated_config) end end + + @spec check_activity_expiration_config() :: :ok | nil + def check_activity_expiration_config do + warning_preface = """ + !!!DEPRECATION WARNING!!! + Your config is using old namespace for activity expiration configuration. Setting should work for now, but you are advised to change to new namespace to prevent possible issues later: + """ + + move_namespace_and_warn( + [ + {Pleroma.ActivityExpiration, Pleroma.Workers.PurgeExpiredActivity, + "\n* `config :pleroma, Pleroma.ActivityExpiration` is now `config :pleroma, Pleroma.Workers.PurgeExpiredActivity`"} + ], + warning_preface + ) + end end diff --git a/lib/pleroma/workers/purge_expired_activity.ex b/lib/pleroma/workers/purge_expired_activity.ex index ba0053008..44a8ad0b9 100644 --- a/lib/pleroma/workers/purge_expired_activity.ex +++ b/lib/pleroma/workers/purge_expired_activity.ex @@ -7,6 +7,8 @@ defmodule Pleroma.Workers.PurgeExpiredActivity do import Ecto.Query + alias Pleroma.Activity + def enqueue(args) do with true <- enabled?(), args when is_map(args) <- validate_expires_at(args) do @@ -20,7 +22,7 @@ def enqueue(args) do @impl true def perform(%Oban.Job{args: %{"activity_id" => id}}) do - with %Pleroma.Activity{} = activity <- find_activity(id), + with %Activity{} = activity <- find_activity(id), %Pleroma.User{} = user <- find_user(activity.object.data["actor"]), false <- pinned_by_actor?(activity, user) do Pleroma.Web.CommonAPI.delete(activity.id, user) @@ -53,7 +55,7 @@ defp validate_expires_at(args) do end defp find_activity(id) do - with nil <- Pleroma.Activity.get_by_id_with_object(id) do + with nil <- Activity.get_by_id_with_object(id) do {:error, :activity_not_found} end end @@ -65,7 +67,7 @@ defp find_user(ap_id) do end defp pinned_by_actor?(activity, user) do - with true <- Pleroma.Activity.pinned_by_actor?(activity, user) do + with true <- Activity.pinned_by_actor?(activity, user) do :pinned_by_actor end end diff --git a/priv/repo/migrations/20200824115541_rename_activity_expiration_setting.exs b/priv/repo/migrations/20200824115541_rename_activity_expiration_setting.exs new file mode 100644 index 000000000..241882ef6 --- /dev/null +++ b/priv/repo/migrations/20200824115541_rename_activity_expiration_setting.exs @@ -0,0 +1,13 @@ +defmodule Pleroma.Repo.Migrations.RenameActivityExpirationSetting do + use Ecto.Migration + + def change do + config = Pleroma.ConfigDB.get_by_params(%{group: :pleroma, key: Pleroma.ActivityExpiration}) + + if config do + config + |> Ecto.Changeset.change(key: Pleroma.Workers.PurgeExpiredActivity) + |> Pleroma.Repo.update() + end + end +end From 5ad0cc4c863f7f8a1e6fdfa40eb884a5c94ebf67 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 25 Aug 2020 12:30:00 +0300 Subject: [PATCH 111/128] move old expirations into Oban --- ...1316_move_activity_expirations_to_oban.exs | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 priv/repo/migrations/20200825061316_move_activity_expirations_to_oban.exs diff --git a/priv/repo/migrations/20200825061316_move_activity_expirations_to_oban.exs b/priv/repo/migrations/20200825061316_move_activity_expirations_to_oban.exs new file mode 100644 index 000000000..585d1a600 --- /dev/null +++ b/priv/repo/migrations/20200825061316_move_activity_expirations_to_oban.exs @@ -0,0 +1,29 @@ +defmodule Pleroma.Repo.Migrations.MoveActivityExpirationsToOban do + use Ecto.Migration + + import Ecto.Query, only: [from: 2] + + def change do + Supervisor.start_link([{Oban, Pleroma.Config.get(Oban)}], + strategy: :one_for_one, + name: Pleroma.Supervisor + ) + + from(e in "activity_expirations", + select: %{id: e.id, activity_id: e.activity_id, scheduled_at: e.scheduled_at} + ) + |> Pleroma.RepoStreamer.chunk_stream(500) + |> Stream.each(fn expirations -> + Enum.each(expirations, fn expiration -> + with {:ok, expires_at} <- DateTime.from_naive(expiration.scheduled_at, "Etc/UTC") do + Pleroma.Workers.PurgeExpiredActivity.enqueue(%{ + activity_id: FlakeId.to_string(expiration.activity_id), + expires_at: expires_at, + validate: false + }) + end + end) + end) + |> Stream.run() + end +end From 5381d4b78b6ed550102008cbae7f578dab06f22f Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 25 Aug 2020 12:33:38 +0300 Subject: [PATCH 112/128] drop activity_expirations table --- .../20200825093037_drop_activity_expirations_table.exs | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 priv/repo/migrations/20200825093037_drop_activity_expirations_table.exs diff --git a/priv/repo/migrations/20200825093037_drop_activity_expirations_table.exs b/priv/repo/migrations/20200825093037_drop_activity_expirations_table.exs new file mode 100644 index 000000000..11c461427 --- /dev/null +++ b/priv/repo/migrations/20200825093037_drop_activity_expirations_table.exs @@ -0,0 +1,7 @@ +defmodule Pleroma.Repo.Migrations.DropActivityExpirationsTable do + use Ecto.Migration + + def change do + drop(table("activity_expirations")) + end +end From 4981b5a1a3c097ca849552c3c6f650efd22c7451 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 25 Aug 2020 12:45:06 +0300 Subject: [PATCH 113/128] copyright header --- lib/pleroma/workers/purge_expired_activity.ex | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/pleroma/workers/purge_expired_activity.ex b/lib/pleroma/workers/purge_expired_activity.ex index 44a8ad0b9..42e2ae79c 100644 --- a/lib/pleroma/workers/purge_expired_activity.ex +++ b/lib/pleroma/workers/purge_expired_activity.ex @@ -1,3 +1,7 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + defmodule Pleroma.Workers.PurgeExpiredActivity do @moduledoc """ Worker which purges expired activity. From 93e1c8df9dca697e7bdb822a8a5b3848b7870f53 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 3 Sep 2020 13:30:39 +0300 Subject: [PATCH 114/128] reject activity creation if passed expires_at option and expiring activities are not configured --- lib/pleroma/web/activity_pub/activity_pub.ex | 44 +++++++++----- lib/pleroma/workers/purge_expired_activity.ex | 4 ++ test/web/activity_pub/activity_pub_test.exs | 57 +++++++++++++++---- 3 files changed, 80 insertions(+), 25 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index c33848277..ee6dcf58a 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -110,23 +110,14 @@ def persist(object, meta) do def insert(map, local \\ true, fake \\ false, bypass_actor_check \\ false) when is_map(map) do with nil <- Activity.normalize(map), map <- lazy_put_activity_defaults(map, fake), - true <- bypass_actor_check || check_actor_is_active(map["actor"]), - {_, true} <- {:remote_limit_error, check_remote_limit(map)}, + {_, true} <- {:actor_check, bypass_actor_check || check_actor_is_active(map["actor"])}, + {_, true} <- {:remote_limit_pass, check_remote_limit(map)}, {:ok, map} <- MRF.filter(map), {recipients, _, _} = get_recipients(map), {:fake, false, map, recipients} <- {:fake, fake, map, recipients}, {:containment, :ok} <- {:containment, Containment.contain_child(map)}, - {:ok, map, object} <- insert_full_object(map) do - {:ok, activity} = - %Activity{ - data: map, - local: local, - actor: map["actor"], - recipients: recipients - } - |> Repo.insert() - |> maybe_create_activity_expiration() - + {:ok, map, object} <- insert_full_object(map), + {:ok, activity} <- insert_activity_with_expiration(map, local, recipients) do # Splice in the child object if we have one. activity = Maps.put_if_present(activity, :object, object) @@ -137,6 +128,15 @@ def insert(map, local \\ true, fake \\ false, bypass_actor_check \\ false) when %Activity{} = activity -> {:ok, activity} + {:actor_check, _} -> + {:error, false} + + {:containment, _} = error -> + error + + {:error, _} = error -> + error + {:fake, true, map, recipients} -> activity = %Activity{ data: map, @@ -149,11 +149,25 @@ def insert(map, local \\ true, fake \\ false, bypass_actor_check \\ false) when Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) {:ok, activity} - error -> - {:error, error} + {:remote_limit_pass, _} -> + {:error, :remote_limit} + + {:reject, reason} -> + {:error, reason} end end + defp insert_activity_with_expiration(data, local, recipients) do + %Activity{ + data: data, + local: local, + actor: data["actor"], + recipients: recipients + } + |> Repo.insert() + |> maybe_create_activity_expiration() + end + def notify_and_stream(activity) do Notification.create_notifications(activity) diff --git a/lib/pleroma/workers/purge_expired_activity.ex b/lib/pleroma/workers/purge_expired_activity.ex index 42e2ae79c..c70587b47 100644 --- a/lib/pleroma/workers/purge_expired_activity.ex +++ b/lib/pleroma/workers/purge_expired_activity.ex @@ -13,6 +13,10 @@ defmodule Pleroma.Workers.PurgeExpiredActivity do alias Pleroma.Activity + @spec enqueue(map()) :: + {:ok, Oban.Job.t()} + | {:error, :expired_activities_disabled} + | {:error, :expiration_too_close} def enqueue(args) do with true <- enabled?(), args when is_map(args) <- validate_expires_at(args) do diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 9af573924..d8caa0b00 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -239,7 +239,7 @@ test "drops activities beyond a certain limit" do } } - assert {:error, {:remote_limit_error, _}} = ActivityPub.insert(data) + assert {:error, :remote_limit} = ActivityPub.insert(data) end test "doesn't drop activities with content being null" do @@ -386,9 +386,11 @@ test "can be fetched into a timeline" do end describe "create activities" do - test "it reverts create" do - user = insert(:user) + setup do + [user: insert(:user)] + end + test "it reverts create", %{user: user} do with_mock(Utils, [:passthrough], maybe_federate: fn _ -> {:error, :reverted} end) do assert {:error, :reverted} = ActivityPub.create(%{ @@ -407,9 +409,47 @@ test "it reverts create" do assert Repo.aggregate(Object, :count, :id) == 0 end - test "removes doubled 'to' recipients" do - user = insert(:user) + test "creates activity if expiration is not configured and expires_at is not passed", %{ + user: user + } do + clear_config([Pleroma.Workers.PurgeExpiredActivity, :enabled], false) + assert {:ok, _} = + ActivityPub.create(%{ + to: ["user1", "user2"], + actor: user, + context: "", + object: %{ + "to" => ["user1", "user2"], + "type" => "Note", + "content" => "testing" + } + }) + end + + test "rejects activity if expires_at present but expiration is not configured", %{user: user} do + clear_config([Pleroma.Workers.PurgeExpiredActivity, :enabled], false) + + assert {:error, :expired_activities_disabled} = + ActivityPub.create(%{ + to: ["user1", "user2"], + actor: user, + context: "", + object: %{ + "to" => ["user1", "user2"], + "type" => "Note", + "content" => "testing" + }, + additional: %{ + "expires_at" => DateTime.utc_now() + } + }) + + assert Repo.aggregate(Activity, :count, :id) == 0 + assert Repo.aggregate(Object, :count, :id) == 0 + end + + test "removes doubled 'to' recipients", %{user: user} do {:ok, activity} = ActivityPub.create(%{ to: ["user1", "user1", "user2"], @@ -427,9 +467,7 @@ test "removes doubled 'to' recipients" do assert activity.recipients == ["user1", "user2", user.ap_id] end - test "increases user note count only for public activities" do - user = insert(:user) - + test "increases user note count only for public activities", %{user: user} do {:ok, _} = CommonAPI.post(User.get_cached_by_id(user.id), %{ status: "1", @@ -458,8 +496,7 @@ test "increases user note count only for public activities" do assert user.note_count == 2 end - test "increases replies count" do - user = insert(:user) + test "increases replies count", %{user: user} do user2 = insert(:user) {:ok, activity} = CommonAPI.post(user, %{status: "1", visibility: "public"}) From 357d971a10c28780795af4d19b37b0ac80d6ad09 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 3 Sep 2020 17:56:20 +0300 Subject: [PATCH 115/128] expiration for new pipeline --- lib/pleroma/web/activity_pub/activity_pub.ex | 18 ++++++++++++------ lib/pleroma/web/activity_pub/side_effects.ex | 7 ------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index ee6dcf58a..66a9f78a3 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -101,7 +101,9 @@ def persist(object, meta) do local: local, recipients: recipients, actor: object["actor"] - }) do + }), + # TODO: add tests for expired activities, when Note type will be supported in new pipeline + {:ok, _} <- maybe_create_activity_expiration(activity) do {:ok, activity, meta} end end @@ -158,14 +160,16 @@ def insert(map, local \\ true, fake \\ false, bypass_actor_check \\ false) when end defp insert_activity_with_expiration(data, local, recipients) do - %Activity{ + struct = %Activity{ data: data, local: local, actor: data["actor"], recipients: recipients } - |> Repo.insert() - |> maybe_create_activity_expiration() + + with {:ok, activity} <- Repo.insert(struct) do + maybe_create_activity_expiration(activity) + end end def notify_and_stream(activity) do @@ -177,7 +181,9 @@ def notify_and_stream(activity) do stream_out_participations(participations) end - defp maybe_create_activity_expiration({:ok, %{data: %{"expires_at" => expires_at}} = activity}) do + defp maybe_create_activity_expiration( + %{data: %{"expires_at" => %DateTime{} = expires_at}} = activity + ) do with {:ok, _job} <- Pleroma.Workers.PurgeExpiredActivity.enqueue(%{ activity_id: activity.id, @@ -187,7 +193,7 @@ defp maybe_create_activity_expiration({:ok, %{data: %{"expires_at" => expires_at end end - defp maybe_create_activity_expiration(result), do: result + defp maybe_create_activity_expiration(activity), do: {:ok, activity} defp create_or_bump_conversation(activity, actor) do with {:ok, conversation} <- Conversation.create_or_bump_for(activity), diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index b30ca1bd7..46a8be767 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -187,13 +187,6 @@ def handle(%{data: %{"type" => "Create"}} = activity, meta) do Object.increase_replies_count(in_reply_to) end - if expires_at = activity.data["expires_at"] do - Pleroma.Workers.PurgeExpiredActivity.enqueue(%{ - activity_id: activity.id, - expires_at: expires_at - }) - end - BackgroundWorker.enqueue("fetch_data_for_activity", %{"activity_id" => activity.id}) meta = From 6f2d1145183389c415e4d5a915e0c3965c00a3fb Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 3 Sep 2020 18:08:19 +0300 Subject: [PATCH 116/128] use another stream function in migration --- ...1316_move_activity_expirations_to_oban.exs | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/priv/repo/migrations/20200825061316_move_activity_expirations_to_oban.exs b/priv/repo/migrations/20200825061316_move_activity_expirations_to_oban.exs index 585d1a600..2bfefceb0 100644 --- a/priv/repo/migrations/20200825061316_move_activity_expirations_to_oban.exs +++ b/priv/repo/migrations/20200825061316_move_activity_expirations_to_oban.exs @@ -12,17 +12,15 @@ def change do from(e in "activity_expirations", select: %{id: e.id, activity_id: e.activity_id, scheduled_at: e.scheduled_at} ) - |> Pleroma.RepoStreamer.chunk_stream(500) - |> Stream.each(fn expirations -> - Enum.each(expirations, fn expiration -> - with {:ok, expires_at} <- DateTime.from_naive(expiration.scheduled_at, "Etc/UTC") do - Pleroma.Workers.PurgeExpiredActivity.enqueue(%{ - activity_id: FlakeId.to_string(expiration.activity_id), - expires_at: expires_at, - validate: false - }) - end - end) + |> Pleroma.Repo.stream() + |> Enum.each(fn expiration -> + with {:ok, expires_at} <- DateTime.from_naive(expiration.scheduled_at, "Etc/UTC") do + Pleroma.Workers.PurgeExpiredActivity.enqueue(%{ + activity_id: FlakeId.to_string(expiration.activity_id), + expires_at: expires_at, + validate: false + }) + end end) |> Stream.run() end From b3485a6dbfb1a16dd5604294074ef5139fbf3ce9 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 3 Sep 2020 19:02:22 +0300 Subject: [PATCH 117/128] little clean up --- lib/pleroma/workers/purge_expired_activity.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/workers/purge_expired_activity.ex b/lib/pleroma/workers/purge_expired_activity.ex index c70587b47..4be146194 100644 --- a/lib/pleroma/workers/purge_expired_activity.ex +++ b/lib/pleroma/workers/purge_expired_activity.ex @@ -23,7 +23,7 @@ def enqueue(args) do {scheduled_at, args} = Map.pop(args, :expires_at) args - |> __MODULE__.new(scheduled_at: scheduled_at) + |> new(scheduled_at: scheduled_at) |> Oban.insert() end end From eb5ff715f7917e174b9ae104a5d82779ff925301 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 4 Sep 2020 11:40:32 +0300 Subject: [PATCH 118/128] pin/unpin for activities with expires_at option --- lib/pleroma/activity.ex | 5 -- lib/pleroma/user.ex | 17 ++++++- lib/pleroma/workers/purge_expired_activity.ex | 18 +------ .../controllers/status_controller_test.exs | 49 ++++++++++++++++++- test/workers/purge_expired_activity_test.exs | 21 -------- 5 files changed, 64 insertions(+), 46 deletions(-) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 84aba9572..17af04257 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -343,9 +343,4 @@ def pinned_by_actor?(%Activity{} = activity) do actor = user_actor(activity) activity.id in actor.pinned_activities end - - @spec pinned_by_actor?(Activity.t(), User.t()) :: boolean() - def pinned_by_actor?(%Activity{id: id}, %User{} = user) do - id in user.pinned_activities - end end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index f323fc6ed..e73d19964 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -2315,6 +2315,11 @@ def add_pinnned_activity(user, %Pleroma.Activity{id: id}) do max_pinned_statuses = Config.get([:instance, :max_pinned_statuses], 0) params = %{pinned_activities: user.pinned_activities ++ [id]} + # if pinned activity was scheduled for deletion, we remove job + if expiration = Pleroma.Workers.PurgeExpiredActivity.get_expiration(id) do + Oban.cancel_job(expiration.id) + end + user |> cast(params, [:pinned_activities]) |> validate_length(:pinned_activities, @@ -2327,9 +2332,19 @@ def add_pinnned_activity(user, %Pleroma.Activity{id: id}) do |> update_and_set_cache() end - def remove_pinnned_activity(user, %Pleroma.Activity{id: id}) do + def remove_pinnned_activity(user, %Pleroma.Activity{id: id, data: data}) do params = %{pinned_activities: List.delete(user.pinned_activities, id)} + # if pinned activity was scheduled for deletion, we reschedule it for deletion + if data["expires_at"] do + {:ok, expires_at, _} = DateTime.from_iso8601(data["expires_at"]) + + Pleroma.Workers.PurgeExpiredActivity.enqueue(%{ + activity_id: id, + expires_at: expires_at + }) + end + user |> cast(params, [:pinned_activities]) |> update_and_set_cache() diff --git a/lib/pleroma/workers/purge_expired_activity.ex b/lib/pleroma/workers/purge_expired_activity.ex index 4be146194..f981eda8e 100644 --- a/lib/pleroma/workers/purge_expired_activity.ex +++ b/lib/pleroma/workers/purge_expired_activity.ex @@ -31,18 +31,8 @@ def enqueue(args) do @impl true def perform(%Oban.Job{args: %{"activity_id" => id}}) do with %Activity{} = activity <- find_activity(id), - %Pleroma.User{} = user <- find_user(activity.object.data["actor"]), - false <- pinned_by_actor?(activity, user) do + %Pleroma.User{} = user <- find_user(activity.object.data["actor"]) do Pleroma.Web.CommonAPI.delete(activity.id, user) - else - :pinned_by_actor -> - # if activity is pinned, schedule deletion on next day - enqueue(%{activity_id: id, expires_at: DateTime.add(DateTime.utc_now(), 24 * 3600)}) - - :ok - - error -> - error end end @@ -74,12 +64,6 @@ defp find_user(ap_id) do end end - defp pinned_by_actor?(activity, user) do - with true <- Activity.pinned_by_actor?(activity, user) do - :pinned_by_actor - end - end - def get_expiration(id) do from(j in Oban.Job, where: j.state == "scheduled", diff --git a/test/web/mastodon_api/controllers/status_controller_test.exs b/test/web/mastodon_api/controllers/status_controller_test.exs index 17a156be8..82ea73898 100644 --- a/test/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/web/mastodon_api/controllers/status_controller_test.exs @@ -115,8 +115,7 @@ test "posting a status", %{conn: conn} do "expires_in" => expires_in }) - assert fourth_response = - %{"id" => fourth_id} = json_response_and_validate_schema(conn_four, 200) + assert %{"id" => fourth_id} = json_response_and_validate_schema(conn_four, 200) assert Activity.get_by_id(fourth_id) @@ -1142,6 +1141,52 @@ test "max pinned statuses", %{conn: conn, user: user, activity: activity_one} do |> post("/api/v1/statuses/#{activity_two.id}/pin") |> json_response_and_validate_schema(400) end + + test "on pin removes deletion job, on unpin reschedule deletion" do + %{conn: conn} = oauth_access(["write:accounts", "write:statuses"]) + expires_in = 2 * 60 * 60 + + expires_at = DateTime.add(DateTime.utc_now(), expires_in) + + assert %{"id" => id} = + conn + |> put_req_header("content-type", "application/json") + |> post("api/v1/statuses", %{ + "status" => "oolong", + "expires_in" => expires_in + }) + |> json_response_and_validate_schema(200) + + assert_enqueued( + worker: Pleroma.Workers.PurgeExpiredActivity, + args: %{activity_id: id}, + scheduled_at: expires_at + ) + + assert %{"id" => ^id, "pinned" => true} = + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/statuses/#{id}/pin") + |> json_response_and_validate_schema(200) + + refute_enqueued( + worker: Pleroma.Workers.PurgeExpiredActivity, + args: %{activity_id: id}, + scheduled_at: expires_at + ) + + assert %{"id" => ^id, "pinned" => false} = + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/statuses/#{id}/unpin") + |> json_response_and_validate_schema(200) + + assert_enqueued( + worker: Pleroma.Workers.PurgeExpiredActivity, + args: %{activity_id: id}, + scheduled_at: expires_at + ) + end end describe "cards" do diff --git a/test/workers/purge_expired_activity_test.exs b/test/workers/purge_expired_activity_test.exs index 736d7d567..8b5dc9fd2 100644 --- a/test/workers/purge_expired_activity_test.exs +++ b/test/workers/purge_expired_activity_test.exs @@ -44,25 +44,4 @@ test "enqueue job" do assert %Oban.Job{} = Pleroma.Workers.PurgeExpiredActivity.get_expiration(activity.id) end - - test "don't delete pinned posts, schedule deletion on next day" do - activity = insert(:note_activity) - - assert {:ok, _} = - PurgeExpiredActivity.enqueue(%{ - activity_id: activity.id, - expires_at: DateTime.utc_now(), - validate: false - }) - - user = Pleroma.User.get_by_ap_id(activity.actor) - {:ok, activity} = Pleroma.Web.CommonAPI.pin(activity.id, user) - - assert %{success: 1, failure: 0} == - Oban.drain_queue(queue: :activity_expiration, with_scheduled: true) - - job = Pleroma.Workers.PurgeExpiredActivity.get_expiration(activity.id) - - assert DateTime.diff(job.scheduled_at, DateTime.add(DateTime.utc_now(), 24 * 3600)) in [0, 1] - end end From 29c1178c2b20eb1b389c7e1d35b58af05f48e8a2 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 4 Sep 2020 12:05:17 +0300 Subject: [PATCH 119/128] migration fix --- .../20200825061316_move_activity_expirations_to_oban.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/priv/repo/migrations/20200825061316_move_activity_expirations_to_oban.exs b/priv/repo/migrations/20200825061316_move_activity_expirations_to_oban.exs index 2bfefceb0..137933368 100644 --- a/priv/repo/migrations/20200825061316_move_activity_expirations_to_oban.exs +++ b/priv/repo/migrations/20200825061316_move_activity_expirations_to_oban.exs @@ -13,7 +13,7 @@ def change do select: %{id: e.id, activity_id: e.activity_id, scheduled_at: e.scheduled_at} ) |> Pleroma.Repo.stream() - |> Enum.each(fn expiration -> + |> Stream.each(fn expiration -> with {:ok, expires_at} <- DateTime.from_naive(expiration.scheduled_at, "Etc/UTC") do Pleroma.Workers.PurgeExpiredActivity.enqueue(%{ activity_id: FlakeId.to_string(expiration.activity_id), From f24828a3e848e6ce3bcdd254e8c6e451898cfdf7 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Mon, 7 Sep 2020 20:21:32 +0300 Subject: [PATCH 120/128] oban warning --- lib/pleroma/config/oban.ex | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/config/oban.ex b/lib/pleroma/config/oban.ex index 81758c93d..9f601b1a3 100644 --- a/lib/pleroma/config/oban.ex +++ b/lib/pleroma/config/oban.ex @@ -5,7 +5,11 @@ def warn do oban_config = Pleroma.Config.get(Oban) crontab = - [Pleroma.Workers.Cron.StatsWorker, Pleroma.Workers.Cron.ClearOauthTokenWorker] + [ + Pleroma.Workers.Cron.StatsWorker, + Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker, + Pleroma.Workers.Cron.ClearOauthTokenWorker + ] |> Enum.reduce(oban_config[:crontab], fn removed_worker, acc -> with acc when is_list(acc) <- acc, setting when is_tuple(setting) <- From 4954667fb24ee6ab7b1bf3b676f7e88a582130cf Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Mon, 7 Sep 2020 20:22:14 +0300 Subject: [PATCH 121/128] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 14c0252f7..79cf02c96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - **Breaking:** Removed `Pleroma.Workers.Cron.StatsWorker` setting from Oban `:crontab`. - **Breaking:** Removed `Pleroma.Workers.Cron.ClearOauthTokenWorker` setting from Oban `:crontab` config. +- **Breaking:** Removed `Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker` setting from Oban `:crontab`. ## [2.1.1] - 2020-09-08 From 2c2094d4b2722cf511e3db8288c3754a48038f05 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Mon, 7 Sep 2020 20:57:38 +0300 Subject: [PATCH 122/128] configurable lifetime for ephemeral activities --- config/config.exs | 2 +- config/description.exs | 6 ++++++ docs/configuration/cheatsheet.md | 13 +++++++++++++ lib/pleroma/workers/purge_expired_activity.ex | 3 ++- .../controllers/status_controller_test.exs | 8 ++++---- 5 files changed, 26 insertions(+), 6 deletions(-) diff --git a/config/config.exs b/config/config.exs index d975db31e..88c47fd03 100644 --- a/config/config.exs +++ b/config/config.exs @@ -654,7 +654,7 @@ account_confirmation_resend: {8_640_000, 5}, ap_routes: {60_000, 15} -config :pleroma, Pleroma.Workers.PurgeExpiredActivity, enabled: true +config :pleroma, Pleroma.Workers.PurgeExpiredActivity, enabled: true, min_lifetime: 600 config :pleroma, Pleroma.Plugs.RemoteIp, enabled: true diff --git a/config/description.exs b/config/description.exs index 1253944de..82c7bc6a7 100644 --- a/config/description.exs +++ b/config/description.exs @@ -2480,6 +2480,12 @@ key: :enabled, type: :boolean, description: "Enables expired activities addition & deletion" + }, + %{ + key: :min_lifetime, + type: :integer, + description: "Minimum lifetime for ephemeral activity (in seconds)", + suggestions: [600] } ] }, diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index d0bebbd45..8f2425384 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -1091,3 +1091,16 @@ config :pleroma, :frontends, ``` This would serve the frontend from the the folder at `$instance_static/frontends/pleroma/stable`. You have to copy the frontend into this folder yourself. You can choose the name and ref any way you like, but they will be used by mix tasks to automate installation in the future, the name referring to the project and the ref referring to a commit. + +## Ephemeral activities + +Settings to enable and configure expiration for ephemeral activities + +* `:enabled` - enables ephemeral activities creation +* `:min_lifetime` - minimum lifetime for ephemeral activities (in seconds). Default: 10 minutes. + +Example: + +```elixir + config :pleroma, Pleroma.Workers.PurgeExpiredActivity, enabled: true, min_lifetime: 600 +``` diff --git a/lib/pleroma/workers/purge_expired_activity.ex b/lib/pleroma/workers/purge_expired_activity.ex index f981eda8e..ffcb89dc3 100644 --- a/lib/pleroma/workers/purge_expired_activity.ex +++ b/lib/pleroma/workers/purge_expired_activity.ex @@ -77,6 +77,7 @@ def get_expiration(id) do def expires_late_enough?(scheduled_at) do now = DateTime.utc_now() diff = DateTime.diff(scheduled_at, now, :millisecond) - diff > :timer.hours(1) + min_lifetime = Pleroma.Config.get([__MODULE__, :min_lifetime], 600) + diff > :timer.seconds(min_lifetime) end end diff --git a/test/web/mastodon_api/controllers/status_controller_test.exs b/test/web/mastodon_api/controllers/status_controller_test.exs index 82ea73898..633a25e50 100644 --- a/test/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/web/mastodon_api/controllers/status_controller_test.exs @@ -129,8 +129,8 @@ test "posting a status", %{conn: conn} do test "it fails to create a status if `expires_in` is less or equal than an hour", %{ conn: conn } do - # 1 hour - expires_in = 60 * 60 + # 1 minute + expires_in = 1 * 60 assert %{"error" => "Expiry date is too soon"} = conn @@ -141,8 +141,8 @@ test "it fails to create a status if `expires_in` is less or equal than an hour" }) |> json_response_and_validate_schema(422) - # 30 minutes - expires_in = 30 * 60 + # 5 minutes + expires_in = 5 * 60 assert %{"error" => "Expiry date is too soon"} = conn From a098e10fd6d9f3b6573e2fb6333335d40a9bf330 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 8 Sep 2020 12:07:33 +0300 Subject: [PATCH 123/128] Document ephemeral activity changes better Also remove the example from the cheatsheet, there is no need for it when the types are simple --- CHANGELOG.md | 3 +++ docs/configuration/cheatsheet.md | 8 +------- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 79cf02c96..a58a18c8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - **Breaking:** Removed `Pleroma.Workers.Cron.ClearOauthTokenWorker` setting from Oban `:crontab` config. - **Breaking:** Removed `Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker` setting from Oban `:crontab`. +### Changed +- Minimum lifetime for ephmeral activities changed to 10 minutes and made configurable (`:min_lifetime` option). + ## [2.1.1] - 2020-09-08 ### Security diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 8f2425384..7cf1d1ce7 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -1092,15 +1092,9 @@ config :pleroma, :frontends, This would serve the frontend from the the folder at `$instance_static/frontends/pleroma/stable`. You have to copy the frontend into this folder yourself. You can choose the name and ref any way you like, but they will be used by mix tasks to automate installation in the future, the name referring to the project and the ref referring to a commit. -## Ephemeral activities +## Ephemeral activities (Pleroma.Workers.PurgeExpiredActivity) Settings to enable and configure expiration for ephemeral activities * `:enabled` - enables ephemeral activities creation * `:min_lifetime` - minimum lifetime for ephemeral activities (in seconds). Default: 10 minutes. - -Example: - -```elixir - config :pleroma, Pleroma.Workers.PurgeExpiredActivity, enabled: true, min_lifetime: 600 -``` From 15aece72382fe1862a58728b9d02990147f91365 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 8 Sep 2020 15:11:18 +0300 Subject: [PATCH 124/128] remove validate_expires_at from enqueue method --- lib/mix/tasks/pleroma/database.ex | 3 +- lib/pleroma/workers/purge_expired_activity.ex | 13 +----- ...1316_move_activity_expirations_to_oban.exs | 3 +- test/workers/purge_expired_activity_test.exs | 42 ++++++++++++------- 4 files changed, 30 insertions(+), 31 deletions(-) diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index aab4b5e9a..7f1108dcf 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -155,8 +155,7 @@ def run(["ensure_expiration"]) do Pleroma.Workers.PurgeExpiredActivity.enqueue(%{ activity_id: activity.id, - expires_at: expires_at, - validate: false + expires_at: expires_at }) end) end) diff --git a/lib/pleroma/workers/purge_expired_activity.ex b/lib/pleroma/workers/purge_expired_activity.ex index ffcb89dc3..c168890a2 100644 --- a/lib/pleroma/workers/purge_expired_activity.ex +++ b/lib/pleroma/workers/purge_expired_activity.ex @@ -18,8 +18,7 @@ defmodule Pleroma.Workers.PurgeExpiredActivity do | {:error, :expired_activities_disabled} | {:error, :expiration_too_close} def enqueue(args) do - with true <- enabled?(), - args when is_map(args) <- validate_expires_at(args) do + with true <- enabled?() do {scheduled_at, args} = Map.pop(args, :expires_at) args @@ -42,16 +41,6 @@ defp enabled? do end end - defp validate_expires_at(%{validate: false} = args), do: Map.delete(args, :validate) - - defp validate_expires_at(args) do - if expires_late_enough?(args[:expires_at]) do - args - else - {:error, :expiration_too_close} - end - end - defp find_activity(id) do with nil <- Activity.get_by_id_with_object(id) do {:error, :activity_not_found} diff --git a/priv/repo/migrations/20200825061316_move_activity_expirations_to_oban.exs b/priv/repo/migrations/20200825061316_move_activity_expirations_to_oban.exs index 137933368..cdc00d20b 100644 --- a/priv/repo/migrations/20200825061316_move_activity_expirations_to_oban.exs +++ b/priv/repo/migrations/20200825061316_move_activity_expirations_to_oban.exs @@ -17,8 +17,7 @@ def change do with {:ok, expires_at} <- DateTime.from_naive(expiration.scheduled_at, "Etc/UTC") do Pleroma.Workers.PurgeExpiredActivity.enqueue(%{ activity_id: FlakeId.to_string(expiration.activity_id), - expires_at: expires_at, - validate: false + expires_at: expires_at }) end end) diff --git a/test/workers/purge_expired_activity_test.exs b/test/workers/purge_expired_activity_test.exs index 8b5dc9fd2..b5938776d 100644 --- a/test/workers/purge_expired_activity_test.exs +++ b/test/workers/purge_expired_activity_test.exs @@ -10,21 +10,6 @@ defmodule Pleroma.Workers.PurgeExpiredActivityTest do alias Pleroma.Workers.PurgeExpiredActivity - test "denies expirations that don't live long enough" do - activity = insert(:note_activity) - - assert {:error, :expiration_too_close} = - PurgeExpiredActivity.enqueue(%{ - activity_id: activity.id, - expires_at: DateTime.utc_now() - }) - - refute_enqueued( - worker: Pleroma.Workers.PurgeExpiredActivity, - args: %{activity_id: activity.id} - ) - end - test "enqueue job" do activity = insert(:note_activity) @@ -44,4 +29,31 @@ test "enqueue job" do assert %Oban.Job{} = Pleroma.Workers.PurgeExpiredActivity.get_expiration(activity.id) end + + test "error if user was not found" do + activity = insert(:note_activity) + + assert {:ok, _} = + PurgeExpiredActivity.enqueue(%{ + activity_id: activity.id, + expires_at: DateTime.add(DateTime.utc_now(), 3601) + }) + + user = Pleroma.User.get_by_ap_id(activity.actor) + Pleroma.Repo.delete(user) + + assert {:error, :user_not_found} = + perform_job(Pleroma.Workers.PurgeExpiredActivity, %{activity_id: activity.id}) + end + + test "error if actiivity was not found" do + assert {:ok, _} = + PurgeExpiredActivity.enqueue(%{ + activity_id: "some_id", + expires_at: DateTime.add(DateTime.utc_now(), 3601) + }) + + assert {:error, :activity_not_found} = + perform_job(Pleroma.Workers.PurgeExpiredActivity, %{activity_id: "some_if"}) + end end From 82b56cdb9bc01dcf4dbd2ac0c06103af0900787d Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 10 Sep 2020 21:53:58 +0300 Subject: [PATCH 125/128] CHANGELOG.md: clarify that the functionality is not removed, just the config options --- CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a58a18c8c..75357f05e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,9 +12,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Removed -- **Breaking:** Removed `Pleroma.Workers.Cron.StatsWorker` setting from Oban `:crontab`. -- **Breaking:** Removed `Pleroma.Workers.Cron.ClearOauthTokenWorker` setting from Oban `:crontab` config. -- **Breaking:** Removed `Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker` setting from Oban `:crontab`. +- **Breaking:** `Pleroma.Workers.Cron.StatsWorker` setting from Oban `:crontab` (moved to a simpler implementation). +- **Breaking:** `Pleroma.Workers.Cron.ClearOauthTokenWorker` setting from Oban `:crontab` (moved to scheduled jobs). +- **Breaking:** `Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker` setting from Oban `:crontab` (moved to scheduled jobs). ### Changed - Minimum lifetime for ephmeral activities changed to 10 minutes and made configurable (`:min_lifetime` option). From e3ca0a7e2d18ca9b3c809282678456d4517d39bc Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 11 Sep 2020 09:09:28 +0300 Subject: [PATCH 126/128] migration to remove old cron jobs --- .../20200911055909_remove_cron_jobs.exs | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 priv/repo/migrations/20200911055909_remove_cron_jobs.exs diff --git a/priv/repo/migrations/20200911055909_remove_cron_jobs.exs b/priv/repo/migrations/20200911055909_remove_cron_jobs.exs new file mode 100644 index 000000000..33897d128 --- /dev/null +++ b/priv/repo/migrations/20200911055909_remove_cron_jobs.exs @@ -0,0 +1,20 @@ +defmodule Pleroma.Repo.Migrations.RemoveCronJobs do + use Ecto.Migration + + import Ecto.Query, only: [from: 2] + + def up do + from(j in "oban_jobs", + where: + j.worker in ^[ + "Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker", + "Pleroma.Workers.Cron.StatsWorker", + "Pleroma.Workers.Cron.ClearOauthTokenWorker" + ], + select: [:id] + ) + |> Pleroma.Repo.delete_all() + end + + def down, do: :ok +end From dbc013f24c3885960714425f201e372335d22345 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 11 Sep 2020 11:22:50 +0200 Subject: [PATCH 127/128] instance: Handle not getting a favicon --- lib/pleroma/instances/instance.ex | 12 ++--- test/web/instances/instance_test.exs | 71 ++++++++++++++++++---------- 2 files changed, 50 insertions(+), 33 deletions(-) diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index 8bf53c090..6948651c7 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -159,13 +159,11 @@ defp scrape_favicon(%URI{} = instance_uri) do Pleroma.HTTP.get(to_string(instance_uri), [{"accept", "text/html"}], adapter: [pool: :media] ), - favicon_rel <- - html - |> Floki.parse_document!() - |> Floki.attribute("link[rel=icon]", "href") - |> List.first(), - favicon <- URI.merge(instance_uri, favicon_rel) |> to_string(), - true <- is_binary(favicon) do + {_, [favicon_rel | _]} when is_binary(favicon_rel) <- + {:parse, + html |> Floki.parse_document!() |> Floki.attribute("link[rel=icon]", "href")}, + {_, favicon} when is_binary(favicon) <- + {:merge, URI.merge(instance_uri, favicon_rel) |> to_string()} do favicon else _ -> nil diff --git a/test/web/instances/instance_test.exs b/test/web/instances/instance_test.exs index dc6ace843..4f0805100 100644 --- a/test/web/instances/instance_test.exs +++ b/test/web/instances/instance_test.exs @@ -99,35 +99,54 @@ test "does NOT modify `unreachable_since` value of existing record in case it's end end - test "Scrapes favicon URLs" do - Tesla.Mock.mock(fn %{url: "https://favicon.example.org/"} -> - %Tesla.Env{ - status: 200, - body: ~s[] - } - end) + describe "get_or_update_favicon/1" do + test "Scrapes favicon URLs" do + Tesla.Mock.mock(fn %{url: "https://favicon.example.org/"} -> + %Tesla.Env{ + status: 200, + body: ~s[] + } + end) - assert "https://favicon.example.org/favicon.png" == - Instance.get_or_update_favicon(URI.parse("https://favicon.example.org/")) - end + assert "https://favicon.example.org/favicon.png" == + Instance.get_or_update_favicon(URI.parse("https://favicon.example.org/")) + end - test "Returns nil on too long favicon URLs" do - long_favicon_url = - "https://Lorem.ipsum.dolor.sit.amet/consecteturadipiscingelit/Praesentpharetrapurusutaliquamtempus/Mauriseulaoreetarcu/atfacilisisorci/Nullamporttitor/nequesedfeugiatmollis/dolormagnaefficiturlorem/nonpretiumsapienorcieurisus/Nullamveleratsem/Maecenassedaccumsanexnam/favicon.png" + test "Returns nil on too long favicon URLs" do + long_favicon_url = + "https://Lorem.ipsum.dolor.sit.amet/consecteturadipiscingelit/Praesentpharetrapurusutaliquamtempus/Mauriseulaoreetarcu/atfacilisisorci/Nullamporttitor/nequesedfeugiatmollis/dolormagnaefficiturlorem/nonpretiumsapienorcieurisus/Nullamveleratsem/Maecenassedaccumsanexnam/favicon.png" - Tesla.Mock.mock(fn %{url: "https://long-favicon.example.org/"} -> - %Tesla.Env{ - status: 200, - body: ~s[] - } - end) + Tesla.Mock.mock(fn %{url: "https://long-favicon.example.org/"} -> + %Tesla.Env{ + status: 200, + body: + ~s[] + } + end) - assert capture_log(fn -> - assert nil == - Instance.get_or_update_favicon( - URI.parse("https://long-favicon.example.org/") - ) - end) =~ - "Instance.get_or_update_favicon(\"long-favicon.example.org\") error: %Postgrex.Error{" + assert capture_log(fn -> + assert nil == + Instance.get_or_update_favicon( + URI.parse("https://long-favicon.example.org/") + ) + end) =~ + "Instance.get_or_update_favicon(\"long-favicon.example.org\") error: %Postgrex.Error{" + end + + test "Handles not getting a favicon URL properly" do + Tesla.Mock.mock(fn %{url: "https://no-favicon.example.org/"} -> + %Tesla.Env{ + status: 200, + body: ~s[

I wil look down and whisper "GNO.."

] + } + end) + + refute capture_log(fn -> + assert nil == + Instance.get_or_update_favicon( + URI.parse("https://no-favicon.example.org/") + ) + end) =~ "Instance.scrape_favicon(\"https://no-favicon.example.org/\") error: " + end end end From 89a7efab69d905cc3521388b1e1cf43851848627 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 11 Sep 2020 14:22:54 +0300 Subject: [PATCH 128/128] ConnectionPool: Log possible HTTP1 blocks --- lib/pleroma/gun/conn.ex | 12 ++++++------ lib/pleroma/gun/connection_pool/worker.ex | 22 ++++++++++++++++------ lib/pleroma/telemetry/logger.ex | 18 ++++++++++++++++-- 3 files changed, 38 insertions(+), 14 deletions(-) diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex index 75b1ffc0a..477e19c6e 100644 --- a/lib/pleroma/gun/conn.ex +++ b/lib/pleroma/gun/conn.ex @@ -50,10 +50,10 @@ defp do_open(uri, %{proxy: {proxy_host, proxy_port}} = opts) do with open_opts <- Map.delete(opts, :tls_opts), {:ok, conn} <- Gun.open(proxy_host, proxy_port, open_opts), - {:ok, _} <- Gun.await_up(conn, opts[:connect_timeout]), + {:ok, protocol} <- Gun.await_up(conn, opts[:connect_timeout]), stream <- Gun.connect(conn, connect_opts), {:response, :fin, 200, _} <- Gun.await(conn, stream) do - {:ok, conn} + {:ok, conn, protocol} else error -> Logger.warn( @@ -88,8 +88,8 @@ defp do_open(uri, %{proxy: {proxy_type, proxy_host, proxy_port}} = opts) do |> Map.put(:socks_opts, socks_opts) with {:ok, conn} <- Gun.open(proxy_host, proxy_port, opts), - {:ok, _} <- Gun.await_up(conn, opts[:connect_timeout]) do - {:ok, conn} + {:ok, protocol} <- Gun.await_up(conn, opts[:connect_timeout]) do + {:ok, conn, protocol} else error -> Logger.warn( @@ -106,8 +106,8 @@ defp do_open(%URI{host: host, port: port} = uri, opts) do host = Pleroma.HTTP.AdapterHelper.parse_host(host) with {:ok, conn} <- Gun.open(host, port, opts), - {:ok, _} <- Gun.await_up(conn, opts[:connect_timeout]) do - {:ok, conn} + {:ok, protocol} <- Gun.await_up(conn, opts[:connect_timeout]) do + {:ok, conn, protocol} else error -> Logger.warn( diff --git a/lib/pleroma/gun/connection_pool/worker.ex b/lib/pleroma/gun/connection_pool/worker.ex index c36332817..49d41e4c7 100644 --- a/lib/pleroma/gun/connection_pool/worker.ex +++ b/lib/pleroma/gun/connection_pool/worker.ex @@ -15,7 +15,7 @@ def init([_key, _uri, _opts, _client_pid] = opts) do @impl true def handle_continue({:connect, [key, uri, opts, client_pid]}, _) do - with {:ok, conn_pid} <- Gun.Conn.open(uri, opts), + with {:ok, conn_pid, protocol} <- Gun.Conn.open(uri, opts), Process.link(conn_pid) do time = :erlang.monotonic_time(:millisecond) @@ -27,8 +27,12 @@ def handle_continue({:connect, [key, uri, opts, client_pid]}, _) do send(client_pid, {:conn_pid, conn_pid}) {:noreply, - %{key: key, timer: nil, client_monitors: %{client_pid => Process.monitor(client_pid)}}, - :hibernate} + %{ + key: key, + timer: nil, + client_monitors: %{client_pid => Process.monitor(client_pid)}, + protocol: protocol + }, :hibernate} else err -> {:stop, {:shutdown, err}, nil} @@ -53,14 +57,20 @@ def handle_cast({:remove_client, client_pid}, state) do end @impl true - def handle_call(:add_client, {client_pid, _}, %{key: key} = state) do + def handle_call(:add_client, {client_pid, _}, %{key: key, protocol: protocol} = state) do time = :erlang.monotonic_time(:millisecond) - {{conn_pid, _, _, _}, _} = + {{conn_pid, used_by, _, _}, _} = Registry.update_value(@registry, key, fn {conn_pid, used_by, crf, last_reference} -> {conn_pid, [client_pid | used_by], crf(time - last_reference, crf), time} end) + :telemetry.execute( + [:pleroma, :connection_pool, :client, :add], + %{client_pid: client_pid, clients: used_by}, + %{key: state.key, protocol: protocol} + ) + state = if state.timer != nil do Process.cancel_timer(state[:timer]) @@ -131,7 +141,7 @@ def handle_info({:gun_down, _pid, _protocol, _reason, _killed_streams}, state) d @impl true def handle_info({:DOWN, _ref, :process, pid, reason}, state) do :telemetry.execute( - [:pleroma, :connection_pool, :client_death], + [:pleroma, :connection_pool, :client, :dead], %{client_pid: pid, reason: reason}, %{key: state.key} ) diff --git a/lib/pleroma/telemetry/logger.ex b/lib/pleroma/telemetry/logger.ex index 4cacae02f..197b1d091 100644 --- a/lib/pleroma/telemetry/logger.ex +++ b/lib/pleroma/telemetry/logger.ex @@ -7,7 +7,8 @@ defmodule Pleroma.Telemetry.Logger do [:pleroma, :connection_pool, :reclaim, :start], [:pleroma, :connection_pool, :reclaim, :stop], [:pleroma, :connection_pool, :provision_failure], - [:pleroma, :connection_pool, :client_death] + [:pleroma, :connection_pool, :client, :dead], + [:pleroma, :connection_pool, :client, :add] ] def attach do :telemetry.attach_many("pleroma-logger", @events, &handle_event/4, []) @@ -62,7 +63,7 @@ def handle_event( end def handle_event( - [:pleroma, :connection_pool, :client_death], + [:pleroma, :connection_pool, :client, :dead], %{client_pid: client_pid, reason: reason}, %{key: key}, _ @@ -73,4 +74,17 @@ def handle_event( }" end) end + + def handle_event( + [:pleroma, :connection_pool, :client, :add], + %{clients: [_, _ | _] = clients}, + %{key: key, protocol: :http}, + _ + ) do + Logger.info(fn -> + "Pool worker for #{key}: #{length(clients)} clients are using an HTTP1 connection at the same time, head-of-line blocking might occur." + end) + end + + def handle_event([:pleroma, :connection_pool, :client, :add], _, _, _), do: :ok end