From fcf2f38d20eed40a53b03374467d9e52b013da07 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 8 May 2019 17:37:00 +0200 Subject: [PATCH 01/40] Conversations: Add a function to 'import' old DMs. --- lib/pleroma/conversation.ex | 14 ++++++++++++++ lib/pleroma/web/activity_pub/activity_pub.ex | 6 ++++++ test/conversation_test.exs | 18 ++++++++++++++++++ 3 files changed, 38 insertions(+) diff --git a/lib/pleroma/conversation.ex b/lib/pleroma/conversation.ex index 6e26c5fd4..aa73edd75 100644 --- a/lib/pleroma/conversation.ex +++ b/lib/pleroma/conversation.ex @@ -72,4 +72,18 @@ defmodule Pleroma.Conversation do e -> {:error, e} end end + + @doc """ + This is only meant to be run by a mix task. It creates conversations/participations for all direct messages in the database. + """ + def bump_for_all_activities() do + stream = + Pleroma.Web.ActivityPub.ActivityPub.fetch_direct_messages_query() + |> Repo.stream() + + Repo.transaction(fn -> + stream + |> Enum.each(&create_or_bump_for/1) + end) + end end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 8f8c23a9b..23cf4e9c4 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1061,4 +1061,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do contain_activity(activity, user) end) end + + def fetch_direct_messages_query() do + Activity + |> restrict_type(%{"type" => "Create"}) + |> restrict_visibility(%{visibility: "direct"}) + end end diff --git a/test/conversation_test.exs b/test/conversation_test.exs index f3300e7d1..59368b0e7 100644 --- a/test/conversation_test.exs +++ b/test/conversation_test.exs @@ -9,6 +9,24 @@ defmodule Pleroma.ConversationTest do import Pleroma.Factory + test "it goes through old direct conversations" do + user = insert(:user) + other_user = insert(:user) + + {:ok, _activity} = + CommonAPI.post(user, %{"visibility" => "direct", "status" => "hey @#{other_user.nickname}"}) + + Repo.delete_all(Conversation) + Repo.delete_all(Conversation.Participation) + + refute Repo.one(Conversation) + + Conversation.bump_for_all_activities() + + assert Repo.one(Conversation) + assert length(Repo.all(Conversation.Participation)) == 2 + end + test "it creates a conversation for given ap_id" do assert {:ok, %Conversation{} = conversation} = Conversation.create_for_ap_id("https://some_ap_id") From 920bd4705526d8dfa8ada516853bbb4e5438cbf1 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 8 May 2019 17:40:24 +0200 Subject: [PATCH 02/40] ActivityPub: Remove leftover printf debugging. --- lib/pleroma/web/activity_pub/activity_pub.ex | 42 ++++++++------------ 1 file changed, 16 insertions(+), 26 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 23cf4e9c4..cd8495035 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -533,22 +533,17 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_visibility(query, %{visibility: visibility}) when is_list(visibility) do if Enum.all?(visibility, &(&1 in @valid_visibilities)) do - query = - from( - a in query, - where: - fragment( - "activity_visibility(?, ?, ?) = ANY (?)", - a.actor, - a.recipients, - a.data, - ^visibility - ) - ) - - Ecto.Adapters.SQL.to_sql(:all, Repo, query) - - query + from( + a in query, + where: + fragment( + "activity_visibility(?, ?, ?) = ANY (?)", + a.actor, + a.recipients, + a.data, + ^visibility + ) + ) else Logger.error("Could not restrict visibility to #{visibility}") end @@ -556,16 +551,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_visibility(query, %{visibility: visibility}) when visibility in @valid_visibilities do - query = - from( - a in query, - where: - fragment("activity_visibility(?, ?, ?) = ?", a.actor, a.recipients, a.data, ^visibility) - ) - - Ecto.Adapters.SQL.to_sql(:all, Repo, query) - - query + from( + a in query, + where: + fragment("activity_visibility(?, ?, ?) = ?", a.actor, a.recipients, a.data, ^visibility) + ) end defp restrict_visibility(_query, %{visibility: visibility}) From a4598b5e8bc640ffc1a052438e21f3573ff837ee Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 8 May 2019 18:08:50 +0200 Subject: [PATCH 03/40] Visibility: Make it more resilient. --- lib/pleroma/web/activity_pub/visibility.ex | 9 +++++---- test/web/activity_pub/visibilty_test.exs | 10 ++++++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/activity_pub/visibility.ex b/lib/pleroma/web/activity_pub/visibility.ex index 6dee61dd6..e7613a5c8 100644 --- a/lib/pleroma/web/activity_pub/visibility.ex +++ b/lib/pleroma/web/activity_pub/visibility.ex @@ -13,11 +13,12 @@ defmodule Pleroma.Web.ActivityPub.Visibility do end def is_private?(activity) do - unless is_public?(activity) do - follower_address = User.get_cached_by_ap_id(activity.data["actor"]).follower_address - Enum.any?(activity.data["to"], &(&1 == follower_address)) + with false <- is_public?(activity), + %User{follower_address: follower_address} <- + User.get_cached_by_ap_id(activity.data["actor"]) do + follower_address in activity.data["to"] else - false + _ -> false end end diff --git a/test/web/activity_pub/visibilty_test.exs b/test/web/activity_pub/visibilty_test.exs index 24b96c4aa..ff0e72401 100644 --- a/test/web/activity_pub/visibilty_test.exs +++ b/test/web/activity_pub/visibilty_test.exs @@ -95,4 +95,14 @@ defmodule Pleroma.Web.ActivityPub.VisibilityTest do refute Visibility.visible_for_user?(private, unrelated) refute Visibility.visible_for_user?(direct, unrelated) end + + test "doesn't die when the user doesn't exist", + %{ + direct: direct, + user: user + } do + Repo.delete(user) + Cachex.clear(:user_cache) + refute Visibility.is_private?(direct) + end end From 6d19bb4eae43270099a68f749519ba0f323da01a Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 8 May 2019 18:09:07 +0200 Subject: [PATCH 04/40] Conversations: Add mix task to 'import' old DMs. --- lib/mix/tasks/pleroma/conversations.ex | 23 +++++++++++++++++++++++ lib/pleroma/conversation.ex | 11 +++++++---- 2 files changed, 30 insertions(+), 4 deletions(-) create mode 100644 lib/mix/tasks/pleroma/conversations.ex diff --git a/lib/mix/tasks/pleroma/conversations.ex b/lib/mix/tasks/pleroma/conversations.ex new file mode 100644 index 000000000..125d8851a --- /dev/null +++ b/lib/mix/tasks/pleroma/conversations.ex @@ -0,0 +1,23 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Mix.Tasks.Pleroma.Conversations do + use Mix.Task + alias Mix.Tasks.Pleroma.Common + alias Pleroma.Conversation + + @shortdoc "Manages Pleroma users" + @moduledoc """ + Manages Pleroma conversations. + + ## Create a conversation for all existing DMs. Can be safely re-run. + + mix pleroma.conversations bump_all + + """ + def run(["bump_all"]) do + Common.start_pleroma() + Conversation.bump_for_all_activities() + end +end diff --git a/lib/pleroma/conversation.ex b/lib/pleroma/conversation.ex index aa73edd75..10c2403e8 100644 --- a/lib/pleroma/conversation.ex +++ b/lib/pleroma/conversation.ex @@ -81,9 +81,12 @@ defmodule Pleroma.Conversation do Pleroma.Web.ActivityPub.ActivityPub.fetch_direct_messages_query() |> Repo.stream() - Repo.transaction(fn -> - stream - |> Enum.each(&create_or_bump_for/1) - end) + Repo.transaction( + fn -> + stream + |> Enum.each(&create_or_bump_for/1) + end, + timeout: :infinity + ) end end From e6d7f8d223b2604df38f8efa8baf09e2c607c487 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 8 May 2019 18:19:20 +0200 Subject: [PATCH 05/40] Credo fixes. --- lib/pleroma/conversation.ex | 2 +- lib/pleroma/web/activity_pub/activity_pub.ex | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/conversation.ex b/lib/pleroma/conversation.ex index 10c2403e8..0c6ca9f72 100644 --- a/lib/pleroma/conversation.ex +++ b/lib/pleroma/conversation.ex @@ -76,7 +76,7 @@ defmodule Pleroma.Conversation do @doc """ This is only meant to be run by a mix task. It creates conversations/participations for all direct messages in the database. """ - def bump_for_all_activities() do + def bump_for_all_activities do stream = Pleroma.Web.ActivityPub.ActivityPub.fetch_direct_messages_query() |> Repo.stream() diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index cd8495035..8137ac83b 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1052,7 +1052,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end) end - def fetch_direct_messages_query() do + def fetch_direct_messages_query do Activity |> restrict_type(%{"type" => "Create"}) |> restrict_visibility(%{visibility: "direct"}) From a33bec7d58091059d578f6b7537513de11eb0679 Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 9 May 2019 16:39:28 +0200 Subject: [PATCH 06/40] Conversations: Import order, import as read. --- lib/pleroma/conversation.ex | 6 +++--- lib/pleroma/conversation/participation.ex | 10 ++++++---- lib/pleroma/web/activity_pub/activity_pub.ex | 1 + test/conversation_test.exs | 4 +++- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/conversation.ex b/lib/pleroma/conversation.ex index 0c6ca9f72..5f6ab902c 100644 --- a/lib/pleroma/conversation.ex +++ b/lib/pleroma/conversation.ex @@ -45,7 +45,7 @@ defmodule Pleroma.Conversation do 2. Create a participation for all the people involved who don't have one already 3. Bump all relevant participations to 'unread' """ - def create_or_bump_for(activity) do + def create_or_bump_for(activity, opts \\ []) do with true <- Pleroma.Web.ActivityPub.Visibility.is_direct?(activity), object <- Pleroma.Object.normalize(activity), "Create" <- activity.data["type"], @@ -58,7 +58,7 @@ defmodule Pleroma.Conversation do participations = Enum.map(users, fn user -> {:ok, participation} = - Participation.create_for_user_and_conversation(user, conversation) + Participation.create_for_user_and_conversation(user, conversation, opts) participation end) @@ -84,7 +84,7 @@ defmodule Pleroma.Conversation do Repo.transaction( fn -> stream - |> Enum.each(&create_or_bump_for/1) + |> Enum.each(fn a -> create_or_bump_for(a, read: true) end) end, timeout: :infinity ) diff --git a/lib/pleroma/conversation/participation.ex b/lib/pleroma/conversation/participation.ex index 61021fb18..2a11f9069 100644 --- a/lib/pleroma/conversation/participation.ex +++ b/lib/pleroma/conversation/participation.ex @@ -22,15 +22,17 @@ defmodule Pleroma.Conversation.Participation do def creation_cng(struct, params) do struct - |> cast(params, [:user_id, :conversation_id]) + |> cast(params, [:user_id, :conversation_id, :read]) |> validate_required([:user_id, :conversation_id]) end - def create_for_user_and_conversation(user, conversation) do + def create_for_user_and_conversation(user, conversation, opts \\ []) do + read = !!opts[:read] + %__MODULE__{} - |> creation_cng(%{user_id: user.id, conversation_id: conversation.id}) + |> creation_cng(%{user_id: user.id, conversation_id: conversation.id, read: read}) |> Repo.insert( - on_conflict: [set: [read: false, updated_at: NaiveDateTime.utc_now()]], + on_conflict: [set: [read: read, updated_at: NaiveDateTime.utc_now()]], returning: true, conflict_target: [:user_id, :conversation_id] ) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 8137ac83b..728761ebd 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1056,5 +1056,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do Activity |> restrict_type(%{"type" => "Create"}) |> restrict_visibility(%{visibility: "direct"}) + |> order_by([activity], asc: activity.id) end end diff --git a/test/conversation_test.exs b/test/conversation_test.exs index 59368b0e7..cdec18f0f 100644 --- a/test/conversation_test.exs +++ b/test/conversation_test.exs @@ -24,7 +24,9 @@ defmodule Pleroma.ConversationTest do Conversation.bump_for_all_activities() assert Repo.one(Conversation) - assert length(Repo.all(Conversation.Participation)) == 2 + [participation, _p2] = Repo.all(Conversation.Participation) + + assert participation.read end test "it creates a conversation for given ap_id" do From 786f2c7a849bc4fa2bd4aac18de59ef6b2ed18c5 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 13 May 2019 11:16:54 -0500 Subject: [PATCH 07/40] Update shortdoc description --- lib/mix/tasks/pleroma/conversations.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mix/tasks/pleroma/conversations.ex b/lib/mix/tasks/pleroma/conversations.ex index 125d8851a..b52b9921a 100644 --- a/lib/mix/tasks/pleroma/conversations.ex +++ b/lib/mix/tasks/pleroma/conversations.ex @@ -7,7 +7,7 @@ defmodule Mix.Tasks.Pleroma.Conversations do alias Mix.Tasks.Pleroma.Common alias Pleroma.Conversation - @shortdoc "Manages Pleroma users" + @shortdoc "Manages Pleroma conversations." @moduledoc """ Manages Pleroma conversations. From 88cc423fa3c4baa698e99e1e62d251b6f81cc93d Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 15 May 2019 16:45:26 +0200 Subject: [PATCH 08/40] Dependencies: Update Ecto SQL. --- mix.exs | 5 ++++- mix.lock | 10 +++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/mix.exs b/mix.exs index b7b9d534d..2535fcf98 100644 --- a/mix.exs +++ b/mix.exs @@ -65,7 +65,10 @@ defmodule Pleroma.Mixfile do {:plug_cowboy, "~> 2.0"}, {:phoenix_pubsub, "~> 1.1"}, {:phoenix_ecto, "~> 4.0"}, - {:ecto_sql, "~>3.0.5"}, + {:ecto_sql, + git: "https://github.com/elixir-ecto/ecto_sql", + ref: "e839a9a327b632d73533ac8105ba360bc831cf83", + override: true}, {:postgrex, ">= 0.13.5"}, {:gettext, "~> 0.15"}, {:comeonin, "~> 4.1.1"}, diff --git a/mix.lock b/mix.lock index 0b24818c5..5ee5634d3 100644 --- a/mix.lock +++ b/mix.lock @@ -16,12 +16,12 @@ "cowlib": {:hex, :cowlib, "2.7.0", "3ef16e77562f9855a2605900cedb15c1462d76fb1be6a32fc3ae91973ee543d2", [:rebar3], [], "hexpm"}, "credo": {:hex, :credo, "0.9.3", "76fa3e9e497ab282e0cf64b98a624aa11da702854c52c82db1bf24e54ab7c97a", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:poison, ">= 0.0.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm"}, "crypt": {:git, "https://github.com/msantos/crypt", "1f2b58927ab57e72910191a7ebaeff984382a1d3", [ref: "1f2b58927ab57e72910191a7ebaeff984382a1d3"]}, - "db_connection": {:hex, :db_connection, "2.0.5", "ddb2ba6761a08b2bb9ca0e7d260e8f4dd39067426d835c24491a321b7f92a4da", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm"}, + "db_connection": {:hex, :db_connection, "2.0.6", "bde2f85d047969c5b5800cb8f4b3ed6316c8cb11487afedac4aa5f93fd39abfa", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm"}, "decimal": {:hex, :decimal, "1.7.0", "30d6b52c88541f9a66637359ddf85016df9eb266170d53105f02e4a67e00c5aa", [:mix], [], "hexpm"}, "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm"}, "earmark": {:hex, :earmark, "1.3.2", "b840562ea3d67795ffbb5bd88940b1bed0ed9fa32834915125ea7d02e35888a5", [:mix], [], "hexpm"}, - "ecto": {:hex, :ecto, "3.0.7", "44dda84ac6b17bbbdeb8ac5dfef08b7da253b37a453c34ab1a98de7f7e5fec7f", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:poison, "~> 2.2 or ~> 3.0", [hex: :poison, repo: "hexpm", optional: true]}], "hexpm"}, - "ecto_sql": {:hex, :ecto_sql, "3.0.5", "7e44172b4f7aca4469f38d7f6a3da394dbf43a1bcf0ca975e958cb957becd74e", [:mix], [{:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.0.6", [hex: :ecto, repo: "hexpm", optional: false]}, {:mariaex, "~> 0.9.1", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.14.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.3.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, + "ecto": {:hex, :ecto, "3.1.4", "69d852da7a9f04ede725855a35ede48d158ca11a404fe94f8b2fb3b2162cd3c9", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, + "ecto_sql": {:git, "https://github.com/elixir-ecto/ecto_sql", "e839a9a327b632d73533ac8105ba360bc831cf83", [ref: "e839a9a327b632d73533ac8105ba360bc831cf83"]}, "esshd": {:hex, :esshd, "0.1.0", "6f93a2062adb43637edad0ea7357db2702a4b80dd9683482fe00f5134e97f4c1", [:mix], [], "hexpm"}, "eternal": {:hex, :eternal, "1.2.0", "e2a6b6ce3b8c248f7dc31451aefca57e3bdf0e48d73ae5043229380a67614c41", [:mix], [], "hexpm"}, "ex2ms": {:hex, :ex2ms, "1.5.0", "19e27f9212be9a96093fed8cdfbef0a2b56c21237196d26760f11dfcfae58e97", [:mix], [], "hexpm"}, @@ -64,7 +64,7 @@ "plug_static_index_html": {:hex, :plug_static_index_html, "1.0.0", "840123d4d3975585133485ea86af73cb2600afd7f2a976f9f5fd8b3808e636a0", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm"}, "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm"}, - "postgrex": {:hex, :postgrex, "0.14.1", "63247d4a5ad6b9de57a0bac5d807e1c32d41e39c04b8a4156a26c63bcd8a2e49", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, + "postgrex": {:hex, :postgrex, "0.14.3", "5754dee2fdf6e9e508cbf49ab138df964278700b764177e8f3871e658b345a1e", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, "prometheus": {:hex, :prometheus, "4.2.2", "a830e77b79dc6d28183f4db050a7cac926a6c58f1872f9ef94a35cd989aceef8", [:mix, :rebar3], [], "hexpm"}, "prometheus_ecto": {:hex, :prometheus_ecto, "1.4.1", "6c768ea9654de871e5b32fab2eac348467b3021604ebebbcbd8bcbe806a65ed5", [:mix], [{:ecto, "~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm"}, "prometheus_ex": {:hex, :prometheus_ex, "3.0.5", "fa58cfd983487fc5ead331e9a3e0aa622c67232b3ec71710ced122c4c453a02f", [:mix], [{:prometheus, "~> 4.0", [hex: :prometheus, repo: "hexpm", optional: false]}], "hexpm"}, @@ -77,7 +77,7 @@ "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"}, "swoosh": {:hex, :swoosh, "0.20.0", "9a6c13822c9815993c03b6f8fccc370fcffb3c158d9754f67b1fdee6b3a5d928", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.12", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug, "~> 1.4", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, - "telemetry": {:hex, :telemetry, "0.3.0", "099a7f3ce31e4780f971b4630a3c22ec66d22208bc090fe33a2a3a6a67754a73", [:rebar3], [], "hexpm"}, + "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, "tesla": {:hex, :tesla, "1.2.1", "864783cc27f71dd8c8969163704752476cec0f3a51eb3b06393b3971dc9733ff", [:mix], [{:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "~> 4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}], "hexpm"}, "timex": {:hex, :timex, "3.5.0", "b0a23167da02d0fe4f1a4e104d1f929a00d348502b52432c05de875d0b9cffa5", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, From 958b4cfde916c9be71d7412fe1c90750ec578fdc Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sun, 24 Mar 2019 23:45:57 +0000 Subject: [PATCH 09/40] migrations: add function to see if a thread can be satisfied --- ...4222404_add_thread_visibility_function.exs | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 priv/repo/migrations/20190324222404_add_thread_visibility_function.exs diff --git a/priv/repo/migrations/20190324222404_add_thread_visibility_function.exs b/priv/repo/migrations/20190324222404_add_thread_visibility_function.exs new file mode 100644 index 000000000..cea0322e7 --- /dev/null +++ b/priv/repo/migrations/20190324222404_add_thread_visibility_function.exs @@ -0,0 +1,76 @@ +defmodule Pleroma.Repo.Migrations.AddThreadVisibilityFunction do + use Ecto.Migration + @disable_ddl_transaction true + + def up do + statement = """ + CREATE OR REPLACE FUNCTION thread_visibility(actor varchar, activity_id varchar) RETURNS boolean AS $$ + DECLARE + public varchar := 'https://www.w3.org/ns/activitystreams#Public'; + child objects%ROWTYPE; + activity activities%ROWTYPE; + actor_user users%ROWTYPE; + author users%ROWTYPE; + author_fa varchar; + BEGIN + --- Fetch our actor. + SELECT * INTO actor_user FROM users WHERE users.ap_id = actor; + + --- Fetch our initial activity. + SELECT * INTO activity FROM activities WHERE activities.data->>'id' = activity_id; + + LOOP + --- Ensure that we have an activity before continuing. + IF activity IS NULL THEN + RETURN true; + END IF; + + --- Normalize the child object into child. + SELECT * INTO child FROM objects + INNER JOIN activities ON COALESCE(activities.data->'object'->>'id', activities.data->>'object') = objects.data->>'id' + WHERE COALESCE(activity.data->'object'->>'id', activity.data->>'object') = objects.data->>'id'; + + --- Fetch the author. + SELECT * INTO author FROM users WHERE users.ap_id = activity.actor; + + --- Prepare author's AS2 followers collection. + SELECT COALESCE(author.follower_address, '') INTO author_fa; + + --- Check visibility. + IF activity.actor = actor THEN + --- activity visible + NULL; + ELSIF ARRAY[public] && activity.recipients THEN + --- activity visible + NULL; + ELSIF ARRAY[author_fa] && activity.recipients AND ARRAY[author_fa] && actor_user.following THEN + --- activity visible + NULL; + ELSIF ARRAY[actor] && activity.recipients THEN + --- activity visible + NULL; + ELSE + --- activity not visible, break out of the loop + RETURN false; + END IF; + + --- If there's a parent, load it and do this all over again. + IF (child.data->'inReplyTo' IS NOT NULL) AND (child.data->'inReplyTo' != 'null'::jsonb) THEN + SELECT * INTO activity FROM activities + INNER JOIN objects ON COALESCE(activities.data->'object'->>'id', activities.data->>'object') = objects.data->>'id' + WHERE child.data->>'inReplyTo' = objects.data->>'id'; + ELSE + RETURN true; + END IF; + END LOOP; + END; + $$ LANGUAGE plpgsql IMMUTABLE; + """ + + execute(statement) + end + + def down do + execute("drop function thread_visibility(actor varchar, activity_id varchar)") + end +end From 0387f5213805cdc4e0bf86f98797cefcd03ba61d Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Mon, 25 Mar 2019 00:06:02 +0000 Subject: [PATCH 10/40] activitypub: add restrict_thread_visibility() --- lib/pleroma/web/activity_pub/activity_pub.ex | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 233fee4fa..fec1bcd3e 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -569,6 +569,20 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_visibility(query, _visibility), do: query + defp restrict_thread_visibility(query, %{"user" => %User{ap_id: ap_id}}) do + query = + from( + a in query, + where: fragment("thread_visibility(?, (?)->>'id') = true", ^ap_id, a.data) + ) + + Ecto.Adapters.SQL.to_sql(:all, Repo, query) + + query + end + + defp restrict_thread_visibility(query, _), do: query + def fetch_user_activities(user, reading_user, params \\ %{}) do params = params @@ -848,6 +862,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> restrict_muted(opts) |> restrict_media(opts) |> restrict_visibility(opts) + |> restrict_thread_visibility(opts) |> restrict_replies(opts) |> restrict_reblogs(opts) |> restrict_pinned(opts) From de114ffbb0f92d24fd370adaaf43ff301ab04b4b Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Mon, 25 Mar 2019 00:10:20 +0000 Subject: [PATCH 11/40] activitypub: remove contain_timeline() --- lib/pleroma/web/activity_pub/activity_pub.ex | 8 -------- lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 1 - lib/pleroma/web/twitter_api/twitter_api_controller.ex | 4 +--- 3 files changed, 1 insertion(+), 12 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index fec1bcd3e..e544d0c50 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -980,12 +980,4 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do def contain_activity(%Activity{} = activity, %User{} = user) do contain_broken_threads(activity, user) end - - # do post-processing on a timeline - def contain_timeline(timeline, user) do - timeline - |> Enum.filter(fn activity -> - contain_activity(activity, user) - end) - end end diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 87e597074..66056a846 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -303,7 +303,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do activities = [user.ap_id | user.following] |> ActivityPub.fetch_activities(params) - |> ActivityPub.contain_timeline(user) |> Enum.reverse() conn diff --git a/lib/pleroma/web/twitter_api/twitter_api_controller.ex b/lib/pleroma/web/twitter_api/twitter_api_controller.ex index 3c5a70be9..31e86685a 100644 --- a/lib/pleroma/web/twitter_api/twitter_api_controller.ex +++ b/lib/pleroma/web/twitter_api/twitter_api_controller.ex @@ -101,9 +101,7 @@ defmodule Pleroma.Web.TwitterAPI.Controller do |> Map.put("blocking_user", user) |> Map.put("user", user) - activities = - ActivityPub.fetch_activities([user.ap_id | user.following], params) - |> ActivityPub.contain_timeline(user) + activities = ActivityPub.fetch_activities([user.ap_id | user.following], params) conn |> put_view(ActivityView) From 31db31c5879a2dedcc8dd4c671c4c9a79656355a Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Mon, 25 Mar 2019 00:38:28 +0000 Subject: [PATCH 12/40] activitypub: visibility: use SQL thread_visibility() function instead of manually walking the thread --- lib/pleroma/web/activity_pub/visibility.ex | 26 +++++++--------------- 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/lib/pleroma/web/activity_pub/visibility.ex b/lib/pleroma/web/activity_pub/visibility.ex index b38ee0442..46dd46575 100644 --- a/lib/pleroma/web/activity_pub/visibility.ex +++ b/lib/pleroma/web/activity_pub/visibility.ex @@ -1,6 +1,7 @@ defmodule Pleroma.Web.ActivityPub.Visibility do alias Pleroma.Activity alias Pleroma.Object + alias Pleroma.Repo alias Pleroma.User def is_public?(%Object{data: %{"type" => "Tombstone"}}), do: false @@ -38,25 +39,14 @@ defmodule Pleroma.Web.ActivityPub.Visibility do visible_for_user?(activity, nil) || Enum.any?(x, &(&1 in y)) end - # guard - def entire_thread_visible_for_user?(nil, _user), do: false + def entire_thread_visible_for_user?(%Activity{} = activity, %User{} = user) do + {:ok, %{rows: [[result]]}} = + Ecto.Adapters.SQL.query(Repo, "SELECT thread_visibility($1, $2)", [ + user.ap_id, + activity.data["id"] + ]) - # XXX: Probably even more inefficient than the previous implementation intended to be a placeholder untill https://git.pleroma.social/pleroma/pleroma/merge_requests/971 is in develop - # credo:disable-for-previous-line Credo.Check.Readability.MaxLineLength - - def entire_thread_visible_for_user?( - %Activity{} = tail, - # %Activity{data: %{"object" => %{"inReplyTo" => parent_id}}} = tail, - user - ) do - case Object.normalize(tail) do - %{data: %{"inReplyTo" => parent_id}} when is_binary(parent_id) -> - parent = Activity.get_in_reply_to_activity(tail) - visible_for_user?(tail, user) && entire_thread_visible_for_user?(parent, user) - - _ -> - visible_for_user?(tail, user) - end + result end def get_visibility(object) do From c7644313e72520a371e4bd417b1ff852365849b6 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Mon, 25 Mar 2019 01:23:15 +0000 Subject: [PATCH 13/40] test: update obsolete test --- test/web/activity_pub/activity_pub_test.exs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 0f90aa1ac..b41f6ab81 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -968,7 +968,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do assert length(activities) == 3 - activities = ActivityPub.contain_timeline(activities, user1) + activities = + ActivityPub.fetch_activities([user1.ap_id | user1.following], %{"user" => user1}) assert [public_activity, private_activity_1] == activities assert length(activities) == 2 From 75ce6adcffd2dbbc2ca2f83d7fe2d7fd659cd2f4 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Mon, 25 Mar 2019 02:56:13 +0000 Subject: [PATCH 14/40] migration: only care about Create activities --- .../20190324222404_add_thread_visibility_function.exs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/priv/repo/migrations/20190324222404_add_thread_visibility_function.exs b/priv/repo/migrations/20190324222404_add_thread_visibility_function.exs index cea0322e7..11aa47e83 100644 --- a/priv/repo/migrations/20190324222404_add_thread_visibility_function.exs +++ b/priv/repo/migrations/20190324222404_add_thread_visibility_function.exs @@ -25,6 +25,11 @@ defmodule Pleroma.Repo.Migrations.AddThreadVisibilityFunction do RETURN true; END IF; + --- We only care about Create activities. + IF activity.data->>'type' != 'Create' THEN + RETURN true; + END IF; + --- Normalize the child object into child. SELECT * INTO child FROM objects INNER JOIN activities ON COALESCE(activities.data->'object'->>'id', activities.data->>'object') = objects.data->>'id' From 0aada88b5594b6714b8d65f8bee9c325d77d6e7b Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 8 May 2019 23:17:51 +0000 Subject: [PATCH 15/40] bbs: chase timeline containment patch --- lib/pleroma/bbs/handler.ex | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/pleroma/bbs/handler.ex b/lib/pleroma/bbs/handler.ex index 106fe5d18..f34be961f 100644 --- a/lib/pleroma/bbs/handler.ex +++ b/lib/pleroma/bbs/handler.ex @@ -95,7 +95,6 @@ defmodule Pleroma.BBS.Handler do activities = [user.ap_id | user.following] |> ActivityPub.fetch_activities(params) - |> ActivityPub.contain_timeline(user) Enum.each(activities, fn activity -> puts_activity(activity) From 12f45e2a8907c74c6b65d866bc3bab547b31edfa Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 15 May 2019 16:22:52 +0000 Subject: [PATCH 16/40] update migration --- ...n.exs => 20190515222404_add_thread_visibility_function.exs} | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) rename priv/repo/migrations/{20190324222404_add_thread_visibility_function.exs => 20190515222404_add_thread_visibility_function.exs} (97%) diff --git a/priv/repo/migrations/20190324222404_add_thread_visibility_function.exs b/priv/repo/migrations/20190515222404_add_thread_visibility_function.exs similarity index 97% rename from priv/repo/migrations/20190324222404_add_thread_visibility_function.exs rename to priv/repo/migrations/20190515222404_add_thread_visibility_function.exs index 11aa47e83..a3f717b89 100644 --- a/priv/repo/migrations/20190324222404_add_thread_visibility_function.exs +++ b/priv/repo/migrations/20190515222404_add_thread_visibility_function.exs @@ -21,8 +21,9 @@ defmodule Pleroma.Repo.Migrations.AddThreadVisibilityFunction do LOOP --- Ensure that we have an activity before continuing. + --- If we don't, the thread is not satisfiable. IF activity IS NULL THEN - RETURN true; + RETURN false; END IF; --- We only care about Create activities. From f09c3afdf51eea17103d1445b31b7a269c474538 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 15 May 2019 16:23:01 +0000 Subject: [PATCH 17/40] chase test failures --- lib/pleroma/filter.ex | 3 ++- test/user_test.exs | 2 -- test/web/activity_pub/activity_pub_test.exs | 9 ++++++--- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/filter.ex b/lib/pleroma/filter.ex index 79efc29f0..90457dadf 100644 --- a/lib/pleroma/filter.ex +++ b/lib/pleroma/filter.ex @@ -38,7 +38,8 @@ defmodule Pleroma.Filter do query = from( f in Pleroma.Filter, - where: f.user_id == ^user_id + where: f.user_id == ^user_id, + order_by: [desc: :id] ) Repo.all(query) diff --git a/test/user_test.exs b/test/user_test.exs index 0b65e89e9..bb47b4958 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -873,7 +873,6 @@ defmodule Pleroma.UserTest do assert [activity] == ActivityPub.fetch_activities([user2.ap_id | user2.following], %{"user" => user2}) - |> ActivityPub.contain_timeline(user2) {:ok, _user} = User.deactivate(user) @@ -882,7 +881,6 @@ defmodule Pleroma.UserTest do assert [] == ActivityPub.fetch_activities([user2.ap_id | user2.following], %{"user" => user2}) - |> ActivityPub.contain_timeline(user2) end end diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index b41f6ab81..34e23b852 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -960,18 +960,21 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do "in_reply_to_status_id" => private_activity_2.id }) - activities = ActivityPub.fetch_activities([user1.ap_id | user1.following]) + activities = + ActivityPub.fetch_activities([user1.ap_id | user1.following]) + |> Enum.map(fn a -> a.id end) private_activity_1 = Activity.get_by_ap_id_with_object(private_activity_1.data["id"]) - assert [public_activity, private_activity_1, private_activity_3] == activities + assert [public_activity.id, private_activity_1.id, private_activity_3.id] == activities assert length(activities) == 3 activities = ActivityPub.fetch_activities([user1.ap_id | user1.following], %{"user" => user1}) + |> Enum.map(fn a -> a.id end) - assert [public_activity, private_activity_1] == activities + assert [public_activity.id, private_activity_1.id] == activities assert length(activities) == 2 end end From 71fa7eeb6fdc7cf2087a32fb515ad11b7bf90c01 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 15 May 2019 16:54:14 +0000 Subject: [PATCH 18/40] thread visibility function: significantly improve efficiency --- ...5222404_add_thread_visibility_function.exs | 27 +++++++------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/priv/repo/migrations/20190515222404_add_thread_visibility_function.exs b/priv/repo/migrations/20190515222404_add_thread_visibility_function.exs index a3f717b89..a4daf680b 100644 --- a/priv/repo/migrations/20190515222404_add_thread_visibility_function.exs +++ b/priv/repo/migrations/20190515222404_add_thread_visibility_function.exs @@ -10,8 +10,8 @@ defmodule Pleroma.Repo.Migrations.AddThreadVisibilityFunction do child objects%ROWTYPE; activity activities%ROWTYPE; actor_user users%ROWTYPE; - author users%ROWTYPE; author_fa varchar; + valid_recipients varchar[]; BEGIN --- Fetch our actor. SELECT * INTO actor_user FROM users WHERE users.ap_id = actor; @@ -36,26 +36,17 @@ defmodule Pleroma.Repo.Migrations.AddThreadVisibilityFunction do INNER JOIN activities ON COALESCE(activities.data->'object'->>'id', activities.data->>'object') = objects.data->>'id' WHERE COALESCE(activity.data->'object'->>'id', activity.data->>'object') = objects.data->>'id'; - --- Fetch the author. - SELECT * INTO author FROM users WHERE users.ap_id = activity.actor; + --- Fetch the author's AS2 following collection. + SELECT COALESCE(author.follower_address, '') INTO author_fa FROM users WHERE users.ap_id = activity.actor; - --- Prepare author's AS2 followers collection. - SELECT COALESCE(author.follower_address, '') INTO author_fa; + --- Prepare valid recipients array. + valid_recipients := ARRAY[actor, public]; + IF ARRAY[author_fa] && actor_user.following THEN + valid_recipients := valid_recipients || author_fa; + END IF; --- Check visibility. - IF activity.actor = actor THEN - --- activity visible - NULL; - ELSIF ARRAY[public] && activity.recipients THEN - --- activity visible - NULL; - ELSIF ARRAY[author_fa] && activity.recipients AND ARRAY[author_fa] && actor_user.following THEN - --- activity visible - NULL; - ELSIF ARRAY[actor] && activity.recipients THEN - --- activity visible - NULL; - ELSE + IF NOT valid_recipients && activity.recipients THEN --- activity not visible, break out of the loop RETURN false; END IF; From a591ab6112abdf162f4d6fdfbbcdd85bbaf75058 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 15 May 2019 16:56:46 +0000 Subject: [PATCH 19/40] activity pub: remove Ecto SQL query dumps --- lib/pleroma/web/activity_pub/activity_pub.ex | 6 ------ 1 file changed, 6 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index e544d0c50..7cd5b889b 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -540,8 +540,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do ) ) - Ecto.Adapters.SQL.to_sql(:all, Repo, query) - query else Logger.error("Could not restrict visibility to #{visibility}") @@ -557,8 +555,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do fragment("activity_visibility(?, ?, ?) = ?", a.actor, a.recipients, a.data, ^visibility) ) - Ecto.Adapters.SQL.to_sql(:all, Repo, query) - query end @@ -576,8 +572,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do where: fragment("thread_visibility(?, (?)->>'id') = true", ^ap_id, a.data) ) - Ecto.Adapters.SQL.to_sql(:all, Repo, query) - query end From f3971cbde3d69faec973717e1421f4a643ef947e Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 15 May 2019 17:02:40 +0000 Subject: [PATCH 20/40] thread visibility function: fix use of no longer used author variable --- .../20190515222404_add_thread_visibility_function.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/priv/repo/migrations/20190515222404_add_thread_visibility_function.exs b/priv/repo/migrations/20190515222404_add_thread_visibility_function.exs index a4daf680b..dc9abc998 100644 --- a/priv/repo/migrations/20190515222404_add_thread_visibility_function.exs +++ b/priv/repo/migrations/20190515222404_add_thread_visibility_function.exs @@ -37,7 +37,7 @@ defmodule Pleroma.Repo.Migrations.AddThreadVisibilityFunction do WHERE COALESCE(activity.data->'object'->>'id', activity.data->>'object') = objects.data->>'id'; --- Fetch the author's AS2 following collection. - SELECT COALESCE(author.follower_address, '') INTO author_fa FROM users WHERE users.ap_id = activity.actor; + SELECT COALESCE(users.follower_address, '') INTO author_fa FROM users WHERE users.ap_id = activity.actor; --- Prepare valid recipients array. valid_recipients := ARRAY[actor, public]; From 64495b96e9159b88ef0b4a6486e60b54722664fb Mon Sep 17 00:00:00 2001 From: Aaron Tinio Date: Wed, 15 May 2019 06:13:47 +0800 Subject: [PATCH 21/40] Add test for blocked domains --- test/web/activity_pub/activity_pub_test.exs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 0f90aa1ac..442120ffd 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -462,6 +462,20 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do refute Enum.member?(activities, activity_three.id) end + test "doesn't return activities from blocked domains" do + domain = "dogwhistle.zone" + domain_user = insert(:user, %{ap_id: "https://#{domain}/@pundit"}) + note = insert(:note, %{data: %{"actor" => domain_user.ap_id}}) + activity = insert(:note_activity, %{note: note}) + user = insert(:user) + {:ok, user} = User.block_domain(user, domain) + + activities = + ActivityPub.fetch_activities([], %{"blocking_user" => user, "skip_preload" => true}) + + refute activity in activities + end + test "doesn't return muted activities" do activity_one = insert(:note_activity) activity_two = insert(:note_activity) From 2b6119dfbf47f118dfb102fc09f6450e59ccf15e Mon Sep 17 00:00:00 2001 From: Aaron Tinio Date: Wed, 15 May 2019 07:59:24 +0800 Subject: [PATCH 22/40] Restrict reblogs of activities from blocked domains --- lib/pleroma/web/activity_pub/activity_pub.ex | 11 ++++++++++- test/web/activity_pub/activity_pub_test.exs | 9 +++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 233fee4fa..6087a1cdc 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -752,6 +752,14 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do from( activity in query, + inner_join: o in Object, + on: + fragment( + "(?->>'id') = COALESCE(?->'object'->>'id', ?->>'object')", + o.data, + activity.data, + activity.data + ), where: fragment("not (? = ANY(?))", activity.actor, ^blocks), where: fragment("not (? && ?)", activity.recipients, ^blocks), where: @@ -761,7 +769,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do activity.data, ^blocks ), - where: fragment("not (split_part(?, '/', 3) = ANY(?))", activity.actor, ^domain_blocks) + where: fragment("not (split_part(?, '/', 3) = ANY(?))", activity.actor, ^domain_blocks), + where: fragment("not (split_part(?->>'actor', '/', 3) = ANY(?))", o.data, ^domain_blocks) ) end diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 442120ffd..dfee93f67 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -474,6 +474,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do ActivityPub.fetch_activities([], %{"blocking_user" => user, "skip_preload" => true}) refute activity in activities + + followed_user = insert(:user) + ActivityPub.follow(user, followed_user) + {:ok, repeat_activity, _} = CommonAPI.repeat(activity.id, followed_user) + + activities = + ActivityPub.fetch_activities([], %{"blocking_user" => user, "skip_preload" => true}) + + refute repeat_activity in activities end test "doesn't return muted activities" do From 793f1834d2bc37ba3d4ba11fb669078950994adb Mon Sep 17 00:00:00 2001 From: Aaron Tinio Date: Thu, 16 May 2019 06:25:14 +0800 Subject: [PATCH 23/40] Use named binding to conditionally join object --- lib/pleroma/activity.ex | 19 +++++++++++-------- lib/pleroma/web/activity_pub/activity_pub.ex | 13 ++++--------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 4a0919478..3fd4003f8 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -60,21 +60,24 @@ defmodule Pleroma.Activity do timestamps() end - def with_preloaded_object(query) do - query - |> join( - :inner, - [activity], - o in Object, + def with_joined_object(query) do + join(query, :inner, [activity], o in Object, on: fragment( "(?->>'id') = COALESCE(?->'object'->>'id', ?->>'object')", o.data, activity.data, activity.data - ) + ), + as: :object ) - |> preload([activity, object], object: object) + end + + def with_preloaded_object(query) do + query + |> has_named_binding?(:object) + |> if(do: query, else: with_joined_object(query)) + |> preload([activity, object: object], object: object) end def with_preloaded_bookmark(query, %User{} = user) do diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 6087a1cdc..2fd073d3a 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -750,16 +750,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do blocks = info.blocks || [] domain_blocks = info.domain_blocks || [] + query = + if has_named_binding?(query, :object), do: query, else: Activity.with_joined_object(query) + from( - activity in query, - inner_join: o in Object, - on: - fragment( - "(?->>'id') = COALESCE(?->'object'->>'id', ?->>'object')", - o.data, - activity.data, - activity.data - ), + [activity, object: o] in query, where: fragment("not (? = ANY(?))", activity.actor, ^blocks), where: fragment("not (? && ?)", activity.recipients, ^blocks), where: From aa11fa4864bf6ebdd4acfdba837675abbdcffc83 Mon Sep 17 00:00:00 2001 From: Alex S Date: Thu, 16 May 2019 12:49:40 +0700 Subject: [PATCH 24/40] add report uri and report to --- CHANGELOG.md | 3 +- config/test.exs | 2 + docs/config.md | 3 +- lib/pleroma/plugs/http_security_plug.ex | 31 ++++-- test/plugs/http_security_plug_test.exs | 133 ++++++++++++++---------- 5 files changed, 107 insertions(+), 65 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b0e849285..ea1f29304 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Configuration: `fetch_initial_posts` option - Configuration: `notify_email` option - Configuration: Media proxy `whitelist` option +- Configuration: `report_uri` option - Pleroma API: User subscriptions - Pleroma API: Healthcheck endpoint - Admin API: Endpoints for listing/revoking invite tokens @@ -98,7 +99,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Make `irreversible` field default to `false` [`POST /api/v1/filters`] ## Removed -- Configuration: `config :pleroma, :fe` in favor of the more flexible `config :pleroma, :frontend_configurations` +- Configuration: `config :pleroma, :fe` in favor of the more flexible `config :pleroma, :frontend_configurations` ## [0.9.9999] - 2019-04-05 ### Security diff --git a/config/test.exs b/config/test.exs index a0c90c371..40db66170 100644 --- a/config/test.exs +++ b/config/test.exs @@ -61,6 +61,8 @@ config :pleroma, Pleroma.ScheduledActivity, config :pleroma, :app_account_creation, max_requests: 5 +config :pleroma, :http_security, report_uri: "https://endpoint.com" + try do import_config "test.secret.exs" rescue diff --git a/docs/config.md b/docs/config.md index 470f71b7c..c2af5c012 100644 --- a/docs/config.md +++ b/docs/config.md @@ -286,7 +286,8 @@ This will make Pleroma listen on `127.0.0.1` port `8080` and generate urls start * ``sts``: Whether to additionally send a `Strict-Transport-Security` header * ``sts_max_age``: The maximum age for the `Strict-Transport-Security` header if sent * ``ct_max_age``: The maximum age for the `Expect-CT` header if sent -* ``referrer_policy``: The referrer policy to use, either `"same-origin"` or `"no-referrer"`. +* ``referrer_policy``: The referrer policy to use, either `"same-origin"` or `"no-referrer"` +* ``report_uri``: Adds the specified url to `report-uri` and `report-to` group in CSP header. ## :mrf_user_allowlist diff --git a/lib/pleroma/plugs/http_security_plug.ex b/lib/pleroma/plugs/http_security_plug.ex index a476f1d49..485ddfbc7 100644 --- a/lib/pleroma/plugs/http_security_plug.ex +++ b/lib/pleroma/plugs/http_security_plug.ex @@ -20,8 +20,9 @@ defmodule Pleroma.Plugs.HTTPSecurityPlug do defp headers do referrer_policy = Config.get([:http_security, :referrer_policy]) + report_uri = Config.get([:http_security, :report_uri]) - [ + headers = [ {"x-xss-protection", "1; mode=block"}, {"x-permitted-cross-domain-policies", "none"}, {"x-frame-options", "DENY"}, @@ -30,12 +31,27 @@ defmodule Pleroma.Plugs.HTTPSecurityPlug do {"x-download-options", "noopen"}, {"content-security-policy", csp_string() <> ";"} ] + + if report_uri do + report_group = %{ + "group" => "csp-endpoint", + "max-age" => 10_886_400, + "endpoints" => [ + %{"url" => report_uri} + ] + } + + headers ++ [{"reply-to", Jason.encode!(report_group)}] + else + headers + end end defp csp_string do scheme = Config.get([Pleroma.Web.Endpoint, :url])[:scheme] static_url = Pleroma.Web.Endpoint.static_url() websocket_url = Pleroma.Web.Endpoint.websocket_url() + report_uri = Config.get([:http_security, :report_uri]) connect_src = "connect-src 'self' #{static_url} #{websocket_url}" @@ -53,7 +69,7 @@ defmodule Pleroma.Plugs.HTTPSecurityPlug do "script-src 'self'" end - [ + main_part = [ "default-src 'none'", "base-uri 'self'", "frame-ancestors 'none'", @@ -63,11 +79,14 @@ defmodule Pleroma.Plugs.HTTPSecurityPlug do "font-src 'self'", "manifest-src 'self'", connect_src, - script_src, - if scheme == "https" do - "upgrade-insecure-requests" - end + script_src ] + + report = if report_uri, do: ["report-uri #{report_uri}; report-to csp-endpoint"], else: [] + + insecure = if scheme == "https", do: ["upgrade-insecure-requests"], else: [] + + (main_part ++ report ++ insecure) |> Enum.join("; ") end diff --git a/test/plugs/http_security_plug_test.exs b/test/plugs/http_security_plug_test.exs index 0cbb7e4b1..7dfd50c1f 100644 --- a/test/plugs/http_security_plug_test.exs +++ b/test/plugs/http_security_plug_test.exs @@ -7,28 +7,89 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlugTest do alias Pleroma.Config alias Plug.Conn - test "it sends CSP headers when enabled", %{conn: conn} do - Config.put([:http_security, :enabled], true) + describe "http security enabled" do + setup do + enabled = Config.get([:http_securiy, :enabled]) - conn = - conn - |> get("/api/v1/instance") + Config.put([:http_security, :enabled], true) - refute Conn.get_resp_header(conn, "x-xss-protection") == [] - refute Conn.get_resp_header(conn, "x-permitted-cross-domain-policies") == [] - refute Conn.get_resp_header(conn, "x-frame-options") == [] - refute Conn.get_resp_header(conn, "x-content-type-options") == [] - refute Conn.get_resp_header(conn, "x-download-options") == [] - refute Conn.get_resp_header(conn, "referrer-policy") == [] - refute Conn.get_resp_header(conn, "content-security-policy") == [] + on_exit(fn -> + Config.put([:http_security, :enabled], enabled) + end) + + :ok + end + + test "it sends CSP headers when enabled", %{conn: conn} do + conn = get(conn, "/api/v1/instance") + + refute Conn.get_resp_header(conn, "x-xss-protection") == [] + refute Conn.get_resp_header(conn, "x-permitted-cross-domain-policies") == [] + refute Conn.get_resp_header(conn, "x-frame-options") == [] + refute Conn.get_resp_header(conn, "x-content-type-options") == [] + refute Conn.get_resp_header(conn, "x-download-options") == [] + refute Conn.get_resp_header(conn, "referrer-policy") == [] + refute Conn.get_resp_header(conn, "content-security-policy") == [] + end + + test "it sends STS headers when enabled", %{conn: conn} do + Config.put([:http_security, :sts], true) + + conn = get(conn, "/api/v1/instance") + + refute Conn.get_resp_header(conn, "strict-transport-security") == [] + refute Conn.get_resp_header(conn, "expect-ct") == [] + end + + test "it does not send STS headers when disabled", %{conn: conn} do + Config.put([:http_security, :sts], false) + + conn = get(conn, "/api/v1/instance") + + assert Conn.get_resp_header(conn, "strict-transport-security") == [] + assert Conn.get_resp_header(conn, "expect-ct") == [] + end + + test "referrer-policy header reflects configured value", %{conn: conn} do + conn = get(conn, "/api/v1/instance") + + assert Conn.get_resp_header(conn, "referrer-policy") == ["same-origin"] + + Config.put([:http_security, :referrer_policy], "no-referrer") + + conn = + build_conn() + |> get("/api/v1/instance") + + assert Conn.get_resp_header(conn, "referrer-policy") == ["no-referrer"] + end + + test "it sends `report-to` & `report-uri` CSP response headers" do + conn = + build_conn() + |> get("/api/v1/instance") + + [csp] = Conn.get_resp_header(conn, "content-security-policy") + + assert csp =~ ~r|report-uri https://endpoint.com; report-to csp-endpoint;| + + [reply_to] = Conn.get_resp_header(conn, "reply-to") + + assert reply_to == + "{\"endpoints\":[{\"url\":\"https://endpoint.com\"}],\"group\":\"csp-endpoint\",\"max-age\":10886400}" + end end test "it does not send CSP headers when disabled", %{conn: conn} do + enabled = Config.get([:http_securiy, :enabled]) + Config.put([:http_security, :enabled], false) - conn = - conn - |> get("/api/v1/instance") + on_exit(fn -> + Config.put([:http_security, :enabled], enabled) + end) + + conn = get(conn, "/api/v1/instance") assert Conn.get_resp_header(conn, "x-xss-protection") == [] assert Conn.get_resp_header(conn, "x-permitted-cross-domain-policies") == [] @@ -38,46 +99,4 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlugTest do assert Conn.get_resp_header(conn, "referrer-policy") == [] assert Conn.get_resp_header(conn, "content-security-policy") == [] end - - test "it sends STS headers when enabled", %{conn: conn} do - Config.put([:http_security, :enabled], true) - Config.put([:http_security, :sts], true) - - conn = - conn - |> get("/api/v1/instance") - - refute Conn.get_resp_header(conn, "strict-transport-security") == [] - refute Conn.get_resp_header(conn, "expect-ct") == [] - end - - test "it does not send STS headers when disabled", %{conn: conn} do - Config.put([:http_security, :enabled], true) - Config.put([:http_security, :sts], false) - - conn = - conn - |> get("/api/v1/instance") - - assert Conn.get_resp_header(conn, "strict-transport-security") == [] - assert Conn.get_resp_header(conn, "expect-ct") == [] - end - - test "referrer-policy header reflects configured value", %{conn: conn} do - Config.put([:http_security, :enabled], true) - - conn = - conn - |> get("/api/v1/instance") - - assert Conn.get_resp_header(conn, "referrer-policy") == ["same-origin"] - - Config.put([:http_security, :referrer_policy], "no-referrer") - - conn = - build_conn() - |> get("/api/v1/instance") - - assert Conn.get_resp_header(conn, "referrer-policy") == ["no-referrer"] - end end From 8355d0607e986f5a2cc5addfb2bcb35f799f10cb Mon Sep 17 00:00:00 2001 From: Alex S Date: Thu, 16 May 2019 17:03:12 +0700 Subject: [PATCH 25/40] trailing spaces --- docs/introduction.md | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/docs/introduction.md b/docs/introduction.md index 4af0747fe..045dc7c05 100644 --- a/docs/introduction.md +++ b/docs/introduction.md @@ -1,30 +1,30 @@ # Introduction to Pleroma ## What is Pleroma? -Pleroma is a federated social networking platform, compatible with GNU social, Mastodon and other OStatus and ActivityPub implementations. It is free software licensed under the AGPLv3. -It actually consists of two components: a backend, named simply Pleroma, and a user-facing frontend, named Pleroma-FE. It also includes the Mastodon frontend, if that's your thing. -It's part of what we call the fediverse, a federated network of instances which speak common protocols and can communicate with each other. +Pleroma is a federated social networking platform, compatible with GNU social, Mastodon and other OStatus and ActivityPub implementations. It is free software licensed under the AGPLv3. +It actually consists of two components: a backend, named simply Pleroma, and a user-facing frontend, named Pleroma-FE. It also includes the Mastodon frontend, if that's your thing. +It's part of what we call the fediverse, a federated network of instances which speak common protocols and can communicate with each other. One account on a instance is enough to talk to the entire fediverse! - + ## How can I use it? -Pleroma instances are already widely deployed, a list can be found here: +Pleroma instances are already widely deployed, a list can be found here: http://distsn.org/pleroma-instances.html -If you don't feel like joining an existing instance, but instead prefer to deploy your own instance, that's easy too! -Installation instructions can be found here: +If you don't feel like joining an existing instance, but instead prefer to deploy your own instance, that's easy too! +Installation instructions can be found here: [main Pleroma wiki](/) - + ## I got an account, now what? -Great! Now you can explore the fediverse! -- Open the login page for your Pleroma instance (for ex. https://pleroma.soykaf.com) and login with your username and password. -(If you don't have one yet, click on Register) :slightly_smiling_face: +Great! Now you can explore the fediverse! +- Open the login page for your Pleroma instance (for ex. https://pleroma.soykaf.com) and login with your username and password. +(If you don't have one yet, click on Register) :slightly_smiling_face: At this point you will have two columns in front of you. ### Left column -- first block: here you can see your avatar, your nickname a bio, and statistics (Statuses, Following, Followers). -Under that you have a text form which allows you to post new statuses. The icon on the left is for uploading media files and attach them to your post. The number under the text form is a character counter, every instance can have a different character limit (the default is 5000). -If you want to mention someone, type @ + name of the person. A drop-down menu will help you in finding the right person. :slight_smile: +- first block: here you can see your avatar, your nickname a bio, and statistics (Statuses, Following, Followers). +Under that you have a text form which allows you to post new statuses. The icon on the left is for uploading media files and attach them to your post. The number under the text form is a character counter, every instance can have a different character limit (the default is 5000). +If you want to mention someone, type @ + name of the person. A drop-down menu will help you in finding the right person. :slight_smile: To post your status, simply press Submit. - second block: Here you can switch between the different timelines: @@ -38,7 +38,7 @@ To post your status, simply press Submit. - fourth block: This is the Notifications block, here you will get notified whenever somebody mentions you, follows you, repeats or favorites one of your statuses. ### Right column -This is where the interesting stuff happens! :slight_smile: +This is where the interesting stuff happens! :slight_smile: Depending on the timeline you will see different statuses, but each status has a standard structure: - Icon + name + link to profile. An optional left-arrow if it's a reply to another status (hovering will reveal the replied-to status). - A + button on the right allows you to Expand/Collapse an entire discussion thread. It also updates in realtime! @@ -47,9 +47,9 @@ Depending on the timeline you will see different statuses, but each status has a - Four buttons (left to right): Reply, Repeat, Favorite, Delete. ## Mastodon interface -If the Pleroma interface isn't your thing, or you're just trying something new but you want to keep using the familiar Mastodon interface, we got that too! :smile: -Just add a "/web" after your instance url (for ex. https://pleroma.soycaf.com/web) and you'll end on the Mastodon web interface, but with a Pleroma backend! MAGIC! :fireworks: -For more information on the Mastodon interface, please look here: +If the Pleroma interface isn't your thing, or you're just trying something new but you want to keep using the familiar Mastodon interface, we got that too! :smile: +Just add a "/web" after your instance url (for ex. https://pleroma.soycaf.com/web) and you'll end on the Mastodon web interface, but with a Pleroma backend! MAGIC! :fireworks: +For more information on the Mastodon interface, please look here: https://github.com/tootsuite/documentation/blob/master/Using-Mastodon/User-guide.md Remember, what you see is only the frontend part of Mastodon, the backend is still Pleroma. From 4711d8151c8cb7ce2024f378e17b3a113a3aac1c Mon Sep 17 00:00:00 2001 From: Alex S Date: Thu, 16 May 2019 17:35:07 +0700 Subject: [PATCH 26/40] try this --- test/web/common_api/common_api_test.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs index 8d4f401ee..b0c441a37 100644 --- a/test/web/common_api/common_api_test.exs +++ b/test/web/common_api/common_api_test.exs @@ -279,14 +279,14 @@ defmodule Pleroma.Web.CommonAPITest do test "add a reblog mute", %{muter: muter, muted: muted} do {:ok, muter} = CommonAPI.hide_reblogs(muter, muted) - assert Pleroma.User.showing_reblogs?(muter, muted) == false + assert User.showing_reblogs?(muter, muted) == false end test "remove a reblog mute", %{muter: muter, muted: muted} do {:ok, muter} = CommonAPI.hide_reblogs(muter, muted) {:ok, muter} = CommonAPI.show_reblogs(muter, muted) - assert Pleroma.User.showing_reblogs?(muter, muted) == true + assert User.showing_reblogs?(muter, muted) == true end end end From 73ae58fdfaf0b9dc9630929b0b84ae3b6083684a Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 16 May 2019 13:23:41 +0000 Subject: [PATCH 27/40] Feature/896 toggling confirmation --- CHANGELOG.md | 3 ++- lib/mix/tasks/pleroma/user.ex | 19 +++++++++++++++++++ lib/pleroma/user.ex | 13 +++++++++++++ lib/pleroma/user/info.ex | 2 +- test/tasks/user_test.exs | 27 +++++++++++++++++++++++++++ test/user_test.exs | 18 ++++++++++++++++++ 6 files changed, 80 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b0e849285..b3d8e1e0c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - [Prometheus](https://prometheus.io/) metrics - Support for Mastodon's remote interaction - Mix Tasks: `mix pleroma.database remove_embedded_objects` +- Mix Tasks: `mix pleroma.user toggle_confirmed` - Federation: Support for reports - Configuration: `safe_dm_mentions` option - Configuration: `link_name` option @@ -98,7 +99,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Make `irreversible` field default to `false` [`POST /api/v1/filters`] ## Removed -- Configuration: `config :pleroma, :fe` in favor of the more flexible `config :pleroma, :frontend_configurations` +- Configuration: `config :pleroma, :fe` in favor of the more flexible `config :pleroma, :frontend_configurations` ## [0.9.9999] - 2019-04-05 ### Security diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex index d130ff8c9..25fc40ea7 100644 --- a/lib/mix/tasks/pleroma/user.ex +++ b/lib/mix/tasks/pleroma/user.ex @@ -77,6 +77,10 @@ defmodule Mix.Tasks.Pleroma.User do ## Delete tags from a user. mix pleroma.user untag NICKNAME TAGS + + ## Toggle confirmation of the user's account. + + mix pleroma.user toggle_confirmed NICKNAME """ def run(["new", nickname, email | rest]) do {options, [], []} = @@ -388,6 +392,21 @@ defmodule Mix.Tasks.Pleroma.User do end end + def run(["toggle_confirmed", nickname]) do + Common.start_pleroma() + + with %User{} = user <- User.get_cached_by_nickname(nickname) do + {:ok, user} = User.toggle_confirmation(user) + + message = if user.info.confirmation_pending, do: "needs", else: "doesn't need" + + Mix.shell().info("#{nickname} #{message} confirmation.") + else + _ -> + Mix.shell().error("No local user #{nickname}") + end + end + defp set_moderator(user, value) do info_cng = User.Info.admin_api_update(user.info, %{is_moderator: value}) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index c6a562a61..1aa966dfc 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1378,4 +1378,17 @@ defmodule Pleroma.User do def showing_reblogs?(%User{} = user, %User{} = target) do target.ap_id not in user.info.muted_reblogs end + + @spec toggle_confirmation(User.t()) :: {:ok, User.t()} | {:error, Changeset.t()} + def toggle_confirmation(%User{} = user) do + need_confirmation? = !user.info.confirmation_pending + + info_changeset = + User.Info.confirmation_changeset(user.info, need_confirmation: need_confirmation?) + + user + |> change() + |> put_embed(:info, info_changeset) + |> update_and_set_cache() + end end diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index 5a50ee639..5f0cefc00 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -212,7 +212,7 @@ defmodule Pleroma.User.Info do ]) end - @spec confirmation_changeset(Info.t(), keyword()) :: Ecto.Changerset.t() + @spec confirmation_changeset(Info.t(), keyword()) :: Changeset.t() def confirmation_changeset(info, opts) do need_confirmation? = Keyword.get(opts, :need_confirmation) diff --git a/test/tasks/user_test.exs b/test/tasks/user_test.exs index eaf4ecf84..1f97740be 100644 --- a/test/tasks/user_test.exs +++ b/test/tasks/user_test.exs @@ -338,4 +338,31 @@ defmodule Mix.Tasks.Pleroma.UserTest do assert message == "User #{nickname} statuses deleted." end end + + describe "running toggle_confirmed" do + test "user is confirmed" do + %{id: id, nickname: nickname} = insert(:user, info: %{confirmation_pending: false}) + + assert :ok = Mix.Tasks.Pleroma.User.run(["toggle_confirmed", nickname]) + assert_received {:mix_shell, :info, [message]} + assert message == "#{nickname} needs confirmation." + + user = Repo.get(User, id) + assert user.info.confirmation_pending + assert user.info.confirmation_token + end + + test "user is not confirmed" do + %{id: id, nickname: nickname} = + insert(:user, info: %{confirmation_pending: true, confirmation_token: "some token"}) + + assert :ok = Mix.Tasks.Pleroma.User.run(["toggle_confirmed", nickname]) + assert_received {:mix_shell, :info, [message]} + assert message == "#{nickname} doesn't need confirmation." + + user = Repo.get(User, id) + refute user.info.confirmation_pending + refute user.info.confirmation_token + end + end end diff --git a/test/user_test.exs b/test/user_test.exs index 0b65e89e9..f256396c0 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -1204,4 +1204,22 @@ defmodule Pleroma.UserTest do assert Map.get(user_show, "followers_count") == 2 end + + describe "toggle_confirmation/1" do + test "if user is confirmed" do + user = insert(:user, info: %{confirmation_pending: false}) + {:ok, user} = User.toggle_confirmation(user) + + assert user.info.confirmation_pending + assert user.info.confirmation_token + end + + test "if user is unconfirmed" do + user = insert(:user, info: %{confirmation_pending: true, confirmation_token: "some token"}) + {:ok, user} = User.toggle_confirmation(user) + + refute user.info.confirmation_pending + refute user.info.confirmation_token + end + end end From efa61c161085fd24f7e85ccf7f32ef823e335d7b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 16 May 2019 13:14:48 -0500 Subject: [PATCH 28/40] Move to pleroma.database mix task --- lib/mix/tasks/pleroma/conversations.ex | 23 ----------------------- lib/mix/tasks/pleroma/database.ex | 11 +++++++++++ 2 files changed, 11 insertions(+), 23 deletions(-) delete mode 100644 lib/mix/tasks/pleroma/conversations.ex diff --git a/lib/mix/tasks/pleroma/conversations.ex b/lib/mix/tasks/pleroma/conversations.ex deleted file mode 100644 index b52b9921a..000000000 --- a/lib/mix/tasks/pleroma/conversations.ex +++ /dev/null @@ -1,23 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Mix.Tasks.Pleroma.Conversations do - use Mix.Task - alias Mix.Tasks.Pleroma.Common - alias Pleroma.Conversation - - @shortdoc "Manages Pleroma conversations." - @moduledoc """ - Manages Pleroma conversations. - - ## Create a conversation for all existing DMs. Can be safely re-run. - - mix pleroma.conversations bump_all - - """ - def run(["bump_all"]) do - Common.start_pleroma() - Conversation.bump_for_all_activities() - end -end diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index ab9a3a7ff..42753a1a4 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -4,6 +4,7 @@ defmodule Mix.Tasks.Pleroma.Database do alias Mix.Tasks.Pleroma.Common + alias Pleroma.Conversation require Logger use Mix.Task @@ -19,6 +20,11 @@ defmodule Mix.Tasks.Pleroma.Database do Options: - `--vacuum` - run `VACUUM FULL` after the embedded objects are replaced with their references + + ## Create a conversation for all existing DMs. Can be safely re-run. + + mix pleroma.database bump_all_conversations + """ def run(["remove_embedded_objects" | args]) do {options, [], []} = @@ -48,4 +54,9 @@ defmodule Mix.Tasks.Pleroma.Database do ) end end + + def run(["bump_all_conversations"]) do + Common.start_pleroma() + Conversation.bump_for_all_activities() + end end From cd127d2fa43e6b1ecd7fadfc1d6293b2a432be18 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 16 May 2019 13:17:09 -0500 Subject: [PATCH 29/40] Update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3d8e1e0c..fe6ab002c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - A [job queue](https://git.pleroma.social/pleroma/pleroma_job_queue) for federation, emails, web push, etc. - [Prometheus](https://prometheus.io/) metrics - Support for Mastodon's remote interaction +- Mix Tasks: `mix pleroma.database bump_all_conversations` - Mix Tasks: `mix pleroma.database remove_embedded_objects` - Mix Tasks: `mix pleroma.user toggle_confirmed` - Federation: Support for reports From e2b3a27204ca511a2e455a1151fdea36fdc0e53d Mon Sep 17 00:00:00 2001 From: Sergey Suprunenko Date: Thu, 16 May 2019 19:09:18 +0000 Subject: [PATCH 30/40] Add Reports to Admin API --- CHANGELOG.md | 2 + docs/api/admin_api.md | 289 +++++++++++++++- lib/pleroma/activity.ex | 2 +- lib/pleroma/emails/admin_email.ex | 2 +- lib/pleroma/web/activity_pub/activity_pub.ex | 7 + lib/pleroma/web/activity_pub/utils.ex | 78 ++++- .../web/admin_api/admin_api_controller.ex | 81 +++++ .../web/admin_api/views/report_view.ex | 41 +++ lib/pleroma/web/common_api/common_api.ex | 57 +++ lib/pleroma/web/common_api/utils.ex | 10 +- lib/pleroma/web/router.ex | 8 + ...511191044_set_default_state_to_reports.exs | 19 + test/support/factory.ex | 5 +- .../admin_api/admin_api_controller_test.exs | 327 ++++++++++++++++++ test/web/common_api/common_api_test.exs | 33 +- .../mastodon_api_controller_test.exs | 4 +- 16 files changed, 945 insertions(+), 20 deletions(-) create mode 100644 lib/pleroma/web/admin_api/views/report_view.ex create mode 100644 priv/repo/migrations/20190511191044_set_default_state_to_reports.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index fe6ab002c..dc09d4f6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Admin API: Endpoints for listing/revoking invite tokens - Admin API: Endpoints for making users follow/unfollow each other - Admin API: added filters (role, tags, email, name) for users endpoint +- Admin API: Endpoints for managing reports +- Admin API: Endpoints for deleting and changing the scope of individual reported statuses - AdminFE: initial release with basic user management accessible at /pleroma/admin/ - Mastodon API: [Scheduled statuses](https://docs.joinmastodon.org/api/rest/scheduled-statuses/) - Mastodon API: `/api/v1/notifications/destroy_multiple` (glitch-soc extension) diff --git a/docs/api/admin_api.md b/docs/api/admin_api.md index 59578f8d1..b45c5e285 100644 --- a/docs/api/admin_api.md +++ b/docs/api/admin_api.md @@ -24,7 +24,7 @@ Authentication is required and the user must be an admin. - Example: `https://mypleroma.org/api/pleroma/admin/users?query=john&filters=local,active&page=1&page_size=10&tags[]=some_tag&tags[]=another_tag&name=display_name&email=email@example.com` - Response: -```JSON +```json { "page_size": integer, "count": integer, @@ -92,7 +92,7 @@ Authentication is required and the user must be an admin. - `nickname` - Response: User’s object -```JSON +```json { "deactivated": bool, "id": integer, @@ -124,7 +124,7 @@ Authentication is required and the user must be an admin. - Params: none - Response: -```JSON +```json { "is_moderator": bool, "is_admin": bool @@ -141,7 +141,7 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret - Params: none - Response: -```JSON +```json { "is_moderator": bool, "is_admin": bool @@ -223,7 +223,7 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret - Params: none - Response: -```JSON +```json { "invites": [ @@ -250,7 +250,7 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret - `token` - Response: -```JSON +```json { "id": integer, "token": string, @@ -280,3 +280,280 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret - Methods: `GET` - Params: none - Response: password reset token (base64 string) + +## `/api/pleroma/admin/reports` +### Get a list of reports +- Method `GET` +- Params: + - `state`: optional, the state of reports. Valid values are `open`, `closed` and `resolved` + - `limit`: optional, the number of records to retrieve + - `since_id`: optional, returns results that are more recent than the specified id + - `max_id`: optional, returns results that are older than the specified id +- Response: + - On failure: 403 Forbidden error `{"error": "error_msg"}` when requested by anonymous or non-admin + - On success: JSON, returns a list of reports, where: + - `account`: the user who has been reported + - `actor`: the user who has sent the report + - `statuses`: list of statuses that have been included to the report + +```json +{ + "reports": [ + { + "account": { + "acct": "user", + "avatar": "https://pleroma.example.org/images/avi.png", + "avatar_static": "https://pleroma.example.org/images/avi.png", + "bot": false, + "created_at": "2019-04-23T17:32:04.000Z", + "display_name": "User", + "emojis": [], + "fields": [], + "followers_count": 1, + "following_count": 1, + "header": "https://pleroma.example.org/images/banner.png", + "header_static": "https://pleroma.example.org/images/banner.png", + "id": "9i6dAJqSGSKMzLG2Lo", + "locked": false, + "note": "", + "pleroma": { + "confirmation_pending": false, + "hide_favorites": true, + "hide_followers": false, + "hide_follows": false, + "is_admin": false, + "is_moderator": false, + "relationship": {}, + "tags": [] + }, + "source": { + "note": "", + "pleroma": {}, + "sensitive": false + }, + "statuses_count": 3, + "url": "https://pleroma.example.org/users/user", + "username": "user" + }, + "actor": { + "acct": "lain", + "avatar": "https://pleroma.example.org/images/avi.png", + "avatar_static": "https://pleroma.example.org/images/avi.png", + "bot": false, + "created_at": "2019-03-28T17:36:03.000Z", + "display_name": "Roger Braun", + "emojis": [], + "fields": [], + "followers_count": 1, + "following_count": 1, + "header": "https://pleroma.example.org/images/banner.png", + "header_static": "https://pleroma.example.org/images/banner.png", + "id": "9hEkA5JsvAdlSrocam", + "locked": false, + "note": "", + "pleroma": { + "confirmation_pending": false, + "hide_favorites": false, + "hide_followers": false, + "hide_follows": false, + "is_admin": false, + "is_moderator": false, + "relationship": {}, + "tags": [] + }, + "source": { + "note": "", + "pleroma": {}, + "sensitive": false + }, + "statuses_count": 1, + "url": "https://pleroma.example.org/users/lain", + "username": "lain" + }, + "content": "Please delete it", + "created_at": "2019-04-29T19:48:15.000Z", + "id": "9iJGOv1j8hxuw19bcm", + "state": "open", + "statuses": [ + { + "account": { ... }, + "application": { + "name": "Web", + "website": null + }, + "bookmarked": false, + "card": null, + "content": "@lain click on my link https://www.google.com/", + "created_at": "2019-04-23T19:15:47.000Z", + "emojis": [], + "favourited": false, + "favourites_count": 0, + "id": "9i6mQ9uVrrOmOime8m", + "in_reply_to_account_id": null, + "in_reply_to_id": null, + "language": null, + "media_attachments": [], + "mentions": [ + { + "acct": "lain", + "id": "9hEkA5JsvAdlSrocam", + "url": "https://pleroma.example.org/users/lain", + "username": "lain" + }, + { + "acct": "user", + "id": "9i6dAJqSGSKMzLG2Lo", + "url": "https://pleroma.example.org/users/user", + "username": "user" + } + ], + "muted": false, + "pinned": false, + "pleroma": { + "content": { + "text/plain": "@lain click on my link https://www.google.com/" + }, + "conversation_id": 28, + "in_reply_to_account_acct": null, + "local": true, + "spoiler_text": { + "text/plain": "" + } + }, + "reblog": null, + "reblogged": false, + "reblogs_count": 0, + "replies_count": 0, + "sensitive": false, + "spoiler_text": "", + "tags": [], + "uri": "https://pleroma.example.org/objects/8717b90f-8e09-4b58-97b0-e3305472b396", + "url": "https://pleroma.example.org/notice/9i6mQ9uVrrOmOime8m", + "visibility": "direct" + } + ] + } + ] +} +``` + +## `/api/pleroma/admin/reports/:id` +### Get an individual report +- Method `GET` +- Params: + - `id` +- Response: + - On failure: + - 403 Forbidden `{"error": "error_msg"}` + - 404 Not Found `"Not found"` + - On success: JSON, Report object (see above) + +## `/api/pleroma/admin/reports/:id` +### Change the state of the report +- Method `PUT` +- Params: + - `id` + - `state`: required, the new state. Valid values are `open`, `closed` and `resolved` +- Response: + - On failure: + - 400 Bad Request `"Unsupported state"` + - 403 Forbidden `{"error": "error_msg"}` + - 404 Not Found `"Not found"` + - On success: JSON, Report object (see above) + +## `/api/pleroma/admin/reports/:id/respond` +### Respond to a report +- Method `POST` +- Params: + - `id` + - `status`: required, the message +- Response: + - On failure: + - 400 Bad Request `"Invalid parameters"` when `status` is missing + - 403 Forbidden `{"error": "error_msg"}` + - 404 Not Found `"Not found"` + - On success: JSON, created Mastodon Status entity + +```json +{ + "account": { ... }, + "application": { + "name": "Web", + "website": null + }, + "bookmarked": false, + "card": null, + "content": "Your claim is going to be closed", + "created_at": "2019-05-11T17:13:03.000Z", + "emojis": [], + "favourited": false, + "favourites_count": 0, + "id": "9ihuiSL1405I65TmEq", + "in_reply_to_account_id": null, + "in_reply_to_id": null, + "language": null, + "media_attachments": [], + "mentions": [ + { + "acct": "user", + "id": "9i6dAJqSGSKMzLG2Lo", + "url": "https://pleroma.example.org/users/user", + "username": "user" + }, + { + "acct": "admin", + "id": "9hEkA5JsvAdlSrocam", + "url": "https://pleroma.example.org/users/admin", + "username": "admin" + } + ], + "muted": false, + "pinned": false, + "pleroma": { + "content": { + "text/plain": "Your claim is going to be closed" + }, + "conversation_id": 35, + "in_reply_to_account_acct": null, + "local": true, + "spoiler_text": { + "text/plain": "" + } + }, + "reblog": null, + "reblogged": false, + "reblogs_count": 0, + "replies_count": 0, + "sensitive": false, + "spoiler_text": "", + "tags": [], + "uri": "https://pleroma.example.org/objects/cab0836d-9814-46cd-a0ea-529da9db5fcb", + "url": "https://pleroma.example.org/notice/9ihuiSL1405I65TmEq", + "visibility": "direct" +} +``` + +## `/api/pleroma/admin/statuses/:id` +### Change the scope of an individual reported status +- Method `PUT` +- Params: + - `id` + - `sensitive`: optional, valid values are `true` or `false` + - `visibility`: optional, valid values are `public`, `private` and `unlisted` +- Response: + - On failure: + - 400 Bad Request `"Unsupported visibility"` + - 403 Forbidden `{"error": "error_msg"}` + - 404 Not Found `"Not found"` + - On success: JSON, Mastodon Status entity + +## `/api/pleroma/admin/statuses/:id` +### Delete an individual reported status +- Method `DELETE` +- Params: + - `id` +- Response: + - On failure: + - 403 Forbidden `{"error": "error_msg"}` + - 404 Not Found `"Not found"` + - On success: 200 OK `{}` diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 3fd4003f8..4e54b15ba 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -111,7 +111,7 @@ defmodule Pleroma.Activity do def change(struct, params \\ %{}) do struct - |> cast(params, [:data]) + |> cast(params, [:data, :recipients]) |> validate_required([:data]) |> unique_constraint(:ap_id, name: :activities_unique_apid_index) end diff --git a/lib/pleroma/emails/admin_email.ex b/lib/pleroma/emails/admin_email.ex index df0f72f96..d0e254362 100644 --- a/lib/pleroma/emails/admin_email.ex +++ b/lib/pleroma/emails/admin_email.ex @@ -29,7 +29,7 @@ defmodule Pleroma.Emails.AdminEmail do end statuses_html = - if length(statuses) > 0 do + if is_list(statuses) && length(statuses) > 0 do statuses_list_html = statuses |> Enum.map(fn diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 6a186efbf..5c3156978 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -703,6 +703,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_type(query, _), do: query + defp restrict_state(query, %{"state" => state}) do + from(activity in query, where: fragment("?->>'state' = ?", activity.data, ^state)) + end + + defp restrict_state(query, _), do: query + defp restrict_favorited_by(query, %{"favorited_by" => ap_id}) do from( activity in query, @@ -855,6 +861,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> restrict_local(opts) |> restrict_actor(opts) |> restrict_type(opts) + |> restrict_state(opts) |> restrict_favorited_by(opts) |> restrict_blocked(opts) |> restrict_muted(opts) diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 236d1b4ac..ca8a0844b 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -20,6 +20,8 @@ defmodule Pleroma.Web.ActivityPub.Utils do require Logger @supported_object_types ["Article", "Note", "Video", "Page"] + @supported_report_states ~w(open closed resolved) + @valid_visibilities ~w(public unlisted private direct) # Some implementations send the actor URI as the actor field, others send the entire actor object, # so figure out what the actor's URI is based on what we have. @@ -670,7 +672,8 @@ defmodule Pleroma.Web.ActivityPub.Utils do "actor" => params.actor.ap_id, "content" => params.content, "object" => object, - "context" => params.context + "context" => params.context, + "state" => "open" } |> Map.merge(additional) end @@ -713,4 +716,77 @@ defmodule Pleroma.Web.ActivityPub.Utils do end end end + + #### Report-related helpers + + def update_report_state(%Activity{} = activity, state) when state in @supported_report_states do + with new_data <- Map.put(activity.data, "state", state), + changeset <- Changeset.change(activity, data: new_data), + {:ok, activity} <- Repo.update(changeset) do + {:ok, activity} + end + end + + def update_report_state(_, _), do: {:error, "Unsupported state"} + + def update_activity_visibility(activity, visibility) when visibility in @valid_visibilities do + [to, cc, recipients] = + activity + |> get_updated_targets(visibility) + |> Enum.map(&Enum.uniq/1) + + object_data = + activity.object.data + |> Map.put("to", to) + |> Map.put("cc", cc) + + {:ok, object} = + activity.object + |> Object.change(%{data: object_data}) + |> Object.update_and_set_cache() + + activity_data = + activity.data + |> Map.put("to", to) + |> Map.put("cc", cc) + + activity + |> Map.put(:object, object) + |> Activity.change(%{data: activity_data, recipients: recipients}) + |> Repo.update() + end + + def update_activity_visibility(_, _), do: {:error, "Unsupported visibility"} + + defp get_updated_targets( + %Activity{data: %{"to" => to} = data, recipients: recipients}, + visibility + ) do + cc = Map.get(data, "cc", []) + follower_address = User.get_cached_by_ap_id(data["actor"]).follower_address + public = "https://www.w3.org/ns/activitystreams#Public" + + case visibility do + "public" -> + to = [public | List.delete(to, follower_address)] + cc = [follower_address | List.delete(cc, public)] + recipients = [public | recipients] + [to, cc, recipients] + + "private" -> + to = [follower_address | List.delete(to, public)] + cc = List.delete(cc, public) + recipients = List.delete(recipients, public) + [to, cc, recipients] + + "unlisted" -> + to = [follower_address | List.delete(to, public)] + cc = [public | List.delete(cc, follower_address)] + recipients = recipients ++ [follower_address, public] + [to, cc, recipients] + + _ -> + [to, cc, recipients] + end + end end diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index e00b33aba..de2a13c01 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -4,11 +4,16 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do use Pleroma.Web, :controller + alias Pleroma.Activity alias Pleroma.User alias Pleroma.UserInviteToken + alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Relay alias Pleroma.Web.AdminAPI.AccountView + alias Pleroma.Web.AdminAPI.ReportView alias Pleroma.Web.AdminAPI.Search + alias Pleroma.Web.CommonAPI + alias Pleroma.Web.MastodonAPI.StatusView import Pleroma.Web.ControllerHelper, only: [json_response: 3] @@ -287,12 +292,88 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do |> json(token.token) end + def list_reports(conn, params) do + params = + params + |> Map.put("type", "Flag") + |> Map.put("skip_preload", true) + + reports = + [] + |> ActivityPub.fetch_activities(params) + |> Enum.reverse() + + conn + |> put_view(ReportView) + |> render("index.json", %{reports: reports}) + end + + def report_show(conn, %{"id" => id}) do + with %Activity{} = report <- Activity.get_by_id(id) do + conn + |> put_view(ReportView) + |> render("show.json", %{report: report}) + else + _ -> {:error, :not_found} + end + end + + def report_update_state(conn, %{"id" => id, "state" => state}) do + with {:ok, report} <- CommonAPI.update_report_state(id, state) do + conn + |> put_view(ReportView) + |> render("show.json", %{report: report}) + end + end + + def report_respond(%{assigns: %{user: user}} = conn, %{"id" => id} = params) do + with false <- is_nil(params["status"]), + %Activity{} <- Activity.get_by_id(id) do + params = + params + |> Map.put("in_reply_to_status_id", id) + |> Map.put("visibility", "direct") + + {:ok, activity} = CommonAPI.post(user, params) + + conn + |> put_view(StatusView) + |> render("status.json", %{activity: activity}) + else + true -> + {:param_cast, nil} + + nil -> + {:error, :not_found} + end + end + + def status_update(conn, %{"id" => id} = params) do + with {:ok, activity} <- CommonAPI.update_activity_scope(id, params) do + conn + |> put_view(StatusView) + |> render("status.json", %{activity: activity}) + end + end + + def status_delete(%{assigns: %{user: user}} = conn, %{"id" => id}) do + with {:ok, %Activity{}} <- CommonAPI.delete(id, user) do + json(conn, %{}) + end + end + def errors(conn, {:error, :not_found}) do conn |> put_status(404) |> json("Not found") end + def errors(conn, {:error, reason}) do + conn + |> put_status(400) + |> json(reason) + end + def errors(conn, {:param_cast, _}) do conn |> put_status(400) diff --git a/lib/pleroma/web/admin_api/views/report_view.ex b/lib/pleroma/web/admin_api/views/report_view.ex new file mode 100644 index 000000000..47a73dc7e --- /dev/null +++ b/lib/pleroma/web/admin_api/views/report_view.ex @@ -0,0 +1,41 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.AdminAPI.ReportView do + use Pleroma.Web, :view + alias Pleroma.Activity + alias Pleroma.User + alias Pleroma.Web.CommonAPI.Utils + alias Pleroma.Web.MastodonAPI.AccountView + alias Pleroma.Web.MastodonAPI.StatusView + + def render("index.json", %{reports: reports}) do + %{ + reports: render_many(reports, __MODULE__, "show.json", as: :report) + } + end + + def render("show.json", %{report: report}) do + user = User.get_cached_by_ap_id(report.data["actor"]) + created_at = Utils.to_masto_date(report.data["published"]) + + [account_ap_id | status_ap_ids] = report.data["object"] + account = User.get_cached_by_ap_id(account_ap_id) + + statuses = + Enum.map(status_ap_ids, fn ap_id -> + Activity.get_by_ap_id_with_object(ap_id) + end) + + %{ + id: report.id, + account: AccountView.render("account.json", %{user: account}), + actor: AccountView.render("account.json", %{user: user}), + content: report.data["content"], + created_at: created_at, + statuses: StatusView.render("index.json", %{activities: statuses, as: :activity}), + state: report.data["state"] + } + end +end diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 29c4c1014..208c12c7b 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -71,6 +71,9 @@ defmodule Pleroma.Web.CommonAPI do {:ok, _} <- unpin(activity_id, user), {:ok, delete} <- ActivityPub.delete(object) do {:ok, delete} + else + _ -> + {:error, "Could not delete"} end end @@ -315,6 +318,60 @@ defmodule Pleroma.Web.CommonAPI do end end + def update_report_state(activity_id, state) do + with %Activity{} = activity <- Activity.get_by_id(activity_id), + {:ok, activity} <- Utils.update_report_state(activity, state) do + {:ok, activity} + else + nil -> + {:error, :not_found} + + {:error, reason} -> + {:error, reason} + + _ -> + {:error, "Could not update state"} + end + end + + def update_activity_scope(activity_id, opts \\ %{}) do + with %Activity{} = activity <- Activity.get_by_id_with_object(activity_id), + {:ok, activity} <- toggle_sensitive(activity, opts), + {:ok, activity} <- set_visibility(activity, opts) do + {:ok, activity} + else + nil -> + {:error, :not_found} + + {:error, reason} -> + {:error, reason} + end + end + + defp toggle_sensitive(activity, %{"sensitive" => sensitive}) when sensitive in ~w(true false) do + toggle_sensitive(activity, %{"sensitive" => String.to_existing_atom(sensitive)}) + end + + defp toggle_sensitive(%Activity{object: object} = activity, %{"sensitive" => sensitive}) + when is_boolean(sensitive) do + new_data = Map.put(object.data, "sensitive", sensitive) + + {:ok, object} = + object + |> Object.change(%{data: new_data}) + |> Object.update_and_set_cache() + + {:ok, Map.put(activity, :object, object)} + end + + defp toggle_sensitive(activity, _), do: {:ok, activity} + + defp set_visibility(activity, %{"visibility" => visibility}) do + Utils.update_activity_visibility(activity, visibility) + end + + defp set_visibility(activity, _), do: {:ok, activity} + def hide_reblogs(user, muted) do ap_id = muted.ap_id diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 1dfe50b40..bee2fd159 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -237,13 +237,11 @@ defmodule Pleroma.Web.CommonAPI.Utils do "tag" => tags |> Enum.map(fn {_, tag} -> tag end) |> Enum.uniq() } - if in_reply_to do - in_reply_to_object = Object.normalize(in_reply_to) - - object - |> Map.put("inReplyTo", in_reply_to_object.data["id"]) + with false <- is_nil(in_reply_to), + %Object{} = in_reply_to_object <- Object.normalize(in_reply_to) do + Map.put(object, "inReplyTo", in_reply_to_object.data["id"]) else - object + _ -> object end end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 7fef82f82..6a4e4a1d4 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -194,6 +194,14 @@ defmodule Pleroma.Web.Router do get("/users", AdminAPIController, :list_users) get("/users/:nickname", AdminAPIController, :user_show) + + get("/reports", AdminAPIController, :list_reports) + get("/reports/:id", AdminAPIController, :report_show) + put("/reports/:id", AdminAPIController, :report_update_state) + post("/reports/:id/respond", AdminAPIController, :report_respond) + + put("/statuses/:id", AdminAPIController, :status_update) + delete("/statuses/:id", AdminAPIController, :status_delete) end scope "/", Pleroma.Web.TwitterAPI do diff --git a/priv/repo/migrations/20190511191044_set_default_state_to_reports.exs b/priv/repo/migrations/20190511191044_set_default_state_to_reports.exs new file mode 100644 index 000000000..0d3d253b6 --- /dev/null +++ b/priv/repo/migrations/20190511191044_set_default_state_to_reports.exs @@ -0,0 +1,19 @@ +defmodule Pleroma.Repo.Migrations.SetDefaultStateToReports do + use Ecto.Migration + + def up do + execute """ + UPDATE activities AS a + SET data = jsonb_set(data, '{state}', '"open"', true) + WHERE data->>'type' = 'Flag' + """ + end + + def down do + execute """ + UPDATE activities AS a + SET data = data #- '{state}' + WHERE data->>'type' = 'Flag' + """ + end +end diff --git a/test/support/factory.ex b/test/support/factory.ex index 2a2954ad6..90c7d80f2 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -43,7 +43,7 @@ defmodule Pleroma.Factory do def note_factory(attrs \\ %{}) do text = sequence(:text, &"This is :moominmamma: note #{&1}") - user = insert(:user) + user = attrs[:user] || insert(:user) data = %{ "type" => "Note", @@ -113,7 +113,8 @@ defmodule Pleroma.Factory do end def note_activity_factory(attrs \\ %{}) do - note = attrs[:note] || insert(:note) + user = attrs[:user] || insert(:user) + note = attrs[:note] || insert(:note, user: user) data = %{ "id" => Pleroma.Web.ActivityPub.Utils.generate_activity_id(), diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index 6c1897b5a..ca12c7215 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -5,8 +5,10 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do use Pleroma.Web.ConnCase + alias Pleroma.Activity alias Pleroma.User alias Pleroma.UserInviteToken + alias Pleroma.Web.CommonAPI import Pleroma.Factory describe "/api/pleroma/admin/users" do @@ -949,4 +951,329 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do } end end + + describe "GET /api/pleroma/admin/reports/:id" do + setup %{conn: conn} do + admin = insert(:user, info: %{is_admin: true}) + + %{conn: assign(conn, :user, admin)} + end + + test "returns report by its id", %{conn: conn} do + [reporter, target_user] = insert_pair(:user) + activity = insert(:note_activity, user: target_user) + + {:ok, %{id: report_id}} = + CommonAPI.report(reporter, %{ + "account_id" => target_user.id, + "comment" => "I feel offended", + "status_ids" => [activity.id] + }) + + response = + conn + |> get("/api/pleroma/admin/reports/#{report_id}") + |> json_response(:ok) + + assert response["id"] == report_id + end + + test "returns 404 when report id is invalid", %{conn: conn} do + conn = get(conn, "/api/pleroma/admin/reports/test") + + assert json_response(conn, :not_found) == "Not found" + end + end + + describe "PUT /api/pleroma/admin/reports/:id" do + setup %{conn: conn} do + admin = insert(:user, info: %{is_admin: true}) + [reporter, target_user] = insert_pair(:user) + activity = insert(:note_activity, user: target_user) + + {:ok, %{id: report_id}} = + CommonAPI.report(reporter, %{ + "account_id" => target_user.id, + "comment" => "I feel offended", + "status_ids" => [activity.id] + }) + + %{conn: assign(conn, :user, admin), id: report_id} + end + + test "mark report as resolved", %{conn: conn, id: id} do + response = + conn + |> put("/api/pleroma/admin/reports/#{id}", %{"state" => "resolved"}) + |> json_response(:ok) + + assert response["state"] == "resolved" + end + + test "closes report", %{conn: conn, id: id} do + response = + conn + |> put("/api/pleroma/admin/reports/#{id}", %{"state" => "closed"}) + |> json_response(:ok) + + assert response["state"] == "closed" + end + + test "returns 400 when state is unknown", %{conn: conn, id: id} do + conn = + conn + |> put("/api/pleroma/admin/reports/#{id}", %{"state" => "test"}) + + assert json_response(conn, :bad_request) == "Unsupported state" + end + + test "returns 404 when report is not exist", %{conn: conn} do + conn = + conn + |> put("/api/pleroma/admin/reports/test", %{"state" => "closed"}) + + assert json_response(conn, :not_found) == "Not found" + end + end + + describe "GET /api/pleroma/admin/reports" do + setup %{conn: conn} do + admin = insert(:user, info: %{is_admin: true}) + + %{conn: assign(conn, :user, admin)} + end + + test "returns empty response when no reports created", %{conn: conn} do + response = + conn + |> get("/api/pleroma/admin/reports") + |> json_response(:ok) + + assert Enum.empty?(response["reports"]) + end + + test "returns reports", %{conn: conn} do + [reporter, target_user] = insert_pair(:user) + activity = insert(:note_activity, user: target_user) + + {:ok, %{id: report_id}} = + CommonAPI.report(reporter, %{ + "account_id" => target_user.id, + "comment" => "I feel offended", + "status_ids" => [activity.id] + }) + + response = + conn + |> get("/api/pleroma/admin/reports") + |> json_response(:ok) + + [report] = response["reports"] + + assert length(response["reports"]) == 1 + assert report["id"] == report_id + end + + test "returns reports with specified state", %{conn: conn} do + [reporter, target_user] = insert_pair(:user) + activity = insert(:note_activity, user: target_user) + + {:ok, %{id: first_report_id}} = + CommonAPI.report(reporter, %{ + "account_id" => target_user.id, + "comment" => "I feel offended", + "status_ids" => [activity.id] + }) + + {:ok, %{id: second_report_id}} = + CommonAPI.report(reporter, %{ + "account_id" => target_user.id, + "comment" => "I don't like this user" + }) + + CommonAPI.update_report_state(second_report_id, "closed") + + response = + conn + |> get("/api/pleroma/admin/reports", %{ + "state" => "open" + }) + |> json_response(:ok) + + [open_report] = response["reports"] + + assert length(response["reports"]) == 1 + assert open_report["id"] == first_report_id + + response = + conn + |> get("/api/pleroma/admin/reports", %{ + "state" => "closed" + }) + |> json_response(:ok) + + [closed_report] = response["reports"] + + assert length(response["reports"]) == 1 + assert closed_report["id"] == second_report_id + + response = + conn + |> get("/api/pleroma/admin/reports", %{ + "state" => "resolved" + }) + |> json_response(:ok) + + assert Enum.empty?(response["reports"]) + end + + test "returns 403 when requested by a non-admin" do + user = insert(:user) + + conn = + build_conn() + |> assign(:user, user) + |> get("/api/pleroma/admin/reports") + + assert json_response(conn, :forbidden) == %{"error" => "User is not admin."} + end + + test "returns 403 when requested by anonymous" do + conn = + build_conn() + |> get("/api/pleroma/admin/reports") + + assert json_response(conn, :forbidden) == %{"error" => "Invalid credentials."} + end + end + + describe "POST /api/pleroma/admin/reports/:id/respond" do + setup %{conn: conn} do + admin = insert(:user, info: %{is_admin: true}) + + %{conn: assign(conn, :user, admin)} + end + + test "returns created dm", %{conn: conn} do + [reporter, target_user] = insert_pair(:user) + activity = insert(:note_activity, user: target_user) + + {:ok, %{id: report_id}} = + CommonAPI.report(reporter, %{ + "account_id" => target_user.id, + "comment" => "I feel offended", + "status_ids" => [activity.id] + }) + + response = + conn + |> post("/api/pleroma/admin/reports/#{report_id}/respond", %{ + "status" => "I will check it out" + }) + |> json_response(:ok) + + recipients = Enum.map(response["mentions"], & &1["username"]) + + assert conn.assigns[:user].nickname in recipients + assert reporter.nickname in recipients + assert response["content"] == "I will check it out" + assert response["visibility"] == "direct" + end + + test "returns 400 when status is missing", %{conn: conn} do + conn = post(conn, "/api/pleroma/admin/reports/test/respond") + + assert json_response(conn, :bad_request) == "Invalid parameters" + end + + test "returns 404 when report id is invalid", %{conn: conn} do + conn = + post(conn, "/api/pleroma/admin/reports/test/respond", %{ + "status" => "foo" + }) + + assert json_response(conn, :not_found) == "Not found" + end + end + + describe "PUT /api/pleroma/admin/statuses/:id" do + setup %{conn: conn} do + admin = insert(:user, info: %{is_admin: true}) + activity = insert(:note_activity) + + %{conn: assign(conn, :user, admin), id: activity.id} + end + + test "toggle sensitive flag", %{conn: conn, id: id} do + response = + conn + |> put("/api/pleroma/admin/statuses/#{id}", %{"sensitive" => "true"}) + |> json_response(:ok) + + assert response["sensitive"] + + response = + conn + |> put("/api/pleroma/admin/statuses/#{id}", %{"sensitive" => "false"}) + |> json_response(:ok) + + refute response["sensitive"] + end + + test "change visibility flag", %{conn: conn, id: id} do + response = + conn + |> put("/api/pleroma/admin/statuses/#{id}", %{"visibility" => "public"}) + |> json_response(:ok) + + assert response["visibility"] == "public" + + response = + conn + |> put("/api/pleroma/admin/statuses/#{id}", %{"visibility" => "private"}) + |> json_response(:ok) + + assert response["visibility"] == "private" + + response = + conn + |> put("/api/pleroma/admin/statuses/#{id}", %{"visibility" => "unlisted"}) + |> json_response(:ok) + + assert response["visibility"] == "unlisted" + end + + test "returns 400 when visibility is unknown", %{conn: conn, id: id} do + conn = + conn + |> put("/api/pleroma/admin/statuses/#{id}", %{"visibility" => "test"}) + + assert json_response(conn, :bad_request) == "Unsupported visibility" + end + end + + describe "DELETE /api/pleroma/admin/statuses/:id" do + setup %{conn: conn} do + admin = insert(:user, info: %{is_admin: true}) + activity = insert(:note_activity) + + %{conn: assign(conn, :user, admin), id: activity.id} + end + + test "deletes status", %{conn: conn, id: id} do + conn + |> delete("/api/pleroma/admin/statuses/#{id}") + |> json_response(:ok) + + refute Activity.get_by_id(id) + end + + test "returns error when status is not exist", %{conn: conn} do + conn = + conn + |> delete("/api/pleroma/admin/statuses/test") + + assert json_response(conn, :bad_request) == "Could not delete" + end + end end diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs index 8d4f401ee..38b5aa65f 100644 --- a/test/web/common_api/common_api_test.exs +++ b/test/web/common_api/common_api_test.exs @@ -261,10 +261,41 @@ defmodule Pleroma.Web.CommonAPITest do data: %{ "type" => "Flag", "content" => ^comment, - "object" => [^target_ap_id, ^activity_ap_id] + "object" => [^target_ap_id, ^activity_ap_id], + "state" => "open" } } = flag_activity end + + test "updates report state" do + [reporter, target_user] = insert_pair(:user) + activity = insert(:note_activity, user: target_user) + + {:ok, %Activity{id: report_id}} = + CommonAPI.report(reporter, %{ + "account_id" => target_user.id, + "comment" => "I feel offended", + "status_ids" => [activity.id] + }) + + {:ok, report} = CommonAPI.update_report_state(report_id, "resolved") + + assert report.data["state"] == "resolved" + end + + test "does not update report state when state is unsupported" do + [reporter, target_user] = insert_pair(:user) + activity = insert(:note_activity, user: target_user) + + {:ok, %Activity{id: report_id}} = + CommonAPI.report(reporter, %{ + "account_id" => target_user.id, + "comment" => "I feel offended", + "status_ids" => [activity.id] + }) + + assert CommonAPI.update_report_state(report_id, "test") == {:error, "Unsupported state"} + end end describe "reblog muting" do diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 40e7739e7..90d67a55f 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -2129,7 +2129,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do |> get("/api/v1/pleroma/accounts/#{user.id}/favourites") |> json_response(:ok) - assert length(anonymous_response) == 0 + assert Enum.empty?(anonymous_response) end test "does not return others' favorited DM when user is not one of recipients", %{ @@ -2153,7 +2153,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do |> get("/api/v1/pleroma/accounts/#{user.id}/favourites") |> json_response(:ok) - assert length(response) == 0 + assert Enum.empty?(response) end test "paginates favorites using since_id and max_id", %{ From 2c303afc8b25ca92eaa229df3e9e4737856e890c Mon Sep 17 00:00:00 2001 From: Sergey Suprunenko Date: Thu, 16 May 2019 20:04:08 +0000 Subject: [PATCH 31/40] Remove duplicated entries in users' following lists --- CHANGELOG.md | 1 + lib/mix/tasks/pleroma/database.ex | 17 +++++++++-- lib/pleroma/user.ex | 14 ++++++++- test/tasks/database_test.exs | 49 +++++++++++++++++++++++++++++++ test/tasks/user_test.exs | 1 + test/user_test.exs | 31 +++++++++++++++++++ 6 files changed, 110 insertions(+), 3 deletions(-) create mode 100644 test/tasks/database_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index dc09d4f6b..4d314817b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Support for Mastodon's remote interaction - Mix Tasks: `mix pleroma.database bump_all_conversations` - Mix Tasks: `mix pleroma.database remove_embedded_objects` +- Mix Tasks: `mix pleroma.database update_users_following_followers_counts` - Mix Tasks: `mix pleroma.user toggle_confirmed` - Federation: Support for reports - Configuration: `safe_dm_mentions` option diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index 42753a1a4..f650b447d 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -5,6 +5,8 @@ defmodule Mix.Tasks.Pleroma.Database do alias Mix.Tasks.Pleroma.Common alias Pleroma.Conversation + alias Pleroma.Repo + alias Pleroma.User require Logger use Mix.Task @@ -25,6 +27,9 @@ defmodule Mix.Tasks.Pleroma.Database do mix pleroma.database bump_all_conversations + ## Remove duplicated items from following and update followers count for all users + + mix pleroma.database update_users_following_followers_counts """ def run(["remove_embedded_objects" | args]) do {options, [], []} = @@ -38,7 +43,7 @@ defmodule Mix.Tasks.Pleroma.Database do Common.start_pleroma() Logger.info("Removing embedded objects") - Pleroma.Repo.query!( + Repo.query!( "update activities set data = jsonb_set(data, '{object}'::text[], data->'object'->'id') where data->'object'->>'id' is not null;", [], timeout: :infinity @@ -47,7 +52,7 @@ defmodule Mix.Tasks.Pleroma.Database do if Keyword.get(options, :vacuum) do Logger.info("Runnning VACUUM FULL") - Pleroma.Repo.query!( + Repo.query!( "vacuum full;", [], timeout: :infinity @@ -59,4 +64,12 @@ defmodule Mix.Tasks.Pleroma.Database do Common.start_pleroma() Conversation.bump_for_all_activities() end + + def run(["update_users_following_followers_counts"]) do + Common.start_pleroma() + + users = Repo.all(User) + Enum.each(users, &User.remove_duplicated_following/1) + Enum.each(users, &User.update_follower_count/1) + end end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 1aa966dfc..9ffb61300 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -166,7 +166,7 @@ defmodule Pleroma.User do def update_changeset(struct, params \\ %{}) do struct - |> cast(params, [:bio, :name, :avatar]) + |> cast(params, [:bio, :name, :avatar, :following]) |> unique_constraint(:nickname) |> validate_format(:nickname, local_nickname_regex()) |> validate_length(:bio, max: 5000) @@ -709,6 +709,18 @@ defmodule Pleroma.User do end end + def remove_duplicated_following(%User{following: following} = user) do + uniq_following = Enum.uniq(following) + + if length(following) == length(uniq_following) do + {:ok, user} + else + user + |> update_changeset(%{following: uniq_following}) + |> update_and_set_cache() + end + end + @spec get_users_from_set([String.t()], boolean()) :: [User.t()] def get_users_from_set(ap_ids, local_only \\ true) do criteria = %{ap_id: ap_ids, deactivated: false} diff --git a/test/tasks/database_test.exs b/test/tasks/database_test.exs new file mode 100644 index 000000000..579130b05 --- /dev/null +++ b/test/tasks/database_test.exs @@ -0,0 +1,49 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Mix.Tasks.Pleroma.DatabaseTest do + alias Pleroma.Repo + alias Pleroma.User + use Pleroma.DataCase + + import Pleroma.Factory + + setup_all do + Mix.shell(Mix.Shell.Process) + + on_exit(fn -> + Mix.shell(Mix.Shell.IO) + end) + + :ok + end + + describe "running update_users_following_followers_counts" do + test "following and followers count are updated" do + [user, user2] = insert_pair(:user) + {:ok, %User{following: following, info: info} = user} = User.follow(user, user2) + + assert length(following) == 2 + assert info.follower_count == 0 + + info_cng = Ecto.Changeset.change(info, %{follower_count: 3}) + + {:ok, user} = + user + |> Ecto.Changeset.change(%{following: following ++ following}) + |> Ecto.Changeset.put_embed(:info, info_cng) + |> Repo.update() + + assert length(user.following) == 4 + assert user.info.follower_count == 3 + + assert :ok == Mix.Tasks.Pleroma.Database.run(["update_users_following_followers_counts"]) + + user = User.get_by_id(user.id) + + assert length(user.following) == 2 + assert user.info.follower_count == 0 + end + end +end diff --git a/test/tasks/user_test.exs b/test/tasks/user_test.exs index 1f97740be..260ce0d95 100644 --- a/test/tasks/user_test.exs +++ b/test/tasks/user_test.exs @@ -3,6 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Mix.Tasks.Pleroma.UserTest do + alias Pleroma.Repo alias Pleroma.User use Pleroma.DataCase diff --git a/test/user_test.exs b/test/user_test.exs index 16a014f2f..721b65693 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -626,6 +626,37 @@ defmodule Pleroma.UserTest do end end + describe "remove duplicates from following list" do + test "it removes duplicates" do + user = insert(:user) + follower = insert(:user) + + {:ok, %User{following: following} = follower} = User.follow(follower, user) + assert length(following) == 2 + + {:ok, follower} = + follower + |> User.update_changeset(%{following: following ++ following}) + |> Repo.update() + + assert length(follower.following) == 4 + + {:ok, follower} = User.remove_duplicated_following(follower) + assert length(follower.following) == 2 + end + + test "it does nothing when following is uniq" do + user = insert(:user) + follower = insert(:user) + + {:ok, follower} = User.follow(follower, user) + assert length(follower.following) == 2 + + {:ok, follower} = User.remove_duplicated_following(follower) + assert length(follower.following) == 2 + end + end + describe "follow_import" do test "it imports user followings from list" do [user1, user2, user3] = insert_list(3, :user) From 70235ce840a4ed834080ab6afc3000185590c6cb Mon Sep 17 00:00:00 2001 From: Aaron Tinio Date: Fri, 17 May 2019 09:02:04 +0800 Subject: [PATCH 32/40] Fix typo: s/"tags"/"tag"/g --- lib/pleroma/web/activity_pub/mrf/simple_policy.ex | 2 +- lib/pleroma/web/activity_pub/mrf/tag_policy.ex | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index 2f105700b..50426e920 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -55,7 +55,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do object = if Enum.member?(Pleroma.Config.get([:mrf_simple, :media_nsfw]), actor_host) do tags = (child_object["tag"] || []) ++ ["nsfw"] - child_object = Map.put(child_object, "tags", tags) + child_object = Map.put(child_object, "tag", tags) child_object = Map.put(child_object, "sensitive", true) Map.put(object, "object", child_object) else diff --git a/lib/pleroma/web/activity_pub/mrf/tag_policy.ex b/lib/pleroma/web/activity_pub/mrf/tag_policy.ex index b52be30e7..6683b8d8e 100644 --- a/lib/pleroma/web/activity_pub/mrf/tag_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/tag_policy.ex @@ -31,7 +31,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do object = object - |> Map.put("tags", tags) + |> Map.put("tag", tags) |> Map.put("sensitive", true) message = Map.put(message, "object", object) From 443cd68a4c37cc8e3a3260feff25791bd5ebabf4 Mon Sep 17 00:00:00 2001 From: Aaron Tinio Date: Fri, 17 May 2019 08:11:17 +0800 Subject: [PATCH 33/40] Add tests for MRF.SimplePolicy --- .../activity_pub/mrf/simple_policy_test.exs | 192 ++++++++++++++++++ 1 file changed, 192 insertions(+) create mode 100644 test/web/activity_pub/mrf/simple_policy_test.exs diff --git a/test/web/activity_pub/mrf/simple_policy_test.exs b/test/web/activity_pub/mrf/simple_policy_test.exs new file mode 100644 index 000000000..1e0511975 --- /dev/null +++ b/test/web/activity_pub/mrf/simple_policy_test.exs @@ -0,0 +1,192 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do + use Pleroma.DataCase + import Pleroma.Factory + alias Pleroma.Config + alias Pleroma.Web.ActivityPub.MRF.SimplePolicy + + setup do + orig = Config.get!(:mrf_simple) + + Config.put(:mrf_simple, + media_removal: [], + media_nsfw: [], + federated_timeline_removal: [], + reject: [], + accept: [] + ) + + on_exit(fn -> + Config.put(:mrf_simple, orig) + end) + end + + describe "when :media_removal" do + test "is empty" do + Config.put([:mrf_simple, :media_removal], []) + media_message = build_media_message() + local_message = build_local_message() + + assert SimplePolicy.filter(media_message) == {:ok, media_message} + assert SimplePolicy.filter(local_message) == {:ok, local_message} + end + + test "has a matching host" do + Config.put([:mrf_simple, :media_removal], ["remote.instance"]) + media_message = build_media_message() + local_message = build_local_message() + + assert SimplePolicy.filter(media_message) == + {:ok, + media_message + |> Map.put("object", Map.delete(media_message["object"], "attachment"))} + + assert SimplePolicy.filter(local_message) == {:ok, local_message} + end + end + + describe "when :media_nsfw" do + test "is empty" do + Config.put([:mrf_simple, :media_nsfw], []) + media_message = build_media_message() + local_message = build_local_message() + + assert SimplePolicy.filter(media_message) == {:ok, media_message} + assert SimplePolicy.filter(local_message) == {:ok, local_message} + end + + test "has a matching host" do + Config.put([:mrf_simple, :media_nsfw], ["remote.instance"]) + media_message = build_media_message() + local_message = build_local_message() + + assert SimplePolicy.filter(media_message) == + {:ok, + media_message + |> put_in(["object", "tag"], ["foo", "nsfw"]) + |> put_in(["object", "sensitive"], true)} + + assert SimplePolicy.filter(local_message) == {:ok, local_message} + end + end + + defp build_media_message do + %{ + "actor" => "https://remote.instance/users/bob", + "type" => "Create", + "object" => %{ + "attachment" => [%{}], + "tag" => ["foo"], + "sensitive" => false + } + } + end + + describe "when :federated_timeline_removal" do + test "is empty" do + Config.put([:mrf_simple, :federated_timeline_removal], []) + {_, ftl_message} = build_ftl_actor_and_message() + local_message = build_local_message() + + assert SimplePolicy.filter(ftl_message) == {:ok, ftl_message} + assert SimplePolicy.filter(local_message) == {:ok, local_message} + end + + test "has a matching host" do + {actor, ftl_message} = build_ftl_actor_and_message() + + ftl_message_actor_host = + ftl_message + |> Map.fetch!("actor") + |> URI.parse() + |> Map.fetch!(:host) + + Config.put([:mrf_simple, :federated_timeline_removal], [ftl_message_actor_host]) + local_message = build_local_message() + + assert {:ok, ftl_message} = SimplePolicy.filter(ftl_message) + assert actor.follower_address in ftl_message["to"] + refute actor.follower_address in ftl_message["cc"] + refute "https://www.w3.org/ns/activitystreams#Public" in ftl_message["to"] + assert "https://www.w3.org/ns/activitystreams#Public" in ftl_message["cc"] + + assert SimplePolicy.filter(local_message) == {:ok, local_message} + end + end + + defp build_ftl_actor_and_message do + actor = insert(:user) + + {actor, + %{ + "actor" => actor.ap_id, + "to" => ["https://www.w3.org/ns/activitystreams#Public", "http://foo.bar/baz"], + "cc" => [actor.follower_address, "http://foo.bar/qux"] + }} + end + + describe "when :reject" do + test "is empty" do + Config.put([:mrf_simple, :reject], []) + + remote_message = build_remote_message() + + assert SimplePolicy.filter(remote_message) == {:ok, remote_message} + end + + test "has a matching host" do + Config.put([:mrf_simple, :reject], ["remote.instance"]) + + remote_message = build_remote_message() + + assert SimplePolicy.filter(remote_message) == {:reject, nil} + end + end + + describe "when :accept" do + test "is empty" do + Config.put([:mrf_simple, :accept], []) + + local_message = build_local_message() + remote_message = build_remote_message() + + assert SimplePolicy.filter(local_message) == {:ok, local_message} + assert SimplePolicy.filter(remote_message) == {:ok, remote_message} + end + + test "is not empty but it doesn't have a matching host" do + Config.put([:mrf_simple, :accept], ["non.matching.remote"]) + + local_message = build_local_message() + remote_message = build_remote_message() + + assert SimplePolicy.filter(local_message) == {:ok, local_message} + assert SimplePolicy.filter(remote_message) == {:reject, nil} + end + + test "has a matching host" do + Config.put([:mrf_simple, :accept], ["remote.instance"]) + + local_message = build_local_message() + remote_message = build_remote_message() + + assert SimplePolicy.filter(local_message) == {:ok, local_message} + assert SimplePolicy.filter(remote_message) == {:ok, remote_message} + end + end + + defp build_local_message do + %{ + "actor" => "#{Pleroma.Web.base_url()}/users/alice", + "to" => [], + "cc" => [] + } + end + + defp build_remote_message do + %{"actor" => "https://remote.instance/users/bob"} + end +end From 68c75a08e8593d41f496a362ab1be43d054c09d5 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 17 May 2019 07:34:52 +0200 Subject: [PATCH 34/40] MastoAPI account_view.ex: requested is false when following is true Closes: https://git.pleroma.social/pleroma/pleroma/issues/903 --- lib/pleroma/web/mastodon_api/views/account_view.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 779b9a382..606a0f2ed 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -40,7 +40,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do follow_activity = Pleroma.Web.ActivityPub.Utils.fetch_latest_follow(user, target) requested = - if follow_activity do + if follow_activity and !User.following?(target, user) do follow_activity.data["state"] == "pending" else false From 7ed682213faf88edc67b4c2a24b277ab1d992813 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 17 May 2019 07:25:20 +0000 Subject: [PATCH 35/40] Fix/902 random compile failing --- lib/pleroma/user.ex | 6 +++--- lib/pleroma/web/activity_pub/transmogrifier.ex | 1 - lib/pleroma/web/common_api/common_api.ex | 6 +++--- lib/pleroma/web/federator/publisher.ex | 8 ++++---- .../web/mastodon_api/mastodon_api_controller.ex | 2 +- lib/pleroma/web/oauth/authorization.ex | 2 +- lib/pleroma/web/oauth/token.ex | 2 +- lib/pleroma/web/web_finger/web_finger.ex | 2 +- test/formatter_test.exs | 2 +- test/plugs/legacy_authentication_plug_test.exs | 2 +- test/repo_test.exs | 7 ++++--- test/support/factory.ex | 9 +++++---- test/user_test.exs | 10 +++++----- test/web/mastodon_api/mastodon_api_controller_test.exs | 8 ++++---- test/web/ostatus/ostatus_test.exs | 4 ++-- 15 files changed, 36 insertions(+), 35 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 9ffb61300..9607c1f03 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -55,7 +55,7 @@ defmodule Pleroma.User do field(:last_refreshed_at, :naive_datetime_usec) has_many(:notifications, Notification) has_many(:registrations, Registration) - embeds_one(:info, Pleroma.User.Info) + embeds_one(:info, User.Info) timestamps() end @@ -233,7 +233,7 @@ defmodule Pleroma.User do |> validate_confirmation(:password) |> unique_constraint(:email) |> unique_constraint(:nickname) - |> validate_exclusion(:nickname, Pleroma.Config.get([Pleroma.User, :restricted_nicknames])) + |> validate_exclusion(:nickname, Pleroma.Config.get([User, :restricted_nicknames])) |> validate_format(:nickname, local_nickname_regex()) |> validate_format(:email, @email_regex) |> validate_length(:bio, max: 1000) @@ -278,7 +278,7 @@ defmodule Pleroma.User do with {:ok, user} <- Repo.insert(changeset), {:ok, user} <- autofollow_users(user), {:ok, user} <- set_cache(user), - {:ok, _} <- Pleroma.User.WelcomeMessage.post_welcome_message_to_user(user), + {:ok, _} <- User.WelcomeMessage.post_welcome_message_to_user(user), {:ok, _} <- try_send_confirmation_email(user) do {:ok, user} end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 508f3532f..5edd8ccc7 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -11,7 +11,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do alias Pleroma.Object.Containment alias Pleroma.Repo alias Pleroma.User - alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.ActivityPub.Visibility diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 208c12c7b..a599ffee5 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -200,7 +200,7 @@ defmodule Pleroma.Web.CommonAPI do user = with emoji <- emoji_from_profile(user), source_data <- (user.info.source_data || %{}) |> Map.put("tag", emoji), - info_cng <- Pleroma.User.Info.set_source_data(user.info, source_data), + info_cng <- User.Info.set_source_data(user.info, source_data), change <- Ecto.Changeset.change(user) |> Ecto.Changeset.put_embed(:info, info_cng), {:ok, user} <- User.update_and_set_cache(change) do user @@ -233,7 +233,7 @@ defmodule Pleroma.Web.CommonAPI do } = activity <- get_by_id_or_ap_id(id_or_ap_id), true <- Enum.member?(object_to, "https://www.w3.org/ns/activitystreams#Public"), %{valid?: true} = info_changeset <- - Pleroma.User.Info.add_pinnned_activity(user.info, activity), + User.Info.add_pinnned_activity(user.info, activity), changeset <- Ecto.Changeset.change(user) |> Ecto.Changeset.put_embed(:info, info_changeset), {:ok, _user} <- User.update_and_set_cache(changeset) do @@ -250,7 +250,7 @@ defmodule Pleroma.Web.CommonAPI do def unpin(id_or_ap_id, user) do with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id), %{valid?: true} = info_changeset <- - Pleroma.User.Info.remove_pinnned_activity(user.info, activity), + User.Info.remove_pinnned_activity(user.info, activity), changeset <- Ecto.Changeset.change(user) |> Ecto.Changeset.put_embed(:info, info_changeset), {:ok, _user} <- User.update_and_set_cache(changeset) do diff --git a/lib/pleroma/web/federator/publisher.ex b/lib/pleroma/web/federator/publisher.ex index fb4e8548d..70f870244 100644 --- a/lib/pleroma/web/federator/publisher.ex +++ b/lib/pleroma/web/federator/publisher.ex @@ -52,9 +52,9 @@ defmodule Pleroma.Web.Federator.Publisher do @doc """ Relays an activity to all specified peers. """ - @callback publish(Pleroma.User.t(), Pleroma.Activity.t()) :: :ok | {:error, any()} + @callback publish(User.t(), Activity.t()) :: :ok | {:error, any()} - @spec publish(Pleroma.User.t(), Pleroma.Activity.t()) :: :ok + @spec publish(User.t(), Activity.t()) :: :ok def publish(%User{} = user, %Activity{} = activity) do Config.get([:instance, :federation_publisher_modules]) |> Enum.each(fn module -> @@ -70,9 +70,9 @@ defmodule Pleroma.Web.Federator.Publisher do @doc """ Gathers links used by an outgoing federation module for WebFinger output. """ - @callback gather_webfinger_links(Pleroma.User.t()) :: list() + @callback gather_webfinger_links(User.t()) :: list() - @spec gather_webfinger_links(Pleroma.User.t()) :: list() + @spec gather_webfinger_links(User.t()) :: list() def gather_webfinger_links(%User{} = user) do Config.get([:instance, :federation_publisher_modules]) |> Enum.reduce([], fn module, links -> diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 66056a846..1b776fbca 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -1222,7 +1222,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do accounts |> Enum.each(fn account_id -> with %Pleroma.List{} = list <- Pleroma.List.get(id, user), - %User{} = followed <- Pleroma.User.get_cached_by_id(account_id) do + %User{} = followed <- User.get_cached_by_id(account_id) do Pleroma.List.unfollow(list, followed) end end) diff --git a/lib/pleroma/web/oauth/authorization.ex b/lib/pleroma/web/oauth/authorization.ex index b47688de1..18973413e 100644 --- a/lib/pleroma/web/oauth/authorization.ex +++ b/lib/pleroma/web/oauth/authorization.ex @@ -20,7 +20,7 @@ defmodule Pleroma.Web.OAuth.Authorization do field(:scopes, {:array, :string}, default: []) field(:valid_until, :naive_datetime_usec) field(:used, :boolean, default: false) - belongs_to(:user, Pleroma.User, type: Pleroma.FlakeId) + belongs_to(:user, User, type: Pleroma.FlakeId) belongs_to(:app, App) timestamps() diff --git a/lib/pleroma/web/oauth/token.ex b/lib/pleroma/web/oauth/token.ex index ef047d565..66c95c2e9 100644 --- a/lib/pleroma/web/oauth/token.ex +++ b/lib/pleroma/web/oauth/token.ex @@ -22,7 +22,7 @@ defmodule Pleroma.Web.OAuth.Token do field(:refresh_token, :string) field(:scopes, {:array, :string}, default: []) field(:valid_until, :naive_datetime_usec) - belongs_to(:user, Pleroma.User, type: Pleroma.FlakeId) + belongs_to(:user, User, type: Pleroma.FlakeId) belongs_to(:app, App) timestamps() diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index 3a3b98a10..1239b962a 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -99,7 +99,7 @@ defmodule Pleroma.Web.WebFinger do info_cng = info - |> Pleroma.User.Info.set_keys(pem) + |> User.Info.set_keys(pem) cng = Ecto.Changeset.change(user) diff --git a/test/formatter_test.exs b/test/formatter_test.exs index 06f4f6e50..5e7011160 100644 --- a/test/formatter_test.exs +++ b/test/formatter_test.exs @@ -125,7 +125,7 @@ defmodule Pleroma.FormatterTest do archaeme = insert(:user, %{ nickname: "archa_eme_", - info: %Pleroma.User.Info{source_data: %{"url" => "https://archeme/@archa_eme_"}} + info: %User.Info{source_data: %{"url" => "https://archeme/@archa_eme_"}} }) archaeme_remote = insert(:user, %{nickname: "archaeme@archae.me"}) diff --git a/test/plugs/legacy_authentication_plug_test.exs b/test/plugs/legacy_authentication_plug_test.exs index 8b0b06772..02f530058 100644 --- a/test/plugs/legacy_authentication_plug_test.exs +++ b/test/plugs/legacy_authentication_plug_test.exs @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Plugs.LegacyAuthenticationPlugTest do - use Pleroma.Web.ConnCase, async: true + use Pleroma.Web.ConnCase alias Pleroma.Plugs.LegacyAuthenticationPlug alias Pleroma.User diff --git a/test/repo_test.exs b/test/repo_test.exs index 5382289c7..85085a1fa 100644 --- a/test/repo_test.exs +++ b/test/repo_test.exs @@ -1,23 +1,24 @@ defmodule Pleroma.RepoTest do use Pleroma.DataCase import Pleroma.Factory + alias Pleroma.User describe "find_resource/1" do test "returns user" do user = insert(:user) - query = from(t in Pleroma.User, where: t.id == ^user.id) + query = from(t in User, where: t.id == ^user.id) assert Repo.find_resource(query) == {:ok, user} end test "returns not_found" do - query = from(t in Pleroma.User, where: t.id == ^"9gBuXNpD2NyDmmxxdw") + query = from(t in User, where: t.id == ^"9gBuXNpD2NyDmmxxdw") assert Repo.find_resource(query) == {:error, :not_found} end end describe "get_assoc/2" do test "get assoc from preloaded data" do - user = %Pleroma.User{name: "Agent Smith"} + user = %User{name: "Agent Smith"} token = %Pleroma.Web.OAuth.Token{insert(:oauth_token) | user: user} assert Repo.get_assoc(token, :user) == {:ok, user} end diff --git a/test/support/factory.ex b/test/support/factory.ex index 90c7d80f2..be6247ca4 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Factory do use ExMachina.Ecto, repo: Pleroma.Repo + alias Pleroma.User def participation_factory do conversation = insert(:conversation) @@ -23,7 +24,7 @@ defmodule Pleroma.Factory do end def user_factory do - user = %Pleroma.User{ + user = %User{ name: sequence(:name, &"Test テスト User #{&1}"), email: sequence(:email, &"user#{&1}@example.com"), nickname: sequence(:nickname, &"nick#{&1}"), @@ -34,9 +35,9 @@ defmodule Pleroma.Factory do %{ user - | ap_id: Pleroma.User.ap_id(user), - follower_address: Pleroma.User.ap_followers(user), - following: [Pleroma.User.ap_id(user)] + | ap_id: User.ap_id(user), + follower_address: User.ap_followers(user), + following: [User.ap_id(user)] } end diff --git a/test/user_test.exs b/test/user_test.exs index 721b65693..10e463ff8 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -277,7 +277,7 @@ defmodule Pleroma.UserTest do end test "it restricts certain nicknames" do - [restricted_name | _] = Pleroma.Config.get([Pleroma.User, :restricted_nicknames]) + [restricted_name | _] = Pleroma.Config.get([User, :restricted_nicknames]) assert is_bitstring(restricted_name) @@ -1223,11 +1223,11 @@ defmodule Pleroma.UserTest do follower2 = insert(:user) follower3 = insert(:user) - {:ok, follower} = Pleroma.User.follow(follower, user) - {:ok, _follower2} = Pleroma.User.follow(follower2, user) - {:ok, _follower3} = Pleroma.User.follow(follower3, user) + {:ok, follower} = User.follow(follower, user) + {:ok, _follower2} = User.follow(follower2, user) + {:ok, _follower3} = User.follow(follower3, user) - {:ok, _} = Pleroma.User.block(user, follower) + {:ok, _} = User.block(user, follower) user_show = Pleroma.Web.TwitterAPI.UserView.render("show.json", %{user: user}) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 90d67a55f..cbff141c8 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -446,7 +446,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do end test "verify_credentials default scope unlisted", %{conn: conn} do - user = insert(:user, %{info: %Pleroma.User.Info{default_scope: "unlisted"}}) + user = insert(:user, %{info: %User.Info{default_scope: "unlisted"}}) conn = conn @@ -1322,7 +1322,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do describe "locked accounts" do test "/api/v1/follow_requests works" do - user = insert(:user, %{info: %Pleroma.User.Info{locked: true}}) + user = insert(:user, %{info: %User.Info{locked: true}}) other_user = insert(:user) {:ok, _activity} = ActivityPub.follow(other_user, user) @@ -1367,7 +1367,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do end test "verify_credentials", %{conn: conn} do - user = insert(:user, %{info: %Pleroma.User.Info{default_scope: "private"}}) + user = insert(:user, %{info: %User.Info{default_scope: "private"}}) conn = conn @@ -1379,7 +1379,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do end test "/api/v1/follow_requests/:id/reject works" do - user = insert(:user, %{info: %Pleroma.User.Info{locked: true}}) + user = insert(:user, %{info: %User.Info{locked: true}}) other_user = insert(:user) {:ok, _activity} = ActivityPub.follow(other_user, user) diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 2916caf8d..3516a13fa 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -355,7 +355,7 @@ defmodule Pleroma.Web.OStatusTest do {:ok, user} = OStatus.find_or_make_user(uri) - user = Pleroma.User.get_cached_by_id(user.id) + user = User.get_cached_by_id(user.id) assert user.name == "Constance Variable" assert user.nickname == "lambadalambda@social.heldscal.la" assert user.local == false @@ -374,7 +374,7 @@ defmodule Pleroma.Web.OStatusTest do {:ok, user} = OStatus.find_or_make_user(uri) assert user.info == - %Pleroma.User.Info{ + %User.Info{ id: user.info.id, ap_enabled: false, background: %{}, From 718b59a2631af6245570a073e3c5b2464b810a27 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 17 May 2019 11:19:30 +0200 Subject: [PATCH 36/40] Dependencies: Update ecto sql once more. --- mix.exs | 2 +- mix.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index 2535fcf98..2888b69e2 100644 --- a/mix.exs +++ b/mix.exs @@ -67,7 +67,7 @@ defmodule Pleroma.Mixfile do {:phoenix_ecto, "~> 4.0"}, {:ecto_sql, git: "https://github.com/elixir-ecto/ecto_sql", - ref: "e839a9a327b632d73533ac8105ba360bc831cf83", + ref: "14cb065a74c488d737d973f7a91bc036c6245f78", override: true}, {:postgrex, ">= 0.13.5"}, {:gettext, "~> 0.15"}, diff --git a/mix.lock b/mix.lock index 5ee5634d3..714a8a2b1 100644 --- a/mix.lock +++ b/mix.lock @@ -21,7 +21,7 @@ "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm"}, "earmark": {:hex, :earmark, "1.3.2", "b840562ea3d67795ffbb5bd88940b1bed0ed9fa32834915125ea7d02e35888a5", [:mix], [], "hexpm"}, "ecto": {:hex, :ecto, "3.1.4", "69d852da7a9f04ede725855a35ede48d158ca11a404fe94f8b2fb3b2162cd3c9", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, - "ecto_sql": {:git, "https://github.com/elixir-ecto/ecto_sql", "e839a9a327b632d73533ac8105ba360bc831cf83", [ref: "e839a9a327b632d73533ac8105ba360bc831cf83"]}, + "ecto_sql": {:git, "https://github.com/elixir-ecto/ecto_sql", "14cb065a74c488d737d973f7a91bc036c6245f78", [ref: "14cb065a74c488d737d973f7a91bc036c6245f78"]}, "esshd": {:hex, :esshd, "0.1.0", "6f93a2062adb43637edad0ea7357db2702a4b80dd9683482fe00f5134e97f4c1", [:mix], [], "hexpm"}, "eternal": {:hex, :eternal, "1.2.0", "e2a6b6ce3b8c248f7dc31451aefca57e3bdf0e48d73ae5043229380a67614c41", [:mix], [], "hexpm"}, "ex2ms": {:hex, :ex2ms, "1.5.0", "19e27f9212be9a96093fed8cdfbef0a2b56c21237196d26760f11dfcfae58e97", [:mix], [], "hexpm"}, From e39c190f046bb345e80a2195278bb59ac81b8002 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 17 May 2019 11:30:35 +0200 Subject: [PATCH 37/40] Configuration: disable migration lock. --- config/config.exs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/config.exs b/config/config.exs index 3d11c3a87..92d3e0dcd 100644 --- a/config/config.exs +++ b/config/config.exs @@ -48,7 +48,8 @@ config :pleroma, ecto_repos: [Pleroma.Repo] config :pleroma, Pleroma.Repo, types: Pleroma.PostgresTypes, - telemetry_event: [Pleroma.Repo.Instrumenter] + telemetry_event: [Pleroma.Repo.Instrumenter], + migration_lock: nil config :pleroma, Pleroma.Captcha, enabled: false, From 249b31ffe53115ad187235890c405138ba190e14 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 17 May 2019 11:49:13 +0200 Subject: [PATCH 38/40] Fix specs. --- lib/pleroma/user.ex | 1 - test/web/ostatus/ostatus_test.exs | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 9607c1f03..28da310ee 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1144,7 +1144,6 @@ defmodule Pleroma.User do stream = ap_id |> Activity.query_by_actor() - |> Activity.with_preloaded_object() |> Repo.stream() Repo.transaction(fn -> Enum.each(stream, &delete_activity(&1)) end, timeout: :infinity) diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 3516a13fa..f6be16862 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -407,7 +407,7 @@ defmodule Pleroma.Web.OStatusTest do {:ok, user} = OStatus.find_or_make_user(uri) old_name = user.name old_bio = user.bio - change = Ecto.Changeset.change(user, %{avatar: nil, bio: nil, old_name: nil}) + change = Ecto.Changeset.change(user, %{avatar: nil, bio: nil, name: nil}) {:ok, user} = Repo.update(change) refute user.avatar From 5743124c7727af3fbcd8243f1f8e71de91c9b470 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 17 May 2019 11:51:50 +0200 Subject: [PATCH 39/40] Dependencies: Update tzdata. This fixes the bug that tz data could not be loaded. Closes #910. --- mix.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.lock b/mix.lock index cd2d2370a..1dc19532d 100644 --- a/mix.lock +++ b/mix.lock @@ -83,7 +83,7 @@ "tesla": {:hex, :tesla, "1.2.1", "864783cc27f71dd8c8969163704752476cec0f3a51eb3b06393b3971dc9733ff", [:mix], [{:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "~> 4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}], "hexpm"}, "timex": {:hex, :timex, "3.5.0", "b0a23167da02d0fe4f1a4e104d1f929a00d348502b52432c05de875d0b9cffa5", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, - "tzdata": {:hex, :tzdata, "0.5.17", "50793e3d85af49736701da1a040c415c97dc1caf6464112fd9bd18f425d3053b", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, + "tzdata": {:hex, :tzdata, "0.5.20", "304b9e98a02840fb32a43ec111ffbe517863c8566eb04a061f1c4dbb90b4d84c", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, "ueberauth": {:hex, :ueberauth, "0.6.1", "9e90d3337dddf38b1ca2753aca9b1e53d8a52b890191cdc55240247c89230412", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "unicode_util_compat": {:hex, :unicode_util_compat, "0.4.1", "d869e4c68901dd9531385bb0c8c40444ebf624e60b6962d95952775cac5e90cd", [:rebar3], [], "hexpm"}, "unsafe": {:hex, :unsafe, "1.0.0", "7c21742cd05380c7875546b023481d3a26f52df8e5dfedcb9f958f322baae305", [:mix], [], "hexpm"}, From 46684db84d0583cb5d88f8fcbd6b970ef95c84de Mon Sep 17 00:00:00 2001 From: lambda Date: Fri, 17 May 2019 09:53:44 +0000 Subject: [PATCH 40/40] Update account_view.ex --- lib/pleroma/web/mastodon_api/views/account_view.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 606a0f2ed..134c07b7e 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -40,7 +40,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do follow_activity = Pleroma.Web.ActivityPub.Utils.fetch_latest_follow(user, target) requested = - if follow_activity and !User.following?(target, user) do + if follow_activity && !User.following?(target, user) do follow_activity.data["state"] == "pending" else false