From 66c3813ea6388e9933af2b15e903f1cf6254cd3a Mon Sep 17 00:00:00 2001 From: Lain Iwakura Date: Tue, 5 Dec 2017 18:21:30 +0100 Subject: [PATCH 1/8] Add basic queue prioritization. --- lib/pleroma/web/federator/federator.ex | 21 +++++++++++++++------ test/web/federator_test.exs | 20 ++++++++++++++++++++ 2 files changed, 35 insertions(+), 6 deletions(-) create mode 100644 test/web/federator_test.exs diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index 9f6f983aa..f384b313c 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -15,8 +15,8 @@ def start_link do enqueue(:refresh_subscriptions, nil) end) GenServer.start_link(__MODULE__, %{ - in: {:sets.new(), :queue.new()}, - out: {:sets.new(), :queue.new()} + in: {:sets.new(), [], + out: {:sets.new(), []} }, name: __MODULE__) end @@ -88,8 +88,8 @@ def enqueue(type, payload) do end def maybe_start_job(running_jobs, queue) do - if (:sets.size(running_jobs) < @max_jobs) && !:queue.is_empty(queue) do - {{:value, {type, payload}}, queue} = :queue.out(queue) + if (:sets.size(running_jobs) < @max_jobs) && queue != [] do + {{:value, {type, payload}}, queue} = queue_pop(queue) {:ok, pid} = Task.start(fn -> handle(type, payload) end) mref = Process.monitor(pid) {:sets.add_element(mref, running_jobs), queue} @@ -100,14 +100,14 @@ def maybe_start_job(running_jobs, queue) do def handle_cast({:enqueue, type, payload}, state) when type in [:incoming_doc] do %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}} = state - i_queue = :queue.in({type, payload}, i_queue) + i_queue = enqueue_sorted(i_queue, {type, payload}, 1) {i_running_jobs, i_queue} = maybe_start_job(i_running_jobs, i_queue) {:noreply, %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}}} end def handle_cast({:enqueue, type, payload}, state) do %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}} = state - o_queue = :queue.in({type, payload}, o_queue) + o_queue = enqueue_sorted(o_queue, {type, payload}, 1) {o_running_jobs, o_queue} = maybe_start_job(o_running_jobs, o_queue) {:noreply, %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}}} end @@ -126,4 +126,13 @@ def handle_info({:DOWN, ref, :process, _pid, _reason}, state) do {:noreply, %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}}} end + + def enqueue_sorted(queue, element, priority) do + [%{item: element, priority: priority} | queue] + |> Enum.sort_by(fn (%{priority: priority}) -> priority end) + end + + def queue_pop([%{item: element} | queue]) do + {element, queue} + end end diff --git a/test/web/federator_test.exs b/test/web/federator_test.exs new file mode 100644 index 000000000..09533362a --- /dev/null +++ b/test/web/federator_test.exs @@ -0,0 +1,20 @@ +defmodule Pleroma.Web.FederatorTest do + alias Pleroma.Web.Federator + use Pleroma.DataCase + + test "enqueues an element according to priority" do + queue = [%{item: 1, priority: 2}] + + new_queue = Federator.enqueue_sorted(queue, 2, 1) + assert new_queue == [%{item: 2, priority: 1}, %{item: 1, priority: 2}] + + new_queue = Federator.enqueue_sorted(queue, 2, 3) + assert new_queue == [%{item: 1, priority: 2}, %{item: 2, priority: 3}] + end + + test "pop first item" do + queue = [%{item: 2, priority: 1}, %{item: 1, priority: 2}] + + assert {2, [%{item: 1, priority: 2}]} = Federator.queue_pop(queue) + end +end From e31a5ff4af04156b9db8f032cf184e1da540f025 Mon Sep 17 00:00:00 2001 From: Lain Iwakura Date: Wed, 6 Dec 2017 16:51:11 +0100 Subject: [PATCH 2/8] Priority queue fixes. --- lib/pleroma/web/activity_pub/utils.ex | 7 ++++++- lib/pleroma/web/federator/federator.ex | 12 ++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 51fac6fe2..ac20a2822 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -29,7 +29,12 @@ def generate_id(type) do Enqueues an activity for federation if it's local """ def maybe_federate(%Activity{local: true} = activity) do - Pleroma.Web.Federator.enqueue(:publish, activity) + priority = case activity.data["type"] do + "Delete" -> 10 + "Create" -> 1 + _ -> 5 + end + Pleroma.Web.Federator.enqueue(:publish, activity, priority) :ok end def maybe_federate(_), do: :ok diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index f384b313c..b23ed5fcc 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -15,7 +15,7 @@ def start_link do enqueue(:refresh_subscriptions, nil) end) GenServer.start_link(__MODULE__, %{ - in: {:sets.new(), [], + in: {:sets.new(), []}, out: {:sets.new(), []} }, name: __MODULE__) end @@ -79,17 +79,17 @@ def handle(type, _) do {:error, "Don't know what do do with this"} end - def enqueue(type, payload) do + def enqueue(type, payload, priority \\ 1) do if Mix.env == :test do handle(type, payload) else - GenServer.cast(__MODULE__, {:enqueue, type, payload}) + GenServer.cast(__MODULE__, {:enqueue, type, payload, priority}) end end def maybe_start_job(running_jobs, queue) do if (:sets.size(running_jobs) < @max_jobs) && queue != [] do - {{:value, {type, payload}}, queue} = queue_pop(queue) + {{type, payload}, queue} = queue_pop(queue) {:ok, pid} = Task.start(fn -> handle(type, payload) end) mref = Process.monitor(pid) {:sets.add_element(mref, running_jobs), queue} @@ -98,14 +98,14 @@ def maybe_start_job(running_jobs, queue) do end end - def handle_cast({:enqueue, type, payload}, state) when type in [:incoming_doc] do + def handle_cast({:enqueue, type, payload, priority}, state) when type in [:incoming_doc] do %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}} = state i_queue = enqueue_sorted(i_queue, {type, payload}, 1) {i_running_jobs, i_queue} = maybe_start_job(i_running_jobs, i_queue) {:noreply, %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}}} end - def handle_cast({:enqueue, type, payload}, state) do + def handle_cast({:enqueue, type, payload, priority}, state) do %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}} = state o_queue = enqueue_sorted(o_queue, {type, payload}, 1) {o_running_jobs, o_queue} = maybe_start_job(o_running_jobs, o_queue) From 0ec5aeb8a76653935caefa0de92861269f98f343 Mon Sep 17 00:00:00 2001 From: Lain Iwakura Date: Thu, 7 Dec 2017 17:41:34 +0100 Subject: [PATCH 3/8] Don't log in deactivated users. --- lib/pleroma/plugs/authentication_plug.ex | 1 + lib/pleroma/plugs/oauth_plug.ex | 3 ++- test/plugs/authentication_plug_test.exs | 27 ++++++++++++++++++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/plugs/authentication_plug.ex b/lib/pleroma/plugs/authentication_plug.ex index beb02eb88..60f6faf49 100644 --- a/lib/pleroma/plugs/authentication_plug.ex +++ b/lib/pleroma/plugs/authentication_plug.ex @@ -12,6 +12,7 @@ def call(%{assigns: %{user: %User{}}} = conn, _), do: conn def call(conn, opts) do with {:ok, username, password} <- decode_header(conn), {:ok, user} <- opts[:fetcher].(username), + false <- !!user.info["deactivated"], saved_user_id <- get_session(conn, :user_id), {:ok, verified_user} <- verify(user, password, saved_user_id) do diff --git a/lib/pleroma/plugs/oauth_plug.ex b/lib/pleroma/plugs/oauth_plug.ex index 775423bb1..be737dc9a 100644 --- a/lib/pleroma/plugs/oauth_plug.ex +++ b/lib/pleroma/plugs/oauth_plug.ex @@ -16,7 +16,8 @@ def call(conn, _) do end with token when not is_nil(token) <- token, %Token{user_id: user_id} <- Repo.get_by(Token, token: token), - %User{} = user <- Repo.get(User, user_id) do + %User{} = user <- Repo.get(User, user_id), + false <- !!user.info["deactivated"] do conn |> assign(:user, user) else diff --git a/test/plugs/authentication_plug_test.exs b/test/plugs/authentication_plug_test.exs index 9d6c2cd70..5480dab43 100644 --- a/test/plugs/authentication_plug_test.exs +++ b/test/plugs/authentication_plug_test.exs @@ -14,6 +14,13 @@ defp fetch_nil(_name) do password_hash: Comeonin.Pbkdf2.hashpwsalt("guy") } + @deactivated %User{ + id: 1, + name: "dude", + password_hash: Comeonin.Pbkdf2.hashpwsalt("guy"), + info: %{"deactivated" => true} + } + @session_opts [ store: :cookie, key: "_test", @@ -131,6 +138,26 @@ test "it assigns the user", %{conn: conn} do end end + describe "with a correct authorization header for an deactiviated user" do + test "it halts the appication", %{conn: conn} do + opts = %{ + optional: false, + fetcher: fn _ -> @deactivated end + } + + header = basic_auth_enc("dude", "guy") + + conn = conn + |> Plug.Session.call(Plug.Session.init(@session_opts)) + |> fetch_session + |> put_req_header("authorization", header) + |> AuthenticationPlug.call(opts) + + assert conn.status == 403 + assert conn.halted == true + end + end + describe "with a user_id in the session for an existing user" do test "it assigns the user", %{conn: conn} do opts = %{ From b727ecc5e75388e2b2d67c76cab6df1a7e6719b3 Mon Sep 17 00:00:00 2001 From: Lain Iwakura Date: Thu, 7 Dec 2017 17:47:23 +0100 Subject: [PATCH 4/8] Add function to deactivate users. --- lib/pleroma/user.ex | 5 +++++ test/user_test.exs | 8 +++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index afc62f265..779a89a12 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -334,4 +334,9 @@ def local_user_query() do where: u.local == true end + def deactivate (%User{} = user) do + new_info = Map.put(user.info, "deactivated", true) + cs = User.info_changeset(user, %{info: new_info}) + Repo.update(cs) + end end diff --git a/test/user_test.exs b/test/user_test.exs index 31c5962e2..6daf1ecb7 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -325,5 +325,11 @@ test "get recipients from activity" do assert user in recipients assert addressed in recipients end -end + test ".deactivate deactivates a user" do + user = insert(:user) + assert false == !!user.info["deactivated"] + {:ok, user} = User.deactivate(user) + assert true == user.info["deactivated"] + end +end From a78ae2a685769cbf7459eb347aca581d389018ad Mon Sep 17 00:00:00 2001 From: Lain Iwakura Date: Thu, 7 Dec 2017 17:51:55 +0100 Subject: [PATCH 5/8] Don't follow deactivated users. --- lib/pleroma/user.ex | 4 ++-- test/user_test.exs | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 779a89a12..b21caba9d 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -142,9 +142,9 @@ def register_changeset(struct, params \\ %{}) do end end - def follow(%User{} = follower, %User{} = followed) do + def follow(%User{} = follower, %User{info: info} = followed) do ap_followers = followed.follower_address - if following?(follower, followed) do + if following?(follower, followed) or info["deactivated"] do {:error, "Could not follow user: #{followed.nickname} is already on your list."} else diff --git a/test/user_test.exs b/test/user_test.exs index 6daf1ecb7..486998907 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -39,6 +39,13 @@ test "follow takes a user and another user" do assert User.ap_followers(followed) in user.following end + test "can't follow a deactivated users" do + user = insert(:user) + followed = insert(:user, info: %{"deactivated" => true}) + + {:error, _} = User.follow(user, followed) + end + test "following a remote user will ensure a websub subscription is present" do user = insert(:user) {:ok, followed} = OStatus.make_user("shp@social.heldscal.la") From 6df6ad0b429150b00b065d95890bd62cd2778fad Mon Sep 17 00:00:00 2001 From: Lain Iwakura Date: Thu, 7 Dec 2017 18:13:05 +0100 Subject: [PATCH 6/8] User deletion: Remove relationships. --- lib/pleroma/user.ex | 15 +++++++++++++++ test/user_test.exs | 29 +++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index b21caba9d..021207de9 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -339,4 +339,19 @@ def deactivate (%User{} = user) do cs = User.info_changeset(user, %{info: new_info}) Repo.update(cs) end + + def delete (%User{} = user) do + {:ok, user} = User.deactivate(user) + + # Remove all relationships + {:ok, followers } = User.get_followers(user) + followers + |> Enum.each(fn (follower) -> User.unfollow(follower, user) end) + + {:ok, friends} = User.get_friends(user) + friends + |> Enum.each(fn (followed) -> User.unfollow(user, followed) end) + + :ok + end end diff --git a/test/user_test.exs b/test/user_test.exs index 486998907..0dd5afb18 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -339,4 +339,33 @@ test ".deactivate deactivates a user" do {:ok, user} = User.deactivate(user) assert true == user.info["deactivated"] end + + test ".delete deactivates a user, all follow relationships and all create activities" do + user = insert(:user) + followed = insert(:user) + follower = insert(:user) + + {:ok, user} = User.follow(user, followed) + {:ok, follower} = User.follow(follower, user) + + {:ok, activity} = CommonAPI.post(user, %{"status" => "2hu"}) + {:ok, activity_two} = CommonAPI.post(follower, %{"status" => "3hu"}) + + {:ok, _, _} = CommonAPI.favorite(activity_two.id, user) + {:ok, _, _} = CommonAPI.favorite(activity.id, follower) + {:ok, _, _} = CommonAPI.repeat(activity.id, follower) + + :ok = User.delete(user) + + followed = Repo.get(User, followed.id) + follower = Repo.get(User, follower.id) + user = Repo.get(User, user.id) + + assert user.info["deactivated"] + + refute User.following?(user, followed) + refute User.following?(followed, follower) + + # TODO: check for activities. + end end From bad499b3fd95d1643890feaedea07123c004be57 Mon Sep 17 00:00:00 2001 From: Lain Iwakura Date: Fri, 8 Dec 2017 17:50:11 +0100 Subject: [PATCH 7/8] Basic user deletion. --- lib/pleroma/user.ex | 42 ++++++++++++++++++++++++++++++++---------- test/user_test.exs | 6 ++++-- 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 021207de9..09bcf0cb4 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -5,7 +5,7 @@ defmodule Pleroma.User do alias Pleroma.{Repo, User, Object, Web, Activity, Notification} alias Comeonin.Pbkdf2 alias Pleroma.Web.{OStatus, Websub} - alias Pleroma.Web.ActivityPub.Utils + alias Pleroma.Web.ActivityPub.{Utils, ActivityPub} schema "users" do field :bio, :string @@ -113,7 +113,7 @@ def password_update_changeset(struct, params) do end def reset_password(user, data) do - Repo.update(password_update_changeset(user, data)) + update_and_set_cache(password_update_changeset(user, data)) end def register_changeset(struct, params \\ %{}) do @@ -157,7 +157,7 @@ def follow(%User{} = follower, %User{info: info} = followed) do follower = follower |> follow_changeset(%{following: following}) - |> Repo.update + |> update_and_set_cache {:ok, _} = update_follower_count(followed) @@ -173,7 +173,7 @@ def unfollow(%User{} = follower, %User{} = followed) do { :ok, follower } = follower |> follow_changeset(%{following: following}) - |> Repo.update + |> update_and_set_cache {:ok, followed} = update_follower_count(followed) @@ -191,6 +191,17 @@ def get_by_ap_id(ap_id) do Repo.get_by(User, ap_id: ap_id) end + def update_and_set_cache(changeset) do + with {:ok, user} <- Repo.update(changeset) do + Cachex.set(:user_cache, "ap_id:#{user.ap_id}", user) + Cachex.set(:user_cache, "nickname:#{user.nickname}", user) + Cachex.set(:user_cache, "user_info:#{user.id}", user_info(user)) + {:ok, user} + else + e -> e + end + end + def get_cached_by_ap_id(ap_id) do key = "ap_id:#{ap_id}" Cachex.get!(:user_cache, key, fallback: fn(_) -> get_by_ap_id(ap_id) end) @@ -245,7 +256,7 @@ def increase_note_count(%User{} = user) do cs = info_changeset(user, %{info: new_info}) - Repo.update(cs) + update_and_set_cache(cs) end def update_note_count(%User{} = user) do @@ -259,7 +270,7 @@ def update_note_count(%User{} = user) do cs = info_changeset(user, %{info: new_info}) - Repo.update(cs) + update_and_set_cache(cs) end def update_follower_count(%User{} = user) do @@ -274,7 +285,7 @@ def update_follower_count(%User{} = user) do cs = info_changeset(user, %{info: new_info}) - Repo.update(cs) + update_and_set_cache(cs) end def get_notified_from_activity(%Activity{data: %{"to" => to}}) do @@ -312,7 +323,7 @@ def block(user, %{ap_id: ap_id}) do new_info = Map.put(user.info, "blocks", new_blocks) cs = User.info_changeset(user, %{info: new_info}) - Repo.update(cs) + update_and_set_cache(cs) end def unblock(user, %{ap_id: ap_id}) do @@ -321,7 +332,7 @@ def unblock(user, %{ap_id: ap_id}) do new_info = Map.put(user.info, "blocks", new_blocks) cs = User.info_changeset(user, %{info: new_info}) - Repo.update(cs) + update_and_set_cache(cs) end def blocks?(user, %{ap_id: ap_id}) do @@ -337,7 +348,7 @@ def local_user_query() do def deactivate (%User{} = user) do new_info = Map.put(user.info, "deactivated", true) cs = User.info_changeset(user, %{info: new_info}) - Repo.update(cs) + update_and_set_cache(cs) end def delete (%User{} = user) do @@ -352,6 +363,17 @@ def delete (%User{} = user) do friends |> Enum.each(fn (followed) -> User.unfollow(user, followed) end) + query = from a in Activity, + where: a.actor == ^user.ap_id + + Repo.all(query) + |> Enum.each(fn (activity) -> + case activity.data["type"] do + "Create" -> ActivityPub.delete(Object.get_by_ap_id(activity.data["object"]["id"])) + _ -> "Doing nothing" # TODO: Do something with likes, follows, repeats. + end + end) + :ok end end diff --git a/test/user_test.exs b/test/user_test.exs index 0dd5afb18..16d43e619 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -1,6 +1,6 @@ defmodule Pleroma.UserTest do alias Pleroma.Builders.UserBuilder - alias Pleroma.{User, Repo} + alias Pleroma.{User, Repo, Activity} alias Pleroma.Web.OStatus alias Pleroma.Web.Websub.WebsubClientSubscription alias Pleroma.Web.CommonAPI @@ -366,6 +366,8 @@ test ".delete deactivates a user, all follow relationships and all create activi refute User.following?(user, followed) refute User.following?(followed, follower) - # TODO: check for activities. + # TODO: Remove favorites, repeats, delete activities. + + refute Repo.get(Activity, activity.id) end end From ef2322bdb7d46882eeae0271c4381efda6835c3c Mon Sep 17 00:00:00 2001 From: Lain Iwakura Date: Fri, 8 Dec 2017 18:17:30 +0100 Subject: [PATCH 8/8] Delete notifications on activity deletion. --- lib/pleroma/activity.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index a35ccc9b4..afd09982f 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -7,7 +7,7 @@ defmodule Pleroma.Activity do field :data, :map field :local, :boolean, default: true field :actor, :string - has_many :notifications, Notification + has_many :notifications, Notification, on_delete: :delete_all timestamps() end