From 75290cbfbd0f93cf3ecec5f44c4624b8c8601c51 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 26 Sep 2019 18:49:57 +0700 Subject: [PATCH 01/20] Add Pleroma.JobQueueMonitor --- lib/pleroma/application.ex | 1 + lib/pleroma/healthcheck.ex | 8 ++ lib/pleroma/job_queue_monitor.ex | 115 +++++++++++++++++++++++++++ lib/pleroma/workers/worker_helper.ex | 1 + test/healthcheck_test.exs | 9 ++- 5 files changed, 133 insertions(+), 1 deletion(-) create mode 100644 lib/pleroma/job_queue_monitor.ex diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 7aec2c545..3e21d4403 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -42,6 +42,7 @@ defmodule Pleroma.Application do hackney_pool_children() ++ [ Pleroma.Stats, + Pleroma.JobQueueMonitor, {Oban, Pleroma.Config.get(Oban)} ] ++ task_children(@env) ++ diff --git a/lib/pleroma/healthcheck.ex b/lib/pleroma/healthcheck.ex index 977b78c26..fc2129815 100644 --- a/lib/pleroma/healthcheck.ex +++ b/lib/pleroma/healthcheck.ex @@ -14,6 +14,7 @@ defmodule Pleroma.Healthcheck do active: 0, idle: 0, memory_used: 0, + job_queue_stats: nil, healthy: true @type t :: %__MODULE__{ @@ -21,6 +22,7 @@ defmodule Pleroma.Healthcheck do active: non_neg_integer(), idle: non_neg_integer(), memory_used: number(), + job_queue_stats: map(), healthy: boolean() } @@ -30,6 +32,7 @@ defmodule Pleroma.Healthcheck do memory_used: Float.round(:erlang.memory(:total) / 1024 / 1024, 2) } |> assign_db_info() + |> assign_job_queue_stats() |> check_health() end @@ -55,6 +58,11 @@ defmodule Pleroma.Healthcheck do Map.merge(healthcheck, db_info) end + defp assign_job_queue_stats(healthcheck) do + stats = Pleroma.JobQueueMonitor.stats() + Map.put(healthcheck, :job_queue_stats, stats) + end + @spec check_health(Healthcheck.t()) :: Healthcheck.t() def check_health(%{pool_size: pool_size, active: active} = check) when active >= pool_size do diff --git a/lib/pleroma/job_queue_monitor.ex b/lib/pleroma/job_queue_monitor.ex new file mode 100644 index 000000000..685ba2ead --- /dev/null +++ b/lib/pleroma/job_queue_monitor.ex @@ -0,0 +1,115 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.JobQueueMonitor do + use GenServer + + @initial_state %{workers: %{}, queues: %{}, processed_jobs: 0, enqueued: 0} + @queue %{processed_jobs: 0, success: 0, failure: 0, enqueued: 0} + @operation %{processed_jobs: 0, success: 0, failure: 0, enqueued: 0} + + def start_link(_) do + GenServer.start_link(__MODULE__, @initial_state, name: __MODULE__) + end + + @impl true + def init(state) do + :telemetry.attach("oban-monitor-failure", [:oban, :failure], &handle_event/4, nil) + :telemetry.attach("oban-monitor-success", [:oban, :success], &handle_event/4, nil) + + {:ok, state} + end + + def stats do + GenServer.call(__MODULE__, :stats) + end + + def enqueue({:ok, job}) do + meta = Map.take(job, [:args, :queue, :worker]) + GenServer.cast(__MODULE__, {:process_enqueue, meta}) + + {:ok, job} + end + + def enqueue(result), do: result + + def handle_event([:oban, status], %{duration: duration}, meta, _) do + GenServer.cast(__MODULE__, {:process_event, status, duration, meta}) + end + + @impl true + def handle_call(:stats, _from, state) do + {:reply, state, state} + end + + def handle_cast({:process_enqueue, meta}, state) do + state = + state + |> Map.update!(:workers, fn workers -> + workers + |> Map.put_new(meta.worker, %{}) + |> Map.update!(meta.worker, &update_worker(&1, :enqueue, meta)) + end) + |> Map.update!(:queues, fn workers -> + workers + |> Map.put_new(meta.queue, @queue) + |> Map.update!(meta.queue, fn queue -> Map.update!(queue, :enqueued, &(&1 + 1)) end) + end) + |> Map.update!(:enqueued, &(&1 + 1)) + + {:noreply, state} + end + + @impl true + def handle_cast({:process_event, status, duration, meta}, state) do + state = + state + |> Map.update!(:workers, fn workers -> + workers + |> Map.put_new(meta.worker, %{}) + |> Map.update!(meta.worker, &update_worker(&1, status, meta, duration)) + end) + |> Map.update!(:queues, fn workers -> + workers + |> Map.put_new(meta.queue, @queue) + |> Map.update!(meta.queue, &update_queue(&1, status, meta, duration)) + end) + |> Map.update!(:processed_jobs, &(&1 + 1)) + |> decr_enqueued() + + {:noreply, state} + end + + defp update_worker(worker, status, meta, duration \\ 0) do + worker + |> Map.put_new(meta.args["op"], @operation) + |> Map.update!(meta.args["op"], &update_op(&1, status, meta, duration)) + end + + defp update_op(op, :enqueue, _meta, _duration) do + op + |> Map.update!(:enqueued, &(&1 + 1)) + end + + defp update_op(op, status, _meta, _duration) do + op + |> Map.update!(:processed_jobs, &(&1 + 1)) + |> Map.update!(status, &(&1 + 1)) + |> decr_enqueued() + end + + defp update_queue(queue, status, _meta, _duration) do + queue + |> Map.update!(:processed_jobs, &(&1 + 1)) + |> Map.update!(status, &(&1 + 1)) + |> decr_enqueued() + end + + defp decr_enqueued(map) do + Map.update!(map, :enqueued, fn + 0 -> 0 + enqueued -> enqueued - 1 + end) + end +end diff --git a/lib/pleroma/workers/worker_helper.ex b/lib/pleroma/workers/worker_helper.ex index 358efa14a..a43ce8bc0 100644 --- a/lib/pleroma/workers/worker_helper.ex +++ b/lib/pleroma/workers/worker_helper.ex @@ -40,6 +40,7 @@ defmodule Pleroma.Workers.WorkerHelper do unquote(caller_module) |> apply(:new, [params, worker_args]) |> Pleroma.Repo.insert() + |> Pleroma.JobQueueMonitor.enqueue() end end end diff --git a/test/healthcheck_test.exs b/test/healthcheck_test.exs index 6bb8d5b7f..66d5026ff 100644 --- a/test/healthcheck_test.exs +++ b/test/healthcheck_test.exs @@ -9,7 +9,14 @@ defmodule Pleroma.HealthcheckTest do test "system_info/0" do result = Healthcheck.system_info() |> Map.from_struct() - assert Map.keys(result) == [:active, :healthy, :idle, :memory_used, :pool_size] + assert Map.keys(result) == [ + :active, + :healthy, + :idle, + :job_queue_stats, + :memory_used, + :pool_size + ] end describe "check_health/1" do From 26693292f4b2062504fc9a24e824a6f56cb6b555 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 2 Oct 2019 14:50:25 +0700 Subject: [PATCH 02/20] Remove `:enqueued` counter --- lib/pleroma/job_queue_monitor.ex | 45 +++------------------------- lib/pleroma/workers/worker_helper.ex | 1 - 2 files changed, 4 insertions(+), 42 deletions(-) diff --git a/lib/pleroma/job_queue_monitor.ex b/lib/pleroma/job_queue_monitor.ex index 685ba2ead..3feea8381 100644 --- a/lib/pleroma/job_queue_monitor.ex +++ b/lib/pleroma/job_queue_monitor.ex @@ -5,9 +5,9 @@ defmodule Pleroma.JobQueueMonitor do use GenServer - @initial_state %{workers: %{}, queues: %{}, processed_jobs: 0, enqueued: 0} - @queue %{processed_jobs: 0, success: 0, failure: 0, enqueued: 0} - @operation %{processed_jobs: 0, success: 0, failure: 0, enqueued: 0} + @initial_state %{workers: %{}, queues: %{}, processed_jobs: 0} + @queue %{processed_jobs: 0, success: 0, failure: 0} + @operation %{processed_jobs: 0, success: 0, failure: 0} def start_link(_) do GenServer.start_link(__MODULE__, @initial_state, name: __MODULE__) @@ -25,15 +25,6 @@ defmodule Pleroma.JobQueueMonitor do GenServer.call(__MODULE__, :stats) end - def enqueue({:ok, job}) do - meta = Map.take(job, [:args, :queue, :worker]) - GenServer.cast(__MODULE__, {:process_enqueue, meta}) - - {:ok, job} - end - - def enqueue(result), do: result - def handle_event([:oban, status], %{duration: duration}, meta, _) do GenServer.cast(__MODULE__, {:process_event, status, duration, meta}) end @@ -43,24 +34,6 @@ defmodule Pleroma.JobQueueMonitor do {:reply, state, state} end - def handle_cast({:process_enqueue, meta}, state) do - state = - state - |> Map.update!(:workers, fn workers -> - workers - |> Map.put_new(meta.worker, %{}) - |> Map.update!(meta.worker, &update_worker(&1, :enqueue, meta)) - end) - |> Map.update!(:queues, fn workers -> - workers - |> Map.put_new(meta.queue, @queue) - |> Map.update!(meta.queue, fn queue -> Map.update!(queue, :enqueued, &(&1 + 1)) end) - end) - |> Map.update!(:enqueued, &(&1 + 1)) - - {:noreply, state} - end - @impl true def handle_cast({:process_event, status, duration, meta}, state) do state = @@ -76,12 +49,11 @@ defmodule Pleroma.JobQueueMonitor do |> Map.update!(meta.queue, &update_queue(&1, status, meta, duration)) end) |> Map.update!(:processed_jobs, &(&1 + 1)) - |> decr_enqueued() {:noreply, state} end - defp update_worker(worker, status, meta, duration \\ 0) do + defp update_worker(worker, status, meta, duration) do worker |> Map.put_new(meta.args["op"], @operation) |> Map.update!(meta.args["op"], &update_op(&1, status, meta, duration)) @@ -96,20 +68,11 @@ defmodule Pleroma.JobQueueMonitor do op |> Map.update!(:processed_jobs, &(&1 + 1)) |> Map.update!(status, &(&1 + 1)) - |> decr_enqueued() end defp update_queue(queue, status, _meta, _duration) do queue |> Map.update!(:processed_jobs, &(&1 + 1)) |> Map.update!(status, &(&1 + 1)) - |> decr_enqueued() - end - - defp decr_enqueued(map) do - Map.update!(map, :enqueued, fn - 0 -> 0 - enqueued -> enqueued - 1 - end) end end diff --git a/lib/pleroma/workers/worker_helper.ex b/lib/pleroma/workers/worker_helper.ex index a43ce8bc0..358efa14a 100644 --- a/lib/pleroma/workers/worker_helper.ex +++ b/lib/pleroma/workers/worker_helper.ex @@ -40,7 +40,6 @@ defmodule Pleroma.Workers.WorkerHelper do unquote(caller_module) |> apply(:new, [params, worker_args]) |> Pleroma.Repo.insert() - |> Pleroma.JobQueueMonitor.enqueue() end end end From 93f966ea4bb2b4d551cb3e248150809554deddc8 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 2 Oct 2019 14:51:30 +0700 Subject: [PATCH 03/20] Update CHANGELOG and pleroma_api.md --- CHANGELOG.md | 1 + docs/api/pleroma_api.md | 9 +++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a76e6cf8..0a88f9d75 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [Unreleased] ### Added - Refreshing poll results for remote polls +- Job queue stats to the healthcheck page - Admin API: Add ability to require password reset ### Changed diff --git a/docs/api/pleroma_api.md b/docs/api/pleroma_api.md index a469ddfbf..dfd6623ec 100644 --- a/docs/api/pleroma_api.md +++ b/docs/api/pleroma_api.md @@ -317,7 +317,8 @@ See [Admin-API](Admin-API.md) "active": 0, # active processes "idle": 0, # idle processes "memory_used": 0.00, # Memory used - "healthy": true # Instance state + "healthy": true, # Instance state + "job_queue_stats": {} # Job queue stats } ``` @@ -391,7 +392,7 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa ### Update a file in a custom emoji pack * Method `POST` * Authentication: required -* Params: +* Params: * if the `action` is `add`, adds an emoji named `shortcode` to the pack `pack_name`, that means that the emoji file needs to be uploaded with the request (thus requiring it to be a multipart request) and be named `file`. @@ -408,7 +409,7 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa ### Updates (replaces) pack metadata * Method `POST` * Authentication: required -* Params: +* Params: * `new_data`: new metadata to replace the old one * Response: JSON, updated "metadata" section of the pack and 200 status or 400 if there was a problem with the new metadata (the error is specified in the "error" part of the response JSON) @@ -417,7 +418,7 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa ### Requests the instance to download the pack from another instance * Method `POST` * Authentication: required -* Params: +* Params: * `instance_address`: the address of the instance to download from * `pack_name`: the pack to download from that instance * Response: JSON, "ok" and 200 status if the pack was downloaded, or 500 if there were From 0fc29deba06b6a897f3534ce68abfdadcab12a6b Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 2 Oct 2019 15:24:21 +0700 Subject: [PATCH 04/20] Add tests for Pleroma.JobQueueMonitor --- test/job_queue_monitor_test.exs | 70 +++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 test/job_queue_monitor_test.exs diff --git a/test/job_queue_monitor_test.exs b/test/job_queue_monitor_test.exs new file mode 100644 index 000000000..17c6f3246 --- /dev/null +++ b/test/job_queue_monitor_test.exs @@ -0,0 +1,70 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.JobQueueMonitorTest do + use ExUnit.Case, async: true + + alias Pleroma.JobQueueMonitor + + @success {:process_event, :success, 1337, + %{ + args: %{"op" => "refresh_subscriptions"}, + attempt: 1, + id: 339, + max_attempts: 5, + queue: "federator_outgoing", + worker: "Pleroma.Workers.SubscriberWorker" + }} + + @failure {:process_event, :failure, 22_521_134, + %{ + args: %{"op" => "force_password_reset", "user_id" => "9nJG6n6Nbu7tj9GJX6"}, + attempt: 1, + error: %RuntimeError{message: "oops"}, + id: 345, + kind: :exception, + max_attempts: 1, + queue: "background", + stack: [ + {Pleroma.Workers.BackgroundWorker, :perform, 2, + [file: 'lib/pleroma/workers/background_worker.ex', line: 31]}, + {Oban.Queue.Executor, :safe_call, 1, + [file: 'lib/oban/queue/executor.ex', line: 42]}, + {:timer, :tc, 3, [file: 'timer.erl', line: 197]}, + {Oban.Queue.Executor, :call, 2, [file: 'lib/oban/queue/executor.ex', line: 23]}, + {Task.Supervised, :invoke_mfa, 2, [file: 'lib/task/supervised.ex', line: 90]}, + {:proc_lib, :init_p_do_apply, 3, [file: 'proc_lib.erl', line: 249]} + ], + worker: "Pleroma.Workers.BackgroundWorker" + }} + + test "stats/0" do + assert %{processed_jobs: _, queues: _, workers: _} = JobQueueMonitor.stats() + end + + test "handle_cast/2" do + state = %{workers: %{}, queues: %{}, processed_jobs: 0} + + assert {:noreply, state} = JobQueueMonitor.handle_cast(@success, state) + assert {:noreply, state} = JobQueueMonitor.handle_cast(@failure, state) + assert {:noreply, state} = JobQueueMonitor.handle_cast(@success, state) + assert {:noreply, state} = JobQueueMonitor.handle_cast(@failure, state) + + assert state == %{ + processed_jobs: 4, + queues: %{ + "background" => %{failure: 2, processed_jobs: 2, success: 0}, + "federator_outgoing" => %{failure: 0, processed_jobs: 2, success: 2} + }, + workers: %{ + "Pleroma.Workers.BackgroundWorker" => %{ + "force_password_reset" => %{failure: 2, processed_jobs: 2, success: 0} + }, + "Pleroma.Workers.SubscriberWorker" => %{ + "refresh_subscriptions" => %{failure: 0, processed_jobs: 2, success: 2} + } + } + } + end +end From b8b98ac40f042a8c3d2562edc095f0e1a309760f Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 3 Oct 2019 10:40:49 -0500 Subject: [PATCH 05/20] Add missing extended_nickname_format setting to the default config --- CHANGELOG.md | 1 + config/config.exs | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a71a9dae6..c22995b82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Fixed - Mastodon API: Fix private and direct statuses not being filtered out from the public timeline for an authenticated user (`GET /api/v1/timelines/public`) +- Added `:instance, extended_nickname_format` setting to the default config ## [1.1.0] - 2019-??-?? ### Security diff --git a/config/config.exs b/config/config.exs index 36bea19a0..ddbfb246a 100644 --- a/config/config.exs +++ b/config/config.exs @@ -279,7 +279,8 @@ config :pleroma, :instance, max_remote_account_fields: 20, account_field_name_length: 512, account_field_value_length: 2048, - external_user_synchronization: true + external_user_synchronization: true, + extended_nickname_format: false config :pleroma, :markup, # XXX - unfortunately, inline images must be enabled by default right now, because From 06d9df79c5c99069dd12e863c99167eb20b6495b Mon Sep 17 00:00:00 2001 From: eugenijm Date: Wed, 2 Oct 2019 00:37:08 +0300 Subject: [PATCH 06/20] Mastodon API: Add `pleroma.unread_conversation_count` to the Account entity --- CHANGELOG.md | 1 + docs/API/differences_in_mastoapi_responses.md | 1 + lib/pleroma/conversation.ex | 2 + lib/pleroma/conversation/participation.ex | 17 +++ lib/pleroma/user.ex | 56 +++++++++ lib/pleroma/user/info.ex | 1 + .../web/mastodon_api/views/account_view.ex | 11 ++ test/conversation/participation_test.exs | 4 + .../conversation_controller_test.exs | 109 +++++++++++++++--- .../mastodon_api/views/account_view_test.exs | 21 ++++ .../pleroma_api_controller_test.exs | 2 + 11 files changed, 211 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a71a9dae6..a38f61fba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Pleroma API: `GET /api/v1/pleroma/accounts/:id/scrobbles` to get a list of recently scrobbled items - Pleroma API: `POST /api/v1/pleroma/scrobble` to scrobble a media item - Mastodon API: Add `upload_limit`, `avatar_upload_limit`, `background_upload_limit`, and `banner_upload_limit` to `/api/v1/instance` +- Mastodon API: Add `pleroma.unread_conversation_count` to the Account entity ### Changed - **Breaking:** Elixir >=1.8 is now required (was >= 1.7) diff --git a/docs/API/differences_in_mastoapi_responses.md b/docs/API/differences_in_mastoapi_responses.md index d007a69c3..21b297529 100644 --- a/docs/API/differences_in_mastoapi_responses.md +++ b/docs/API/differences_in_mastoapi_responses.md @@ -56,6 +56,7 @@ Has these additional fields under the `pleroma` object: - `settings_store`: A generic map of settings for frontends. Opaque to the backend. Only returned in `verify_credentials` and `update_credentials` - `chat_token`: The token needed for Pleroma chat. Only returned in `verify_credentials` - `deactivated`: boolean, true when the user is deactivated +- `unread_conversation_count`: The count of unread conversations. Only returned to the account owner. ### Source diff --git a/lib/pleroma/conversation.ex b/lib/pleroma/conversation.ex index be5821ad7..098016af2 100644 --- a/lib/pleroma/conversation.ex +++ b/lib/pleroma/conversation.ex @@ -67,6 +67,8 @@ defmodule Pleroma.Conversation do participations = Enum.map(users, fn user -> + User.increment_unread_conversation_count(conversation, user) + {:ok, participation} = Participation.create_for_user_and_conversation(user, conversation, opts) diff --git a/lib/pleroma/conversation/participation.ex b/lib/pleroma/conversation/participation.ex index e946f6de2..ab81f3217 100644 --- a/lib/pleroma/conversation/participation.ex +++ b/lib/pleroma/conversation/participation.ex @@ -52,6 +52,15 @@ defmodule Pleroma.Conversation.Participation do participation |> read_cng(%{read: true}) |> Repo.update() + |> case do + {:ok, participation} -> + participation = Repo.preload(participation, :user) + User.set_unread_conversation_count(participation.user) + {:ok, participation} + + error -> + error + end end def mark_as_unread(participation) do @@ -135,4 +144,12 @@ defmodule Pleroma.Conversation.Participation do {:ok, Repo.preload(participation, :recipients, force: true)} end + + def unread_conversation_count_for_user(user) do + from(p in __MODULE__, + where: p.user_id == ^user.id, + where: not p.read, + select: %{count: count(p.id)} + ) + end end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 4c1cdd042..572dd7746 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -11,6 +11,7 @@ defmodule Pleroma.User do alias Comeonin.Pbkdf2 alias Ecto.Multi alias Pleroma.Activity + alias Pleroma.Conversation.Participation alias Pleroma.Delivery alias Pleroma.Keys alias Pleroma.Notification @@ -842,6 +843,61 @@ defmodule Pleroma.User do def maybe_update_following_count(user), do: user + def set_unread_conversation_count(%User{local: true} = user) do + unread_query = Participation.unread_conversation_count_for_user(user) + + User + |> where([u], u.id == ^user.id) + |> join(:inner, [u], p in subquery(unread_query)) + |> update([u, p], + set: [ + info: + fragment( + "jsonb_set(?, '{unread_conversation_count}', ?::varchar::jsonb, true)", + u.info, + p.count + ) + ] + ) + |> select([u], u) + |> Repo.update_all([]) + |> case do + {1, [%{info: %User.Info{}} = user]} -> set_cache(user) + _ -> {:error, user} + end + end + + def set_unread_conversation_count(_), do: :noop + + def increment_unread_conversation_count(conversation, %User{local: true} = user) do + unread_query = + Participation.unread_conversation_count_for_user(user) + |> where([p], p.conversation_id == ^conversation.id) + + User + |> join(:inner, [u], p in subquery(unread_query)) + |> update([u, p], + set: [ + info: + fragment( + "jsonb_set(?, '{unread_conversation_count}', ((?->>'unread_conversation_count')::int + 1)::varchar::jsonb, true)", + u.info, + u.info + ) + ] + ) + |> where([u], u.id == ^user.id) + |> where([u, p], p.count == 0) + |> select([u], u) + |> Repo.update_all([]) + |> case do + {1, [%{info: %User.Info{}} = user]} -> set_cache(user) + _ -> {:error, user} + end + end + + def increment_unread_conversation_count(_, _), do: :noop + def remove_duplicated_following(%User{following: following} = user) do uniq_following = Enum.uniq(following) diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index ebd4ddebf..4b5b43d7f 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -47,6 +47,7 @@ defmodule Pleroma.User.Info do field(:hide_followers, :boolean, default: false) field(:hide_follows, :boolean, default: false) field(:hide_favorites, :boolean, default: true) + field(:unread_conversation_count, :integer, default: 0) field(:pinned_activities, {:array, :string}, default: []) field(:email_notifications, :map, default: %{"digest" => false}) field(:mascot, :map, default: nil) diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 99169ef95..2d4976891 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -167,6 +167,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do |> maybe_put_chat_token(user, opts[:for], opts) |> maybe_put_activation_status(user, opts[:for]) |> maybe_put_follow_requests_count(user, opts[:for]) + |> maybe_put_unread_conversation_count(user, opts[:for]) end defp username_from_nickname(string) when is_binary(string) do @@ -248,6 +249,16 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do defp maybe_put_activation_status(data, _, _), do: data + defp maybe_put_unread_conversation_count(data, %User{id: user_id} = user, %User{id: user_id}) do + data + |> Kernel.put_in( + [:pleroma, :unread_conversation_count], + user.info.unread_conversation_count + ) + end + + defp maybe_put_unread_conversation_count(data, _, _), do: data + defp image_url(%{"url" => [%{"href" => href} | _]}), do: href defp image_url(_), do: nil end diff --git a/test/conversation/participation_test.exs b/test/conversation/participation_test.exs index a27167d42..f430bdf75 100644 --- a/test/conversation/participation_test.exs +++ b/test/conversation/participation_test.exs @@ -6,6 +6,7 @@ defmodule Pleroma.Conversation.ParticipationTest do use Pleroma.DataCase import Pleroma.Factory alias Pleroma.Conversation.Participation + alias Pleroma.User alias Pleroma.Web.CommonAPI test "getting a participation will also preload things" do @@ -30,6 +31,8 @@ defmodule Pleroma.Conversation.ParticipationTest do {:ok, activity} = CommonAPI.post(user, %{"status" => "Hey @#{other_user.nickname}.", "visibility" => "direct"}) + user = User.get_cached_by_id(user.id) + other_user = User.get_cached_by_id(user.id) [participation] = Participation.for_user(user) participation = Pleroma.Repo.preload(participation, :recipients) @@ -155,6 +158,7 @@ defmodule Pleroma.Conversation.ParticipationTest do [participation] = Participation.for_user_with_last_activity_id(user) participation = Repo.preload(participation, :recipients) + user = User.get_cached_by_id(user.id) assert participation.recipients |> length() == 1 assert user in participation.recipients diff --git a/test/web/mastodon_api/controllers/conversation_controller_test.exs b/test/web/mastodon_api/controllers/conversation_controller_test.exs index 7117fc76a..a308a7620 100644 --- a/test/web/mastodon_api/controllers/conversation_controller_test.exs +++ b/test/web/mastodon_api/controllers/conversation_controller_test.exs @@ -10,19 +10,23 @@ defmodule Pleroma.Web.MastodonAPI.ConversationControllerTest do import Pleroma.Factory - test "Conversations", %{conn: conn} do + test "returns a list of conversations", %{conn: conn} do user_one = insert(:user) user_two = insert(:user) user_three = insert(:user) {:ok, user_two} = User.follow(user_two, user_one) + assert User.get_cached_by_id(user_two.id).info.unread_conversation_count == 0 + {:ok, direct} = CommonAPI.post(user_one, %{ "status" => "Hi @#{user_two.nickname}, @#{user_three.nickname}!", "visibility" => "direct" }) + assert User.get_cached_by_id(user_two.id).info.unread_conversation_count == 1 + {:ok, _follower_only} = CommonAPI.post(user_one, %{ "status" => "Hi @#{user_two.nickname}!", @@ -52,23 +56,100 @@ defmodule Pleroma.Web.MastodonAPI.ConversationControllerTest do assert is_binary(res_id) assert unread == true assert res_last_status["id"] == direct.id + assert User.get_cached_by_id(user_one.id).info.unread_conversation_count == 1 + end + + test "updates the last_status on reply", %{conn: conn} do + user_one = insert(:user) + user_two = insert(:user) + + {:ok, direct} = + CommonAPI.post(user_one, %{ + "status" => "Hi @#{user_two.nickname}", + "visibility" => "direct" + }) + + {:ok, direct_reply} = + CommonAPI.post(user_two, %{ + "status" => "reply", + "visibility" => "direct", + "in_reply_to_status_id" => direct.id + }) + + [%{"last_status" => res_last_status}] = + conn + |> assign(:user, user_one) + |> get("/api/v1/conversations") + |> json_response(200) + + assert res_last_status["id"] == direct_reply.id + end + + test "the user marks a conversation as read", %{conn: conn} do + user_one = insert(:user) + user_two = insert(:user) + + {:ok, direct} = + CommonAPI.post(user_one, %{ + "status" => "Hi @#{user_two.nickname}", + "visibility" => "direct" + }) + + [%{"id" => direct_conversation_id, "unread" => true}] = + conn + |> assign(:user, user_one) + |> get("/api/v1/conversations") + |> json_response(200) + + %{"unread" => false} = + conn + |> assign(:user, user_one) + |> post("/api/v1/conversations/#{direct_conversation_id}/read") + |> json_response(200) + + assert User.get_cached_by_id(user_one.id).info.unread_conversation_count == 0 + + # The conversation is marked as unread on reply + {:ok, _} = + CommonAPI.post(user_two, %{ + "status" => "reply", + "visibility" => "direct", + "in_reply_to_status_id" => direct.id + }) + + [%{"unread" => true}] = + conn + |> assign(:user, user_one) + |> get("/api/v1/conversations") + |> json_response(200) + + assert User.get_cached_by_id(user_one.id).info.unread_conversation_count == 1 + + # A reply doesn't increment the user's unread_conversation_count if the conversation is unread + {:ok, _} = + CommonAPI.post(user_two, %{ + "status" => "reply", + "visibility" => "direct", + "in_reply_to_status_id" => direct.id + }) + + assert User.get_cached_by_id(user_one.id).info.unread_conversation_count == 1 + end + + test "(vanilla) Mastodon frontend behaviour", %{conn: conn} do + user_one = insert(:user) + user_two = insert(:user) + + {:ok, direct} = + CommonAPI.post(user_one, %{ + "status" => "Hi @#{user_two.nickname}!", + "visibility" => "direct" + }) - # Apparently undocumented API endpoint res_conn = conn |> assign(:user, user_one) - |> post("/api/v1/conversations/#{res_id}/read") - - assert response = json_response(res_conn, 200) - assert length(response["accounts"]) == 2 - assert response["last_status"]["id"] == direct.id - assert response["unread"] == false - - # (vanilla) Mastodon frontend behaviour - res_conn = - conn - |> assign(:user, user_one) - |> get("/api/v1/statuses/#{res_last_status["id"]}/context") + |> get("/api/v1/statuses/#{direct.id}/context") assert %{"ancestors" => [], "descendants" => []} == json_response(res_conn, 200) end diff --git a/test/web/mastodon_api/views/account_view_test.exs b/test/web/mastodon_api/views/account_view_test.exs index 62b2ab7e3..b7a4938a6 100644 --- a/test/web/mastodon_api/views/account_view_test.exs +++ b/test/web/mastodon_api/views/account_view_test.exs @@ -418,6 +418,27 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do following_count: 1 } = AccountView.render("show.json", %{user: user, for: user}) end + + test "shows unread_conversation_count only to the account owner" do + user = insert(:user) + other_user = insert(:user) + + {:ok, _activity} = + CommonAPI.post(user, %{ + "status" => "Hey @#{other_user.nickname}.", + "visibility" => "direct" + }) + + user = User.get_cached_by_ap_id(user.ap_id) + + assert AccountView.render("show.json", %{user: user, for: other_user})[:pleroma][ + :unread_conversation_count + ] == nil + + assert AccountView.render("show.json", %{user: user, for: user})[:pleroma][ + :unread_conversation_count + ] == 1 + end end describe "follow requests counter" do diff --git a/test/web/pleroma_api/controllers/pleroma_api_controller_test.exs b/test/web/pleroma_api/controllers/pleroma_api_controller_test.exs index 7eaeda4a0..8a6528cbb 100644 --- a/test/web/pleroma_api/controllers/pleroma_api_controller_test.exs +++ b/test/web/pleroma_api/controllers/pleroma_api_controller_test.exs @@ -8,6 +8,7 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIControllerTest do alias Pleroma.Conversation.Participation alias Pleroma.Notification alias Pleroma.Repo + alias Pleroma.User alias Pleroma.Web.CommonAPI import Pleroma.Factory @@ -73,6 +74,7 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIControllerTest do participation = Repo.preload(participation, :recipients) + user = User.get_cached_by_id(user.id) assert [user] == participation.recipients assert other_user not in participation.recipients From 5dc14c89cecc121ffb047c2a7c972af0b0f89ef8 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 4 Oct 2019 06:47:36 +0200 Subject: [PATCH 07/20] =?UTF-8?q?notification=5Fview.ex:=20Make=20sure=20`?= =?UTF-8?q?account`=20isn=E2=80=99t=20empty?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Related: https://git.pleroma.social/pleroma/pleroma/issues/1203 --- .../mastodon_api/views/notification_view.ex | 60 ++++++++++--------- .../views/notification_view_test.exs | 6 ++ 2 files changed, 38 insertions(+), 28 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex index 60b58dc90..5e3dbe728 100644 --- a/lib/pleroma/web/mastodon_api/views/notification_view.ex +++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex @@ -25,40 +25,44 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do parent_activity = Activity.get_create_by_object_ap_id(activity.data["object"]) mastodon_type = Activity.mastodon_notification_type(activity) - response = %{ - id: to_string(notification.id), - type: mastodon_type, - created_at: CommonAPI.Utils.to_masto_date(notification.inserted_at), - account: AccountView.render("show.json", %{user: actor, for: user}), - pleroma: %{ - is_seen: notification.seen + with %{id: _} = account <- AccountView.render("show.json", %{user: actor, for: user}) do + response = %{ + id: to_string(notification.id), + type: mastodon_type, + created_at: CommonAPI.Utils.to_masto_date(notification.inserted_at), + account: account, + pleroma: %{ + is_seen: notification.seen + } } - } - case mastodon_type do - "mention" -> - response - |> Map.merge(%{ - status: StatusView.render("show.json", %{activity: activity, for: user}) - }) + case mastodon_type do + "mention" -> + response + |> Map.merge(%{ + status: StatusView.render("show.json", %{activity: activity, for: user}) + }) - "favourite" -> - response - |> Map.merge(%{ - status: StatusView.render("show.json", %{activity: parent_activity, for: user}) - }) + "favourite" -> + response + |> Map.merge(%{ + status: StatusView.render("show.json", %{activity: parent_activity, for: user}) + }) - "reblog" -> - response - |> Map.merge(%{ - status: StatusView.render("show.json", %{activity: parent_activity, for: user}) - }) + "reblog" -> + response + |> Map.merge(%{ + status: StatusView.render("show.json", %{activity: parent_activity, for: user}) + }) - "follow" -> - response + "follow" -> + response - _ -> - nil + _ -> + nil + end + else + _ -> nil end end end diff --git a/test/web/mastodon_api/views/notification_view_test.exs b/test/web/mastodon_api/views/notification_view_test.exs index 81ab82e2b..c9043a69a 100644 --- a/test/web/mastodon_api/views/notification_view_test.exs +++ b/test/web/mastodon_api/views/notification_view_test.exs @@ -100,5 +100,11 @@ defmodule Pleroma.Web.MastodonAPI.NotificationViewTest do NotificationView.render("index.json", %{notifications: [notification], for: followed}) assert [expected] == result + + User.perform(:delete, follower) + notification = Notification |> Repo.one() |> Repo.preload(:activity) + + assert [] == + NotificationView.render("index.json", %{notifications: [notification], for: followed}) end end From d3ac4e8083f254a6a0e329a5807c0973f55402f4 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Fri, 4 Oct 2019 13:30:46 +0700 Subject: [PATCH 08/20] Fix OAuthController --- lib/pleroma/web/oauth/oauth_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/oauth/oauth_controller.ex b/lib/pleroma/web/oauth/oauth_controller.ex index e418dc70d..1cd7294e7 100644 --- a/lib/pleroma/web/oauth/oauth_controller.ex +++ b/lib/pleroma/web/oauth/oauth_controller.ex @@ -460,7 +460,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do end # Special case: Local MastodonFE - defp redirect_uri(%Plug.Conn{} = conn, "."), do: mastodon_api_url(conn, :login) + defp redirect_uri(%Plug.Conn{} = conn, "."), do: auth_url(conn, :login) defp redirect_uri(%Plug.Conn{}, redirect_uri), do: redirect_uri From 821729208506980bc65e9671c5462b629dffddaa Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 4 Oct 2019 01:05:50 +0300 Subject: [PATCH 09/20] Fix get_cached_by_nickname_or_id not allowing to get local users by nickname Closes #1293 --- CHANGELOG.md | 1 + lib/pleroma/user.ex | 2 +- test/user_test.exs | 57 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a71a9dae6..f677611c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Fixed - Mastodon API: Fix private and direct statuses not being filtered out from the public timeline for an authenticated user (`GET /api/v1/timelines/public`) +- Mastodon API: Inability to get some local users by nickname in `/api/v1/accounts/:id_or_nickname` ## [1.1.0] - 2019-??-?? ### Security diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 4c1cdd042..c2f8fa0d7 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -583,7 +583,7 @@ defmodule Pleroma.User do is_integer(nickname_or_id) or FlakeId.flake_id?(nickname_or_id) -> get_cached_by_id(nickname_or_id) || get_cached_by_nickname(nickname_or_id) - restrict_to_local == false -> + restrict_to_local == false or not String.contains?(nickname_or_id, "@") -> get_cached_by_nickname(nickname_or_id) restrict_to_local == :unauthenticated and match?(%User{}, opts[:for]) -> diff --git a/test/user_test.exs b/test/user_test.exs index 126bd69e8..1bc853c94 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -1725,4 +1725,61 @@ defmodule Pleroma.UserTest do assert %{info: %{hide_follows: true}} = Repo.get(User, user.id) assert {:ok, %{info: %{hide_follows: true}}} = Cachex.get(:user_cache, "ap_id:#{user.ap_id}") end + + describe "get_cached_by_nickname_or_id" do + setup do + limit_to_local_content = Pleroma.Config.get([:instance, :limit_to_local_content]) + local_user = insert(:user) + remote_user = insert(:user, nickname: "nickname@example.com", local: false) + + on_exit(fn -> + Pleroma.Config.put([:instance, :limit_to_local_content], limit_to_local_content) + end) + + [local_user: local_user, remote_user: remote_user] + end + + test "allows getting remote users by id no matter what :limit_to_local_content is set to", %{ + remote_user: remote_user + } do + Pleroma.Config.put([:instance, :limit_to_local_content], false) + assert %User{} = User.get_cached_by_nickname_or_id(remote_user.id) + + Pleroma.Config.put([:instance, :limit_to_local_content], true) + assert %User{} = User.get_cached_by_nickname_or_id(remote_user.id) + + Pleroma.Config.put([:instance, :limit_to_local_content], :unauthenticated) + assert %User{} = User.get_cached_by_nickname_or_id(remote_user.id) + end + + test "disallows getting remote users by nickname without authentication when :limit_to_local_content is set to :unauthenticated", + %{remote_user: remote_user} do + Pleroma.Config.put([:instance, :limit_to_local_content], :unauthenticated) + assert nil == User.get_cached_by_nickname_or_id(remote_user.nickname) + end + + test "allows getting remote users by nickname with authentication when :limit_to_local_content is set to :unauthenticated", + %{remote_user: remote_user, local_user: local_user} do + Pleroma.Config.put([:instance, :limit_to_local_content], :unauthenticated) + assert %User{} = User.get_cached_by_nickname_or_id(remote_user.nickname, for: local_user) + end + + test "disallows getting remote users by nickname when :limit_to_local_content is set to true", + %{remote_user: remote_user} do + Pleroma.Config.put([:instance, :limit_to_local_content], true) + assert nil == User.get_cached_by_nickname_or_id(remote_user.nickname) + end + + test "allows getting local users by nickname no matter what :limit_to_local_content is set to", + %{local_user: local_user} do + Pleroma.Config.put([:instance, :limit_to_local_content], false) + assert %User{} = User.get_cached_by_nickname_or_id(local_user.nickname) + + Pleroma.Config.put([:instance, :limit_to_local_content], true) + assert %User{} = User.get_cached_by_nickname_or_id(local_user.nickname) + + Pleroma.Config.put([:instance, :limit_to_local_content], :unauthenticated) + assert %User{} = User.get_cached_by_nickname_or_id(local_user.nickname) + end + end end From 568a995d64c91f339989077da06e381b4b8cb070 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 4 Oct 2019 16:32:42 +0200 Subject: [PATCH 10/20] ActivityPub: Change addressing of Undo. --- lib/pleroma/web/activity_pub/utils.ex | 12 ++++++++---- test/web/activity_pub/activity_pub_test.exs | 5 +++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 0828591ee..ac5550671 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -461,14 +461,16 @@ defmodule Pleroma.Web.ActivityPub.Utils do """ def make_unannounce_data( %User{ap_id: ap_id} = user, - %Activity{data: %{"context" => context}} = activity, + %Activity{data: %{"context" => context, "object" => object}} = activity, activity_id ) do + object = Object.normalize(object) + %{ "type" => "Undo", "actor" => ap_id, "object" => activity.data, - "to" => [user.follower_address, activity.data["actor"]], + "to" => [user.follower_address, object.data["actor"]], "cc" => [Pleroma.Constants.as_public()], "context" => context } @@ -477,14 +479,16 @@ defmodule Pleroma.Web.ActivityPub.Utils do def make_unlike_data( %User{ap_id: ap_id} = user, - %Activity{data: %{"context" => context}} = activity, + %Activity{data: %{"context" => context, "object" => object}} = activity, activity_id ) do + object = Object.normalize(object) + %{ "type" => "Undo", "actor" => ap_id, "object" => activity.data, - "to" => [user.follower_address, activity.data["actor"]], + "to" => [user.follower_address, object.data["actor"]], "cc" => [Pleroma.Constants.as_public()], "context" => context } diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index f29497847..c9f2a92e7 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -811,10 +811,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do {:ok, like_activity, object} = ActivityPub.like(user, object) assert object.data["like_count"] == 1 - {:ok, _, _, object} = ActivityPub.unlike(user, object) + {:ok, unlike_activity, _, object} = ActivityPub.unlike(user, object) assert object.data["like_count"] == 0 assert Activity.get_by_id(like_activity.id) == nil + assert note_activity.actor in unlike_activity.recipients end end @@ -890,7 +891,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do assert unannounce_activity.data["to"] == [ User.ap_followers(user), - announce_activity.data["actor"] + object.data["actor"] ] assert unannounce_activity.data["type"] == "Undo" From 2a7f44acfe7075947982546f3dfef61a9cbe45e9 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 4 Oct 2019 17:10:49 +0200 Subject: [PATCH 11/20] ActivityPub.Utils: Fix undo test. --- test/web/activity_pub/utils_test.exs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/web/activity_pub/utils_test.exs b/test/web/activity_pub/utils_test.exs index b1c1d6f71..c57ea7eb9 100644 --- a/test/web/activity_pub/utils_test.exs +++ b/test/web/activity_pub/utils_test.exs @@ -106,11 +106,13 @@ defmodule Pleroma.Web.ActivityPub.UtilsTest do user = insert(:user) like_activity = insert(:like_activity, data_attrs: %{"context" => "test context"}) + object = Object.normalize(like_activity.data["object"]) + assert Utils.make_unlike_data(user, like_activity, nil) == %{ "type" => "Undo", "actor" => user.ap_id, "object" => like_activity.data, - "to" => [user.follower_address, like_activity.data["actor"]], + "to" => [user.follower_address, object.data["actor"]], "cc" => [Pleroma.Constants.as_public()], "context" => like_activity.data["context"] } @@ -119,7 +121,7 @@ defmodule Pleroma.Web.ActivityPub.UtilsTest do "type" => "Undo", "actor" => user.ap_id, "object" => like_activity.data, - "to" => [user.follower_address, like_activity.data["actor"]], + "to" => [user.follower_address, object.data["actor"]], "cc" => [Pleroma.Constants.as_public()], "context" => like_activity.data["context"], "id" => "9mJEZK0tky1w2xD2vY" From 8325858ed229304297df9f57d6e4a359cfa2b4a8 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Fri, 4 Oct 2019 15:17:32 +0000 Subject: [PATCH 12/20] tests: streamer: add a test for blocked transitive activities --- test/web/streamer/streamer_test.exs | 74 ++++++++++++++++++++++------- 1 file changed, 56 insertions(+), 18 deletions(-) diff --git a/test/web/streamer/streamer_test.exs b/test/web/streamer/streamer_test.exs index b8fcd41fa..d33eb1e42 100644 --- a/test/web/streamer/streamer_test.exs +++ b/test/web/streamer/streamer_test.exs @@ -233,30 +233,68 @@ defmodule Pleroma.Web.StreamerTest do end end - test "it doesn't send to blocked users" do - user = insert(:user) - blocked_user = insert(:user) - {:ok, user} = User.block(user, blocked_user) + describe "blocks" do + test "it doesn't send messages involving blocked users" do + user = insert(:user) + blocked_user = insert(:user) + {:ok, user} = User.block(user, blocked_user) - task = - Task.async(fn -> - refute_receive {:text, _}, 1_000 - end) + task = + Task.async(fn -> + refute_receive {:text, _}, 1_000 + end) - fake_socket = %StreamerSocket{ - transport_pid: task.pid, - user: user - } + fake_socket = %StreamerSocket{ + transport_pid: task.pid, + user: user + } - {:ok, activity} = CommonAPI.post(blocked_user, %{"status" => "Test"}) + {:ok, activity} = CommonAPI.post(blocked_user, %{"status" => "Test"}) - topics = %{ - "public" => [fake_socket] - } + topics = %{ + "public" => [fake_socket] + } - Worker.push_to_socket(topics, "public", activity) + Worker.push_to_socket(topics, "public", activity) - Task.await(task) + Task.await(task) + end + + test "it doesn't send messages transitively involving blocked users" do + blocker = insert(:user) + blockee = insert(:user) + friend = insert(:user) + + task = + Task.async(fn -> + refute_receive {:text, _}, 1_000 + end) + + fake_socket = %StreamerSocket{ + transport_pid: task.pid, + user: blocker + } + + topics = %{ + "public" => [fake_socket] + } + + {:ok, blocker} = User.block(blocker, blockee) + + {:ok, activity_one} = CommonAPI.post(friend, %{"status" => "hey! @#{blockee.nickname}"}) + + Worker.push_to_socket(topics, "public", activity_one) + + {:ok, activity_two} = CommonAPI.post(blockee, %{"status" => "hey! @#{friend.nickname}"}) + + Worker.push_to_socket(topics, "public", activity_two) + + {:ok, activity_three} = CommonAPI.post(blockee, %{"status" => "hey! @#{blocker.nickname}"}) + + Worker.push_to_socket(topics, "public", activity_three) + + Task.await(task) + end end test "it doesn't send unwanted DMs to list" do From 2417b633ed866e6517a3fa0c30d0e85fc76dd548 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Fri, 4 Oct 2019 15:21:45 +0000 Subject: [PATCH 13/20] streamer: add missing copyright headers --- lib/pleroma/web/streamer/ping.ex | 4 ++++ lib/pleroma/web/streamer/state.ex | 4 ++++ lib/pleroma/web/streamer/streamer_socket.ex | 4 ++++ lib/pleroma/web/streamer/supervisor.ex | 4 ++++ lib/pleroma/web/streamer/worker.ex | 4 ++++ 5 files changed, 20 insertions(+) diff --git a/lib/pleroma/web/streamer/ping.ex b/lib/pleroma/web/streamer/ping.ex index f77cbb95c..db3e68abe 100644 --- a/lib/pleroma/web/streamer/ping.ex +++ b/lib/pleroma/web/streamer/ping.ex @@ -1,3 +1,7 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + defmodule Pleroma.Web.Streamer.Ping do use GenServer require Logger diff --git a/lib/pleroma/web/streamer/state.ex b/lib/pleroma/web/streamer/state.ex index c48752d95..5ce3ebb8a 100644 --- a/lib/pleroma/web/streamer/state.ex +++ b/lib/pleroma/web/streamer/state.ex @@ -1,3 +1,7 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + defmodule Pleroma.Web.Streamer.State do use GenServer require Logger diff --git a/lib/pleroma/web/streamer/streamer_socket.ex b/lib/pleroma/web/streamer/streamer_socket.ex index f006c0306..cf0fa3077 100644 --- a/lib/pleroma/web/streamer/streamer_socket.ex +++ b/lib/pleroma/web/streamer/streamer_socket.ex @@ -1,3 +1,7 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + defmodule Pleroma.Web.Streamer.StreamerSocket do defstruct transport_pid: nil, user: nil diff --git a/lib/pleroma/web/streamer/supervisor.ex b/lib/pleroma/web/streamer/supervisor.ex index 6afe19323..ec5985085 100644 --- a/lib/pleroma/web/streamer/supervisor.ex +++ b/lib/pleroma/web/streamer/supervisor.ex @@ -1,3 +1,7 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + defmodule Pleroma.Web.Streamer.Supervisor do use Supervisor diff --git a/lib/pleroma/web/streamer/worker.ex b/lib/pleroma/web/streamer/worker.ex index 5804508eb..bbb7483e5 100644 --- a/lib/pleroma/web/streamer/worker.ex +++ b/lib/pleroma/web/streamer/worker.ex @@ -1,3 +1,7 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + defmodule Pleroma.Web.Streamer.Worker do use GenServer From 5a0c018d2a6d3cea15761c1cc51691dcb85a0c97 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Fri, 4 Oct 2019 15:41:55 +0000 Subject: [PATCH 14/20] streamer: worker: check for lack of intersectionality between a user's blocklist and an activity's recipientlist --- lib/pleroma/web/streamer/worker.ex | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/pleroma/web/streamer/worker.ex b/lib/pleroma/web/streamer/worker.ex index bbb7483e5..3d2c8f473 100644 --- a/lib/pleroma/web/streamer/worker.ex +++ b/lib/pleroma/web/streamer/worker.ex @@ -132,11 +132,14 @@ defmodule Pleroma.Web.Streamer.Worker do blocks = user.info.blocks || [] mutes = user.info.mutes || [] reblog_mutes = user.info.muted_reblogs || [] + recipient_blocks = MapSet.new(blocks ++ mutes) + recipients = MapSet.new(item.recipients) domain_blocks = Pleroma.Web.ActivityPub.MRF.subdomains_regex(user.info.domain_blocks) with parent when not is_nil(parent) <- Object.normalize(item), true <- Enum.all?([blocks, mutes, reblog_mutes], &(item.actor not in &1)), true <- Enum.all?([blocks, mutes], &(parent.data["actor"] not in &1)), + true <- MapSet.disjoint?(recipients, recipient_blocks), %{host: item_host} <- URI.parse(item.actor), %{host: parent_host} <- URI.parse(parent.data["actor"]), false <- Pleroma.Web.ActivityPub.MRF.subdomain_match?(domain_blocks, item_host), From d1d058bf85a94909cb0c599bb5f2bd469de804d5 Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Fri, 4 Oct 2019 15:42:25 +0000 Subject: [PATCH 15/20] streamer: worker: actually use should_send? consistently --- lib/pleroma/web/streamer/worker.ex | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/pleroma/web/streamer/worker.ex b/lib/pleroma/web/streamer/worker.ex index 3d2c8f473..0ea224874 100644 --- a/lib/pleroma/web/streamer/worker.ex +++ b/lib/pleroma/web/streamer/worker.ex @@ -201,11 +201,8 @@ defmodule Pleroma.Web.Streamer.Worker do # Get the current user so we have up-to-date blocks etc. if socket_user do user = User.get_cached_by_ap_id(socket_user.ap_id) - blocks = user.info.blocks || [] - mutes = user.info.mutes || [] - with true <- Enum.all?([blocks, mutes], &(item.actor not in &1)), - true <- thread_containment(item, user) do + if should_send?(user, item) do send(transport_pid, {:text, StreamerView.render("update.json", item, user)}) end else From dbf5fce67e39821b8f3caa7f3f59deeb95754bce Mon Sep 17 00:00:00 2001 From: Ariadne Conill Date: Fri, 4 Oct 2019 15:45:06 +0000 Subject: [PATCH 16/20] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ebc46b7d..4a904a3f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -76,6 +76,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - ActivityPub: Deactivated user deletion - ActivityPub: Fix `/users/:nickname/inbox` crashing without an authenticated user - MRF: fix ability to follow a relay when AntiFollowbotPolicy was enabled +- Mastodon API: Blocks are now treated consistently between the Streaming API and the Timeline APIs ### Added - Expiring/ephemeral activites. All activities can have expires_at value set, which controls when they should be deleted automatically. From 83631752af053b02a05abe0e9f7d6c7cf9a5154a Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Fri, 4 Oct 2019 22:20:53 +0300 Subject: [PATCH 17/20] removed legacy api: "/objects/:uuid/likes" --- .../activity_pub/activity_pub_controller.ex | 32 ---------- lib/pleroma/web/activity_pub/utils.ex | 10 --- .../web/activity_pub/views/object_view.ex | 36 ----------- lib/pleroma/web/router.ex | 1 - .../activity_pub_controller_test.exs | 63 ------------------- 5 files changed, 142 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index 7cd13b4b8..080030eb5 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -82,38 +82,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do conn end - def object_likes(conn, %{"uuid" => uuid, "page" => page}) do - with ap_id <- o_status_url(conn, :object, uuid), - %Object{} = object <- Object.get_cached_by_ap_id(ap_id), - {_, true} <- {:public?, Visibility.is_public?(object)}, - likes <- Utils.get_object_likes(object) do - {page, _} = Integer.parse(page) - - conn - |> put_resp_content_type("application/activity+json") - |> put_view(ObjectView) - |> render("likes.json", %{ap_id: ap_id, likes: likes, page: page}) - else - {:public?, false} -> - {:error, :not_found} - end - end - - def object_likes(conn, %{"uuid" => uuid}) do - with ap_id <- o_status_url(conn, :object, uuid), - %Object{} = object <- Object.get_cached_by_ap_id(ap_id), - {_, true} <- {:public?, Visibility.is_public?(object)}, - likes <- Utils.get_object_likes(object) do - conn - |> put_resp_content_type("application/activity+json") - |> put_view(ObjectView) - |> render("likes.json", %{ap_id: ap_id, likes: likes}) - else - {:public?, false} -> - {:error, :not_found} - end - end - def activity(conn, %{"uuid" => uuid}) do with ap_id <- o_status_url(conn, :activity, uuid), %Activity{} = activity <- Activity.normalize(ap_id), diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index ac5550671..272011a9f 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -251,16 +251,6 @@ defmodule Pleroma.Web.ActivityPub.Utils do |> Repo.one() end - @doc """ - Returns like activities targeting an object - """ - def get_object_likes(%{data: %{"id" => id}}) do - id - |> Activity.Queries.by_object_id() - |> Activity.Queries.by_type("Like") - |> Repo.all() - end - @spec make_like_data(User.t(), map(), String.t()) :: map() def make_like_data( %User{ap_id: ap_id} = actor, diff --git a/lib/pleroma/web/activity_pub/views/object_view.ex b/lib/pleroma/web/activity_pub/views/object_view.ex index 88c55acdd..d8a3ec288 100644 --- a/lib/pleroma/web/activity_pub/views/object_view.ex +++ b/lib/pleroma/web/activity_pub/views/object_view.ex @@ -37,40 +37,4 @@ defmodule Pleroma.Web.ActivityPub.ObjectView do Map.merge(base, additional) end - - def render("likes.json", %{ap_id: ap_id, likes: likes, page: page}) do - collection(likes, "#{ap_id}/likes", page) - |> Map.merge(Pleroma.Web.ActivityPub.Utils.make_json_ld_header()) - end - - def render("likes.json", %{ap_id: ap_id, likes: likes}) do - %{ - "id" => "#{ap_id}/likes", - "type" => "OrderedCollection", - "totalItems" => length(likes), - "first" => collection(likes, "#{ap_id}/likes", 1) - } - |> Map.merge(Pleroma.Web.ActivityPub.Utils.make_json_ld_header()) - end - - def collection(collection, iri, page) do - offset = (page - 1) * 10 - items = Enum.slice(collection, offset, 10) - items = Enum.map(items, fn object -> Transmogrifier.prepare_object(object.data) end) - total = length(collection) - - map = %{ - "id" => "#{iri}?page=#{page}", - "type" => "OrderedCollectionPage", - "partOf" => iri, - "totalItems" => total, - "orderedItems" => items - } - - if offset + length(items) < total do - Map.put(map, "next", "#{iri}?page=#{page + 1}") - else - map - end - end end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index f91af8137..405ae724e 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -580,7 +580,6 @@ defmodule Pleroma.Web.Router do pipe_through(:ostatus) get("/users/:nickname/outbox", ActivityPubController, :outbox) - get("/objects/:uuid/likes", ActivityPubController, :object_likes) end pipeline :activitypub_client do diff --git a/test/web/activity_pub/activity_pub_controller_test.exs b/test/web/activity_pub/activity_pub_controller_test.exs index 1ffa91b70..6a3e48b5e 100644 --- a/test/web/activity_pub/activity_pub_controller_test.exs +++ b/test/web/activity_pub/activity_pub_controller_test.exs @@ -225,69 +225,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do end end - describe "/object/:uuid/likes" do - setup do - like = insert(:like_activity) - like_object_ap_id = Object.normalize(like).data["id"] - - uuid = - like_object_ap_id - |> String.split("/") - |> List.last() - - [id: like.data["id"], uuid: uuid] - end - - test "it returns the like activities in a collection", %{conn: conn, id: id, uuid: uuid} do - result = - conn - |> put_req_header("accept", "application/activity+json") - |> get("/objects/#{uuid}/likes") - |> json_response(200) - - assert List.first(result["first"]["orderedItems"])["id"] == id - assert result["type"] == "OrderedCollection" - assert result["totalItems"] == 1 - refute result["first"]["next"] - end - - test "it does not crash when page number is exceeded total pages", %{conn: conn, uuid: uuid} do - result = - conn - |> put_req_header("accept", "application/activity+json") - |> get("/objects/#{uuid}/likes?page=2") - |> json_response(200) - - assert result["type"] == "OrderedCollectionPage" - assert result["totalItems"] == 1 - refute result["next"] - assert Enum.empty?(result["orderedItems"]) - end - - test "it contains the next key when likes count is more than 10", %{conn: conn} do - note = insert(:note_activity) - insert_list(11, :like_activity, note_activity: note) - - uuid = - note - |> Object.normalize() - |> Map.get(:data) - |> Map.get("id") - |> String.split("/") - |> List.last() - - result = - conn - |> put_req_header("accept", "application/activity+json") - |> get("/objects/#{uuid}/likes?page=1") - |> json_response(200) - - assert result["totalItems"] == 11 - assert length(result["orderedItems"]) == 10 - assert result["next"] - end - end - describe "/activities/:uuid" do test "it returns a json representation of the activity", %{conn: conn} do activity = insert(:note_activity) From e07e9cb75e6605218acea1ef41772ca29124bd0d Mon Sep 17 00:00:00 2001 From: kaniini Date: Sat, 5 Oct 2019 10:00:05 +0000 Subject: [PATCH 18/20] Revert "Merge branch 'user-info-unread-direct-conversation' into 'develop'" This reverts merge request !1737 --- CHANGELOG.md | 1 - docs/API/differences_in_mastoapi_responses.md | 1 - lib/pleroma/conversation.ex | 2 - lib/pleroma/conversation/participation.ex | 17 --- lib/pleroma/user.ex | 56 --------- lib/pleroma/user/info.ex | 1 - .../web/mastodon_api/views/account_view.ex | 11 -- test/conversation/participation_test.exs | 4 - .../conversation_controller_test.exs | 109 +++--------------- .../mastodon_api/views/account_view_test.exs | 21 ---- .../pleroma_api_controller_test.exs | 2 - 11 files changed, 14 insertions(+), 211 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index db505591b..b42b13018 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Pleroma API: `GET /api/v1/pleroma/accounts/:id/scrobbles` to get a list of recently scrobbled items - Pleroma API: `POST /api/v1/pleroma/scrobble` to scrobble a media item - Mastodon API: Add `upload_limit`, `avatar_upload_limit`, `background_upload_limit`, and `banner_upload_limit` to `/api/v1/instance` -- Mastodon API: Add `pleroma.unread_conversation_count` to the Account entity ### Changed - **Breaking:** Elixir >=1.8 is now required (was >= 1.7) diff --git a/docs/API/differences_in_mastoapi_responses.md b/docs/API/differences_in_mastoapi_responses.md index 21b297529..d007a69c3 100644 --- a/docs/API/differences_in_mastoapi_responses.md +++ b/docs/API/differences_in_mastoapi_responses.md @@ -56,7 +56,6 @@ Has these additional fields under the `pleroma` object: - `settings_store`: A generic map of settings for frontends. Opaque to the backend. Only returned in `verify_credentials` and `update_credentials` - `chat_token`: The token needed for Pleroma chat. Only returned in `verify_credentials` - `deactivated`: boolean, true when the user is deactivated -- `unread_conversation_count`: The count of unread conversations. Only returned to the account owner. ### Source diff --git a/lib/pleroma/conversation.ex b/lib/pleroma/conversation.ex index 098016af2..be5821ad7 100644 --- a/lib/pleroma/conversation.ex +++ b/lib/pleroma/conversation.ex @@ -67,8 +67,6 @@ defmodule Pleroma.Conversation do participations = Enum.map(users, fn user -> - User.increment_unread_conversation_count(conversation, user) - {:ok, participation} = Participation.create_for_user_and_conversation(user, conversation, opts) diff --git a/lib/pleroma/conversation/participation.ex b/lib/pleroma/conversation/participation.ex index ab81f3217..e946f6de2 100644 --- a/lib/pleroma/conversation/participation.ex +++ b/lib/pleroma/conversation/participation.ex @@ -52,15 +52,6 @@ defmodule Pleroma.Conversation.Participation do participation |> read_cng(%{read: true}) |> Repo.update() - |> case do - {:ok, participation} -> - participation = Repo.preload(participation, :user) - User.set_unread_conversation_count(participation.user) - {:ok, participation} - - error -> - error - end end def mark_as_unread(participation) do @@ -144,12 +135,4 @@ defmodule Pleroma.Conversation.Participation do {:ok, Repo.preload(participation, :recipients, force: true)} end - - def unread_conversation_count_for_user(user) do - from(p in __MODULE__, - where: p.user_id == ^user.id, - where: not p.read, - select: %{count: count(p.id)} - ) - end end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 494a67f22..c2f8fa0d7 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -11,7 +11,6 @@ defmodule Pleroma.User do alias Comeonin.Pbkdf2 alias Ecto.Multi alias Pleroma.Activity - alias Pleroma.Conversation.Participation alias Pleroma.Delivery alias Pleroma.Keys alias Pleroma.Notification @@ -843,61 +842,6 @@ defmodule Pleroma.User do def maybe_update_following_count(user), do: user - def set_unread_conversation_count(%User{local: true} = user) do - unread_query = Participation.unread_conversation_count_for_user(user) - - User - |> where([u], u.id == ^user.id) - |> join(:inner, [u], p in subquery(unread_query)) - |> update([u, p], - set: [ - info: - fragment( - "jsonb_set(?, '{unread_conversation_count}', ?::varchar::jsonb, true)", - u.info, - p.count - ) - ] - ) - |> select([u], u) - |> Repo.update_all([]) - |> case do - {1, [%{info: %User.Info{}} = user]} -> set_cache(user) - _ -> {:error, user} - end - end - - def set_unread_conversation_count(_), do: :noop - - def increment_unread_conversation_count(conversation, %User{local: true} = user) do - unread_query = - Participation.unread_conversation_count_for_user(user) - |> where([p], p.conversation_id == ^conversation.id) - - User - |> join(:inner, [u], p in subquery(unread_query)) - |> update([u, p], - set: [ - info: - fragment( - "jsonb_set(?, '{unread_conversation_count}', ((?->>'unread_conversation_count')::int + 1)::varchar::jsonb, true)", - u.info, - u.info - ) - ] - ) - |> where([u], u.id == ^user.id) - |> where([u, p], p.count == 0) - |> select([u], u) - |> Repo.update_all([]) - |> case do - {1, [%{info: %User.Info{}} = user]} -> set_cache(user) - _ -> {:error, user} - end - end - - def increment_unread_conversation_count(_, _), do: :noop - def remove_duplicated_following(%User{following: following} = user) do uniq_following = Enum.uniq(following) diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index 4b5b43d7f..ebd4ddebf 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -47,7 +47,6 @@ defmodule Pleroma.User.Info do field(:hide_followers, :boolean, default: false) field(:hide_follows, :boolean, default: false) field(:hide_favorites, :boolean, default: true) - field(:unread_conversation_count, :integer, default: 0) field(:pinned_activities, {:array, :string}, default: []) field(:email_notifications, :map, default: %{"digest" => false}) field(:mascot, :map, default: nil) diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 2d4976891..99169ef95 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -167,7 +167,6 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do |> maybe_put_chat_token(user, opts[:for], opts) |> maybe_put_activation_status(user, opts[:for]) |> maybe_put_follow_requests_count(user, opts[:for]) - |> maybe_put_unread_conversation_count(user, opts[:for]) end defp username_from_nickname(string) when is_binary(string) do @@ -249,16 +248,6 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do defp maybe_put_activation_status(data, _, _), do: data - defp maybe_put_unread_conversation_count(data, %User{id: user_id} = user, %User{id: user_id}) do - data - |> Kernel.put_in( - [:pleroma, :unread_conversation_count], - user.info.unread_conversation_count - ) - end - - defp maybe_put_unread_conversation_count(data, _, _), do: data - defp image_url(%{"url" => [%{"href" => href} | _]}), do: href defp image_url(_), do: nil end diff --git a/test/conversation/participation_test.exs b/test/conversation/participation_test.exs index f430bdf75..a27167d42 100644 --- a/test/conversation/participation_test.exs +++ b/test/conversation/participation_test.exs @@ -6,7 +6,6 @@ defmodule Pleroma.Conversation.ParticipationTest do use Pleroma.DataCase import Pleroma.Factory alias Pleroma.Conversation.Participation - alias Pleroma.User alias Pleroma.Web.CommonAPI test "getting a participation will also preload things" do @@ -31,8 +30,6 @@ defmodule Pleroma.Conversation.ParticipationTest do {:ok, activity} = CommonAPI.post(user, %{"status" => "Hey @#{other_user.nickname}.", "visibility" => "direct"}) - user = User.get_cached_by_id(user.id) - other_user = User.get_cached_by_id(user.id) [participation] = Participation.for_user(user) participation = Pleroma.Repo.preload(participation, :recipients) @@ -158,7 +155,6 @@ defmodule Pleroma.Conversation.ParticipationTest do [participation] = Participation.for_user_with_last_activity_id(user) participation = Repo.preload(participation, :recipients) - user = User.get_cached_by_id(user.id) assert participation.recipients |> length() == 1 assert user in participation.recipients diff --git a/test/web/mastodon_api/controllers/conversation_controller_test.exs b/test/web/mastodon_api/controllers/conversation_controller_test.exs index a308a7620..7117fc76a 100644 --- a/test/web/mastodon_api/controllers/conversation_controller_test.exs +++ b/test/web/mastodon_api/controllers/conversation_controller_test.exs @@ -10,23 +10,19 @@ defmodule Pleroma.Web.MastodonAPI.ConversationControllerTest do import Pleroma.Factory - test "returns a list of conversations", %{conn: conn} do + test "Conversations", %{conn: conn} do user_one = insert(:user) user_two = insert(:user) user_three = insert(:user) {:ok, user_two} = User.follow(user_two, user_one) - assert User.get_cached_by_id(user_two.id).info.unread_conversation_count == 0 - {:ok, direct} = CommonAPI.post(user_one, %{ "status" => "Hi @#{user_two.nickname}, @#{user_three.nickname}!", "visibility" => "direct" }) - assert User.get_cached_by_id(user_two.id).info.unread_conversation_count == 1 - {:ok, _follower_only} = CommonAPI.post(user_one, %{ "status" => "Hi @#{user_two.nickname}!", @@ -56,100 +52,23 @@ defmodule Pleroma.Web.MastodonAPI.ConversationControllerTest do assert is_binary(res_id) assert unread == true assert res_last_status["id"] == direct.id - assert User.get_cached_by_id(user_one.id).info.unread_conversation_count == 1 - end - - test "updates the last_status on reply", %{conn: conn} do - user_one = insert(:user) - user_two = insert(:user) - - {:ok, direct} = - CommonAPI.post(user_one, %{ - "status" => "Hi @#{user_two.nickname}", - "visibility" => "direct" - }) - - {:ok, direct_reply} = - CommonAPI.post(user_two, %{ - "status" => "reply", - "visibility" => "direct", - "in_reply_to_status_id" => direct.id - }) - - [%{"last_status" => res_last_status}] = - conn - |> assign(:user, user_one) - |> get("/api/v1/conversations") - |> json_response(200) - - assert res_last_status["id"] == direct_reply.id - end - - test "the user marks a conversation as read", %{conn: conn} do - user_one = insert(:user) - user_two = insert(:user) - - {:ok, direct} = - CommonAPI.post(user_one, %{ - "status" => "Hi @#{user_two.nickname}", - "visibility" => "direct" - }) - - [%{"id" => direct_conversation_id, "unread" => true}] = - conn - |> assign(:user, user_one) - |> get("/api/v1/conversations") - |> json_response(200) - - %{"unread" => false} = - conn - |> assign(:user, user_one) - |> post("/api/v1/conversations/#{direct_conversation_id}/read") - |> json_response(200) - - assert User.get_cached_by_id(user_one.id).info.unread_conversation_count == 0 - - # The conversation is marked as unread on reply - {:ok, _} = - CommonAPI.post(user_two, %{ - "status" => "reply", - "visibility" => "direct", - "in_reply_to_status_id" => direct.id - }) - - [%{"unread" => true}] = - conn - |> assign(:user, user_one) - |> get("/api/v1/conversations") - |> json_response(200) - - assert User.get_cached_by_id(user_one.id).info.unread_conversation_count == 1 - - # A reply doesn't increment the user's unread_conversation_count if the conversation is unread - {:ok, _} = - CommonAPI.post(user_two, %{ - "status" => "reply", - "visibility" => "direct", - "in_reply_to_status_id" => direct.id - }) - - assert User.get_cached_by_id(user_one.id).info.unread_conversation_count == 1 - end - - test "(vanilla) Mastodon frontend behaviour", %{conn: conn} do - user_one = insert(:user) - user_two = insert(:user) - - {:ok, direct} = - CommonAPI.post(user_one, %{ - "status" => "Hi @#{user_two.nickname}!", - "visibility" => "direct" - }) + # Apparently undocumented API endpoint res_conn = conn |> assign(:user, user_one) - |> get("/api/v1/statuses/#{direct.id}/context") + |> post("/api/v1/conversations/#{res_id}/read") + + assert response = json_response(res_conn, 200) + assert length(response["accounts"]) == 2 + assert response["last_status"]["id"] == direct.id + assert response["unread"] == false + + # (vanilla) Mastodon frontend behaviour + res_conn = + conn + |> assign(:user, user_one) + |> get("/api/v1/statuses/#{res_last_status["id"]}/context") assert %{"ancestors" => [], "descendants" => []} == json_response(res_conn, 200) end diff --git a/test/web/mastodon_api/views/account_view_test.exs b/test/web/mastodon_api/views/account_view_test.exs index b7a4938a6..62b2ab7e3 100644 --- a/test/web/mastodon_api/views/account_view_test.exs +++ b/test/web/mastodon_api/views/account_view_test.exs @@ -418,27 +418,6 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do following_count: 1 } = AccountView.render("show.json", %{user: user, for: user}) end - - test "shows unread_conversation_count only to the account owner" do - user = insert(:user) - other_user = insert(:user) - - {:ok, _activity} = - CommonAPI.post(user, %{ - "status" => "Hey @#{other_user.nickname}.", - "visibility" => "direct" - }) - - user = User.get_cached_by_ap_id(user.ap_id) - - assert AccountView.render("show.json", %{user: user, for: other_user})[:pleroma][ - :unread_conversation_count - ] == nil - - assert AccountView.render("show.json", %{user: user, for: user})[:pleroma][ - :unread_conversation_count - ] == 1 - end end describe "follow requests counter" do diff --git a/test/web/pleroma_api/controllers/pleroma_api_controller_test.exs b/test/web/pleroma_api/controllers/pleroma_api_controller_test.exs index 8a6528cbb..7eaeda4a0 100644 --- a/test/web/pleroma_api/controllers/pleroma_api_controller_test.exs +++ b/test/web/pleroma_api/controllers/pleroma_api_controller_test.exs @@ -8,7 +8,6 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIControllerTest do alias Pleroma.Conversation.Participation alias Pleroma.Notification alias Pleroma.Repo - alias Pleroma.User alias Pleroma.Web.CommonAPI import Pleroma.Factory @@ -74,7 +73,6 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIControllerTest do participation = Repo.preload(participation, :recipients) - user = User.get_cached_by_id(user.id) assert [user] == participation.recipients assert other_user not in participation.recipients From 4b8524f392e659d568ba6b0648952d7a1e314be0 Mon Sep 17 00:00:00 2001 From: lain Date: Sat, 5 Oct 2019 14:49:45 +0200 Subject: [PATCH 19/20] ActivityPub / Transmogrifier: Correctly store incoming Update id. --- lib/pleroma/web/activity_pub/activity_pub.ex | 4 +++- lib/pleroma/web/activity_pub/transmogrifier.ex | 3 ++- lib/pleroma/web/activity_pub/utils.ex | 4 ++-- test/web/activity_pub/transmogrifier_test.exs | 2 ++ 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index c52efb578..5052d1304 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -17,6 +17,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do alias Pleroma.User alias Pleroma.Web.ActivityPub.MRF alias Pleroma.Web.ActivityPub.Transmogrifier + alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.Streamer alias Pleroma.Web.WebFinger alias Pleroma.Workers.BackgroundWorker @@ -291,8 +292,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end def update(%{to: to, cc: cc, actor: actor, object: object} = params) do - # only accept false as false value local = !(params[:local] == false) + activity_id = params[:activity_id] with data <- %{ "to" => to, @@ -301,6 +302,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do "actor" => actor, "object" => object }, + data <- Utils.maybe_put(data, "id", activity_id), {:ok, activity} <- insert(data, local), :ok <- maybe_federate(activity) do {:ok, activity} diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 64c470fc8..c56d2dd11 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -621,7 +621,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do to: data["to"] || [], cc: data["cc"] || [], object: object, - actor: actor_id + actor: actor_id, + activity_id: data["id"] }) else e -> diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 272011a9f..4ef479f96 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -739,6 +739,6 @@ defmodule Pleroma.Web.ActivityPub.Utils do |> Repo.all() end - defp maybe_put(map, _key, nil), do: map - defp maybe_put(map, key, value), do: Map.put(map, key, value) + def maybe_put(map, _key, nil), do: map + def maybe_put(map, key, value), do: Map.put(map, key, value) end diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 6c208bdc0..475313316 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -546,6 +546,8 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(update_data) + assert data["id"] == update_data["id"] + user = User.get_cached_by_ap_id(data["actor"]) assert user.name == "gargle" From 276a52016307707fa6a2f91bd9a3038dcfb71ba7 Mon Sep 17 00:00:00 2001 From: lain Date: Sat, 5 Oct 2019 14:53:50 +0200 Subject: [PATCH 20/20] CommonAPI: Create profile updates as public. This saves us lots of sending out because we can use sharedInbox. --- lib/pleroma/web/common_api/common_api.ex | 4 +++- test/web/common_api/common_api_test.exs | 6 +++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index ce73b3270..386408d51 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -16,6 +16,8 @@ defmodule Pleroma.Web.CommonAPI do import Pleroma.Web.Gettext import Pleroma.Web.CommonAPI.Utils + require Pleroma.Constants + def follow(follower, followed) do timeout = Pleroma.Config.get([:activitypub, :follow_handshake_timeout]) @@ -271,7 +273,7 @@ defmodule Pleroma.Web.CommonAPI do ActivityPub.update(%{ local: true, - to: [user.follower_address], + to: [Pleroma.Constants.as_public(), user.follower_address], cc: [], actor: user.ap_id, object: Pleroma.Web.ActivityPub.UserView.render("user.json", %{user: user}) diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs index 2d3c41e82..83df44c36 100644 --- a/test/web/common_api/common_api_test.exs +++ b/test/web/common_api/common_api_test.exs @@ -14,6 +14,8 @@ defmodule Pleroma.Web.CommonAPITest do import Pleroma.Factory + require Pleroma.Constants + clear_config([:instance, :safe_dm_mentions]) clear_config([:instance, :limit]) clear_config([:instance, :max_pinned_statuses]) @@ -96,11 +98,13 @@ defmodule Pleroma.Web.CommonAPITest do test "it adds emoji when updating profiles" do user = insert(:user, %{name: ":firefox:"}) - CommonAPI.update(user) + {:ok, activity} = CommonAPI.update(user) user = User.get_cached_by_ap_id(user.ap_id) [firefox] = user.info.source_data["tag"] assert firefox["name"] == ":firefox:" + + assert Pleroma.Constants.as_public() in activity.recipients end describe "posting" do