From df469b4468168cf072e73df73e0fdde2bbab1da5 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 11 Dec 2019 12:52:57 -0600 Subject: [PATCH 01/24] Benchmark env uses test database so we should be able to use test.secret.exs --- config/benchmark.exs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/config/benchmark.exs b/config/benchmark.exs index dd99cf5fd..c7ddb80e7 100644 --- a/config/benchmark.exs +++ b/config/benchmark.exs @@ -82,3 +82,11 @@ config :pleroma, :database, rum_enabled: rum_enabled IO.puts("RUM enabled: #{rum_enabled}") config :pleroma, Pleroma.ReverseProxy.Client, Pleroma.ReverseProxy.ClientMock + +if File.exists?("./config/test.secret.exs") do + import_config "test.secret.exs" +else + IO.puts( + "You may want to create test.secret.exs to declare custom database connection parameters." + ) +end From 6c39fa20b191f985a2be704089c20acbcfe0035a Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 17 Dec 2019 17:00:46 +0700 Subject: [PATCH 02/24] Add support for `account_id` param to filter notifications by the account --- CHANGELOG.md | 1 + lib/pleroma/web/mastodon_api/mastodon_api.ex | 11 ++++++++- .../notification_controller_test.exs | 23 +++++++++++++++++++ 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c133cd9ec..f0274ca01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -86,6 +86,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: `/api/v1/update_credentials` accepts `actor_type` field. - Captcha: Support native provider - Captcha: Enable by default +- Mastodon API: Add support for `account_id` param to filter notifications by the account ### Fixed diff --git a/lib/pleroma/web/mastodon_api/mastodon_api.ex b/lib/pleroma/web/mastodon_api/mastodon_api.ex index b1816370e..6c13d4df6 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api.ex @@ -56,6 +56,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do user |> Notification.for_user_query(options) |> restrict(:exclude_types, options) + |> restrict(:account_id, options) |> Pagination.fetch_paginated(params) end @@ -71,7 +72,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do exclude_visibilities: {:array, :string}, reblogs: :boolean, with_muted: :boolean, - with_move: :boolean + with_move: :boolean, + account_id: :string } changeset = cast({%{}, param_types}, params, Map.keys(param_types)) @@ -88,5 +90,12 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do |> where([q, a], not fragment("? @> ARRAY[?->>'type']::varchar[]", ^ap_types, a.data)) end + defp restrict(query, :account_id, %{account_id: account_id}) do + case User.get_cached_by_id(account_id) do + %{ap_id: ap_id} -> where(query, [n, a], a.actor == ^ap_id) + _ -> where(query, [n, a], a.actor == "fake ap id") + end + end + defp restrict(query, _, _), do: query end diff --git a/test/web/mastodon_api/controllers/notification_controller_test.exs b/test/web/mastodon_api/controllers/notification_controller_test.exs index 6635ea7a2..3458776ab 100644 --- a/test/web/mastodon_api/controllers/notification_controller_test.exs +++ b/test/web/mastodon_api/controllers/notification_controller_test.exs @@ -463,6 +463,29 @@ defmodule Pleroma.Web.MastodonAPI.NotificationControllerTest do assert length(json_response(conn, 200)) == 1 end + describe "from specified user" do + test "account_id", %{conn: conn} do + user = insert(:user) + %{id: account_id} = other_user1 = insert(:user) + other_user2 = insert(:user) + + {:ok, _activity} = CommonAPI.post(other_user1, %{"status" => "hi @#{user.nickname}"}) + {:ok, _activity} = CommonAPI.post(other_user2, %{"status" => "bye @#{user.nickname}"}) + + assert [%{"account" => %{"id" => ^account_id}}] = + conn + |> assign(:user, user) + |> get("/api/v1/notifications", %{account_id: account_id}) + |> json_response(200) + + assert [] = + conn + |> assign(:user, user) + |> get("/api/v1/notifications", %{account_id: "cofe"}) + |> json_response(200) + end + end + defp get_notification_id_by_activity(%{id: id}) do Notification |> Repo.get_by(activity_id: id) From 34d85f8a5473fe0f85e8a8e9e8f58e40b3964ba4 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 19 Dec 2019 20:45:44 +0700 Subject: [PATCH 03/24] Return 404 if account to filter notifications from is not found --- .../controllers/notification_controller.ex | 17 +++++++++++++++++ lib/pleroma/web/mastodon_api/mastodon_api.ex | 11 ++++------- .../notification_controller_test.exs | 4 ++-- 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex b/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex index 16759be6a..f2508aca4 100644 --- a/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex @@ -23,6 +23,23 @@ defmodule Pleroma.Web.MastodonAPI.NotificationController do plug(Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug) # GET /api/v1/notifications + def index(conn, %{"account_id" => account_id} = params) do + case Pleroma.User.get_cached_by_id(account_id) do + %{ap_id: account_ap_id} -> + params = + params + |> Map.delete("account_id") + |> Map.put("account_ap_id", account_ap_id) + + index(conn, params) + + _ -> + conn + |> put_status(:not_found) + |> json(%{"error" => "Account is not found"}) + end + end + def index(%{assigns: %{user: user}} = conn, params) do notifications = MastodonAPI.get_notifications(user, params) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api.ex b/lib/pleroma/web/mastodon_api/mastodon_api.ex index 6c13d4df6..390a2b190 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api.ex @@ -56,7 +56,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do user |> Notification.for_user_query(options) |> restrict(:exclude_types, options) - |> restrict(:account_id, options) + |> restrict(:account_ap_id, options) |> Pagination.fetch_paginated(params) end @@ -73,7 +73,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do reblogs: :boolean, with_muted: :boolean, with_move: :boolean, - account_id: :string + account_ap_id: :string } changeset = cast({%{}, param_types}, params, Map.keys(param_types)) @@ -90,11 +90,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do |> where([q, a], not fragment("? @> ARRAY[?->>'type']::varchar[]", ^ap_types, a.data)) end - defp restrict(query, :account_id, %{account_id: account_id}) do - case User.get_cached_by_id(account_id) do - %{ap_id: ap_id} -> where(query, [n, a], a.actor == ^ap_id) - _ -> where(query, [n, a], a.actor == "fake ap id") - end + defp restrict(query, :account_ap_id, %{account_ap_id: account_ap_id}) do + where(query, [n, a], a.actor == ^account_ap_id) end defp restrict(query, _, _), do: query diff --git a/test/web/mastodon_api/controllers/notification_controller_test.exs b/test/web/mastodon_api/controllers/notification_controller_test.exs index 3458776ab..24d0d49ed 100644 --- a/test/web/mastodon_api/controllers/notification_controller_test.exs +++ b/test/web/mastodon_api/controllers/notification_controller_test.exs @@ -478,11 +478,11 @@ defmodule Pleroma.Web.MastodonAPI.NotificationControllerTest do |> get("/api/v1/notifications", %{account_id: account_id}) |> json_response(200) - assert [] = + assert %{"error" => "Account is not found"} = conn |> assign(:user, user) |> get("/api/v1/notifications", %{account_id: "cofe"}) - |> json_response(200) + |> json_response(404) end end From b7811dfb7b612e0f6cf1d9f2451e381d525d965b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 19 Dec 2019 12:16:53 -0600 Subject: [PATCH 04/24] Instead allow a dedicated benchmark config --- config/benchmark.exs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/benchmark.exs b/config/benchmark.exs index c7ddb80e7..84c6782a2 100644 --- a/config/benchmark.exs +++ b/config/benchmark.exs @@ -83,10 +83,10 @@ IO.puts("RUM enabled: #{rum_enabled}") config :pleroma, Pleroma.ReverseProxy.Client, Pleroma.ReverseProxy.ClientMock -if File.exists?("./config/test.secret.exs") do - import_config "test.secret.exs" +if File.exists?("./config/benchmark.secret.exs") do + import_config "benchmark.secret.exs" else IO.puts( - "You may want to create test.secret.exs to declare custom database connection parameters." + "You may want to create benchmark.secret.exs to declare custom database connection parameters." ) end From 7bd0bca2abadb96aa13ace36b968d57872681f7a Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Fri, 20 Dec 2019 16:33:44 +0300 Subject: [PATCH 05/24] fixed remote follow --- lib/pleroma/web/activity_pub/publisher.ex | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index 4073d3d63..0cc8fab27 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -264,6 +264,10 @@ defmodule Pleroma.Web.ActivityPub.Publisher do "rel" => "self", "type" => "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"", "href" => user.ap_id + }, + %{ + "rel" => "http://ostatus.org/schema/1.0/subscribe", + "template" => "#{Pleroma.Web.base_url()}/ostatus_subscribe?acct={uri}" } ] end From 5b8415601346447b9a66b1eabfc7538191892a76 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Fri, 20 Dec 2019 16:34:14 +0300 Subject: [PATCH 06/24] moved remote follow in separate controller --- lib/pleroma/web/router.ex | 4 +- .../twitter_api/remote_follow/follow.html.eex | 11 + .../remote_follow/follow_login.html.eex | 14 ++ .../{util => remote_follow}/followed.html.eex | 0 .../twitter_api/util/follow.html.eex | 11 - .../twitter_api/util/follow_login.html.eex | 14 -- .../controllers/remote_follow_controller.ex | 102 +++++++++ .../controllers/util_controller.ex | 91 -------- .../twitter_api/views/remote_follow_view.ex | 10 + test/web/activity_pub/publisher_test.exs | 21 ++ .../remote_follow_controller_test.exs | 211 ++++++++++++++++++ test/web/twitter_api/util_controller_test.exs | 194 +--------------- 12 files changed, 373 insertions(+), 310 deletions(-) create mode 100644 lib/pleroma/web/templates/twitter_api/remote_follow/follow.html.eex create mode 100644 lib/pleroma/web/templates/twitter_api/remote_follow/follow_login.html.eex rename lib/pleroma/web/templates/twitter_api/{util => remote_follow}/followed.html.eex (100%) delete mode 100644 lib/pleroma/web/templates/twitter_api/util/follow.html.eex delete mode 100644 lib/pleroma/web/templates/twitter_api/util/follow_login.html.eex create mode 100644 lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex create mode 100644 lib/pleroma/web/twitter_api/views/remote_follow_view.ex create mode 100644 test/web/twitter_api/remote_follow_controller_test.exs diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index f6c128283..9654ab8a3 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -229,9 +229,9 @@ defmodule Pleroma.Web.Router do pipe_through(:pleroma_html) post("/main/ostatus", UtilController, :remote_subscribe) - get("/ostatus_subscribe", UtilController, :remote_follow) + get("/ostatus_subscribe", RemoteFollowController, :follow) - post("/ostatus_subscribe", UtilController, :do_remote_follow) + post("/ostatus_subscribe", RemoteFollowController, :do_follow) end scope "/api/pleroma", Pleroma.Web.TwitterAPI do diff --git a/lib/pleroma/web/templates/twitter_api/remote_follow/follow.html.eex b/lib/pleroma/web/templates/twitter_api/remote_follow/follow.html.eex new file mode 100644 index 000000000..5ba192cd7 --- /dev/null +++ b/lib/pleroma/web/templates/twitter_api/remote_follow/follow.html.eex @@ -0,0 +1,11 @@ +<%= if @error == :error do %> +

Error fetching user

+<% else %> +

Remote follow

+ +

<%= @followee.nickname %>

+ <%= form_for @conn, remote_follow_path(@conn, :do_follow), [as: "user"], fn f -> %> + <%= hidden_input f, :id, value: @followee.id %> + <%= submit "Authorize" %> + <% end %> +<% end %> diff --git a/lib/pleroma/web/templates/twitter_api/remote_follow/follow_login.html.eex b/lib/pleroma/web/templates/twitter_api/remote_follow/follow_login.html.eex new file mode 100644 index 000000000..df44988ee --- /dev/null +++ b/lib/pleroma/web/templates/twitter_api/remote_follow/follow_login.html.eex @@ -0,0 +1,14 @@ +<%= if @error do %> +

<%= @error %>

+<% end %> +

Log in to follow

+

<%= @followee.nickname %>

+ +<%= form_for @conn, remote_follow_path(@conn, :do_follow), [as: "authorization"], fn f -> %> +<%= text_input f, :name, placeholder: "Username", required: true %> +
+<%= password_input f, :password, placeholder: "Password", required: true %> +
+<%= hidden_input f, :id, value: @followee.id %> +<%= submit "Authorize" %> +<% end %> diff --git a/lib/pleroma/web/templates/twitter_api/util/followed.html.eex b/lib/pleroma/web/templates/twitter_api/remote_follow/followed.html.eex similarity index 100% rename from lib/pleroma/web/templates/twitter_api/util/followed.html.eex rename to lib/pleroma/web/templates/twitter_api/remote_follow/followed.html.eex diff --git a/lib/pleroma/web/templates/twitter_api/util/follow.html.eex b/lib/pleroma/web/templates/twitter_api/util/follow.html.eex deleted file mode 100644 index 06359fa6c..000000000 --- a/lib/pleroma/web/templates/twitter_api/util/follow.html.eex +++ /dev/null @@ -1,11 +0,0 @@ -<%= if @error == :error do %> -

Error fetching user

-<% else %> -

Remote follow

- -

<%= @name %>

- <%= form_for @conn, util_path(@conn, :do_remote_follow), [as: "user"], fn f -> %> - <%= hidden_input f, :id, value: @id %> - <%= submit "Authorize" %> - <% end %> -<% end %> diff --git a/lib/pleroma/web/templates/twitter_api/util/follow_login.html.eex b/lib/pleroma/web/templates/twitter_api/util/follow_login.html.eex deleted file mode 100644 index 4e3a2be67..000000000 --- a/lib/pleroma/web/templates/twitter_api/util/follow_login.html.eex +++ /dev/null @@ -1,14 +0,0 @@ -<%= if @error do %> -

<%= @error %>

-<% end %> -

Log in to follow

-

<%= @name %>

- -<%= form_for @conn, util_path(@conn, :do_remote_follow), [as: "authorization"], fn f -> %> -<%= text_input f, :name, placeholder: "Username" %> -
-<%= password_input f, :password, placeholder: "Password" %> -
-<%= hidden_input f, :id, value: @id %> -<%= submit "Authorize" %> -<% end %> diff --git a/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex b/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex new file mode 100644 index 000000000..460a42566 --- /dev/null +++ b/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex @@ -0,0 +1,102 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.TwitterAPI.RemoteFollowController do + use Pleroma.Web, :controller + + require Logger + + alias Pleroma.Activity + alias Pleroma.Object.Fetcher + alias Pleroma.Plugs.OAuthScopesPlug + alias Pleroma.User + alias Pleroma.Web.Auth.Authenticator + alias Pleroma.Web.CommonAPI + + @status_types ["Article", "Event", "Note", "Video", "Page", "Question"] + + plug(OAuthScopesPlug, %{scopes: ["follow", "write:follows"]} when action in [:do_follow]) + + # GET /ostatus_subscribe + # + def follow(%{assigns: %{user: user}} = conn, %{"acct" => acct}) do + case is_status?(acct) do + true -> follow_status(conn, user, acct) + _ -> follow_account(conn, user, acct) + end + end + + defp follow_status(conn, _user, acct) do + with {:ok, object} <- Fetcher.fetch_object_from_id(acct), + %Activity{id: activity_id} <- Activity.get_create_by_object_ap_id(object.data["id"]) do + redirect(conn, to: "/notice/#{activity_id}") + else + error -> + handle_follow_error(conn, error) + end + end + + defp follow_account(conn, user, acct) do + with {:ok, followee} <- User.get_or_fetch(acct) do + render(conn, follow_template(user), %{error: false, followee: followee, acct: acct}) + else + {:error, _reason} -> + render(conn, follow_template(user), %{error: :error}) + end + end + + defp follow_template(%User{} = _user), do: "follow.html" + defp follow_template(_), do: "follow_login.html" + + defp is_status?(acct) do + case Fetcher.fetch_and_contain_remote_object_from_id(acct) do + {:ok, %{"type" => type}} when type in @status_types -> + true + + _ -> + false + end + end + + # POST /ostatus_subscribe + # + def do_follow(conn, %{"authorization" => %{"name" => _, "password" => _, "id" => id}}) do + with {:fetch_user, %User{} = followee} <- {:fetch_user, User.get_cached_by_id(id)}, + {_, {:ok, user}, _} <- {:auth, Authenticator.get_user(conn), followee}, + {:ok, _, _, _} <- CommonAPI.follow(user, followee) do + render(conn, "followed.html", %{error: false}) + else + error -> + handle_follow_error(conn, error) + end + end + + def do_follow(%{assigns: %{user: user}} = conn, %{"user" => %{"id" => id}}) do + with {:fetch_user, %User{} = followee} <- {:fetch_user, User.get_cached_by_id(id)}, + {:ok, _, _, _} <- CommonAPI.follow(user, followee) do + render(conn, "followed.html", %{error: false}) + else + error -> + handle_follow_error(conn, error) + end + end + + defp handle_follow_error(conn, {:auth, _, followee} = _) do + render(conn, "follow_login.html", %{error: "Wrong username or password", followee: followee}) + end + + defp handle_follow_error(conn, {:fetch_user, error} = _) do + Logger.debug("Remote follow failed with error #{inspect(error)}") + render(conn, "followed.html", %{error: "Could not find user"}) + end + + defp handle_follow_error(conn, {:error, "Could not follow user:" <> _} = _) do + render(conn, "followed.html", %{error: "Error following account"}) + end + + defp handle_follow_error(conn, error) do + Logger.debug("Remote follow failed with error #{inspect(error)}") + render(conn, "followed.html", %{error: "Something went wrong."}) + end +end diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex index 799dd17ae..a61f891c7 100644 --- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -7,12 +7,10 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do require Logger - alias Pleroma.Activity alias Pleroma.Config alias Pleroma.Emoji alias Pleroma.Healthcheck alias Pleroma.Notification - alias Pleroma.Plugs.AuthenticationPlug alias Pleroma.Plugs.OAuthScopesPlug alias Pleroma.User alias Pleroma.Web @@ -77,95 +75,6 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do end end - def remote_follow(%{assigns: %{user: user}} = conn, %{"acct" => acct}) do - if is_status?(acct) do - {:ok, object} = Pleroma.Object.Fetcher.fetch_object_from_id(acct) - %Activity{id: activity_id} = Activity.get_create_by_object_ap_id(object.data["id"]) - redirect(conn, to: "/notice/#{activity_id}") - else - with {:ok, followee} <- User.get_or_fetch(acct) do - conn - |> render(follow_template(user), %{ - error: false, - acct: acct, - avatar: User.avatar_url(followee), - name: followee.nickname, - id: followee.id - }) - else - {:error, _reason} -> - render(conn, follow_template(user), %{error: :error}) - end - end - end - - defp follow_template(%User{} = _user), do: "follow.html" - defp follow_template(_), do: "follow_login.html" - - defp is_status?(acct) do - case Pleroma.Object.Fetcher.fetch_and_contain_remote_object_from_id(acct) do - {:ok, %{"type" => type}} - when type in ["Article", "Event", "Note", "Video", "Page", "Question"] -> - true - - _ -> - false - end - end - - def do_remote_follow(conn, %{ - "authorization" => %{"name" => username, "password" => password, "id" => id} - }) do - with %User{} = followee <- User.get_cached_by_id(id), - {_, %User{} = user, _} <- {:auth, User.get_cached_by_nickname(username), followee}, - {_, true, _} <- { - :auth, - AuthenticationPlug.checkpw(password, user.password_hash), - followee - }, - {:ok, _follower, _followee, _activity} <- CommonAPI.follow(user, followee) do - conn - |> render("followed.html", %{error: false}) - else - # Was already following user - {:error, "Could not follow user:" <> _rest} -> - render(conn, "followed.html", %{error: "Error following account"}) - - {:auth, _, followee} -> - conn - |> render("follow_login.html", %{ - error: "Wrong username or password", - id: id, - name: followee.nickname, - avatar: User.avatar_url(followee) - }) - - e -> - Logger.debug("Remote follow failed with error #{inspect(e)}") - render(conn, "followed.html", %{error: "Something went wrong."}) - end - end - - def do_remote_follow(%{assigns: %{user: user}} = conn, %{"user" => %{"id" => id}}) do - with {:fetch_user, %User{} = followee} <- {:fetch_user, User.get_cached_by_id(id)}, - {:ok, _follower, _followee, _activity} <- CommonAPI.follow(user, followee) do - conn - |> render("followed.html", %{error: false}) - else - # Was already following user - {:error, "Could not follow user:" <> _rest} -> - render(conn, "followed.html", %{error: "Error following account"}) - - {:fetch_user, error} -> - Logger.debug("Remote follow failed with error #{inspect(error)}") - render(conn, "followed.html", %{error: "Could not find user"}) - - e -> - Logger.debug("Remote follow failed with error #{inspect(e)}") - render(conn, "followed.html", %{error: "Something went wrong."}) - end - end - def notifications_read(%{assigns: %{user: user}} = conn, %{"id" => notification_id}) do with {:ok, _} <- Notification.read_one(user, notification_id) do json(conn, %{status: "success"}) diff --git a/lib/pleroma/web/twitter_api/views/remote_follow_view.ex b/lib/pleroma/web/twitter_api/views/remote_follow_view.ex new file mode 100644 index 000000000..8f1f21bce --- /dev/null +++ b/lib/pleroma/web/twitter_api/views/remote_follow_view.ex @@ -0,0 +1,10 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.TwitterAPI.RemoteFollowView do + use Pleroma.Web, :view + import Phoenix.HTML.Form + + def avatar_url(user), do: Pleroma.User.avatar_url(user) +end diff --git a/test/web/activity_pub/publisher_test.exs b/test/web/activity_pub/publisher_test.exs index e885e5a5a..015af19ab 100644 --- a/test/web/activity_pub/publisher_test.exs +++ b/test/web/activity_pub/publisher_test.exs @@ -23,6 +23,27 @@ defmodule Pleroma.Web.ActivityPub.PublisherTest do :ok end + describe "gather_webfinger_links/1" do + test "it returns links" do + user = insert(:user) + + expected_links = [ + %{"href" => user.ap_id, "rel" => "self", "type" => "application/activity+json"}, + %{ + "href" => user.ap_id, + "rel" => "self", + "type" => "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"" + }, + %{ + "rel" => "http://ostatus.org/schema/1.0/subscribe", + "template" => "#{Pleroma.Web.base_url()}/ostatus_subscribe?acct={uri}" + } + ] + + assert expected_links == Publisher.gather_webfinger_links(user) + end + end + describe "determine_inbox/2" do test "it returns sharedInbox for messages involving as:Public in to" do user = diff --git a/test/web/twitter_api/remote_follow_controller_test.exs b/test/web/twitter_api/remote_follow_controller_test.exs new file mode 100644 index 000000000..a828253b2 --- /dev/null +++ b/test/web/twitter_api/remote_follow_controller_test.exs @@ -0,0 +1,211 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.TwitterAPI.RemoteFollowControllerTest do + use Pleroma.Web.ConnCase + + alias Pleroma.User + alias Pleroma.Web.CommonAPI + import ExUnit.CaptureLog + import Pleroma.Factory + + setup do + Tesla.Mock.mock(fn env -> apply(HttpRequestMock, :request, [env]) end) + :ok + end + + clear_config([:instance]) + clear_config([:frontend_configurations, :pleroma_fe]) + clear_config([:user, :deny_follow_blocked]) + + describe "GET /ostatus_subscribe - remote_follow/2" do + test "adds status to pleroma instance if the `acct` is a status", %{conn: conn} do + conn = + get( + conn, + "/ostatus_subscribe?acct=https://mastodon.social/users/emelie/statuses/101849165031453009" + ) + + assert redirected_to(conn) =~ "/notice/" + end + + test "show follow account page if the `acct` is a account link", %{conn: conn} do + response = + conn + |> get("/ostatus_subscribe?acct=https://mastodon.social/users/emelie") + |> html_response(200) + + assert response =~ "Log in to follow" + end + + test "show follow page if the `acct` is a account link", %{conn: conn} do + user = insert(:user) + + response = + conn + |> assign(:user, user) + |> get("/ostatus_subscribe?acct=https://mastodon.social/users/emelie") + |> html_response(200) + + assert response =~ "Remote follow" + end + + test "show follow page with error when user cannot fecth by `acct` link", %{conn: conn} do + user = insert(:user) + + assert capture_log(fn -> + response = + conn + |> assign(:user, user) + |> get("/ostatus_subscribe?acct=https://mastodon.social/users/not_found") + + assert html_response(response, 200) =~ "Error fetching user" + end) =~ "Object has been deleted" + end + end + + describe "POST /ostatus_subscribe - do_remote_follow/2 with assigned user " do + test "follows user", %{conn: conn} do + user = insert(:user) + user2 = insert(:user) + + response = + conn + |> assign(:user, user) + |> post("/ostatus_subscribe", %{"user" => %{"id" => user2.id}}) + |> response(200) + + assert response =~ "Account followed!" + assert user2.follower_address in User.following(user) + end + + test "returns error when user is deactivated", %{conn: conn} do + user = insert(:user, deactivated: true) + user2 = insert(:user) + + response = + conn + |> assign(:user, user) + |> post("/ostatus_subscribe", %{"user" => %{"id" => user2.id}}) + |> response(200) + + assert response =~ "Error following account" + end + + test "returns error when user is blocked", %{conn: conn} do + Pleroma.Config.put([:user, :deny_follow_blocked], true) + user = insert(:user) + user2 = insert(:user) + + {:ok, _user_block} = Pleroma.User.block(user2, user) + + response = + conn + |> assign(:user, user) + |> post("/ostatus_subscribe", %{"user" => %{"id" => user2.id}}) + |> response(200) + + assert response =~ "Error following account" + end + + test "returns error when followee not found", %{conn: conn} do + user = insert(:user) + + response = + conn + |> assign(:user, user) + |> post("/ostatus_subscribe", %{"user" => %{"id" => "jimm"}}) + |> response(200) + + assert response =~ "Error following account" + end + + test "returns success result when user already in followers", %{conn: conn} do + user = insert(:user) + user2 = insert(:user) + {:ok, _, _, _} = CommonAPI.follow(user, user2) + + response = + conn + |> assign(:user, refresh_record(user)) + |> post("/ostatus_subscribe", %{"user" => %{"id" => user2.id}}) + |> response(200) + + assert response =~ "Account followed!" + end + end + + describe "POST /ostatus_subscribe - do_remote_follow/2 without assigned user " do + test "follows", %{conn: conn} do + user = insert(:user) + user2 = insert(:user) + + response = + conn + |> post("/ostatus_subscribe", %{ + "authorization" => %{"name" => user.nickname, "password" => "test", "id" => user2.id} + }) + |> response(200) + + assert response =~ "Account followed!" + assert user2.follower_address in User.following(user) + end + + test "returns error when followee not found", %{conn: conn} do + user = insert(:user) + + response = + conn + |> post("/ostatus_subscribe", %{ + "authorization" => %{"name" => user.nickname, "password" => "test", "id" => "jimm"} + }) + |> response(200) + + assert response =~ "Error following account" + end + + test "returns error when login invalid", %{conn: conn} do + user = insert(:user) + + response = + conn + |> post("/ostatus_subscribe", %{ + "authorization" => %{"name" => "jimm", "password" => "test", "id" => user.id} + }) + |> response(200) + + assert response =~ "Wrong username or password" + end + + test "returns error when password invalid", %{conn: conn} do + user = insert(:user) + user2 = insert(:user) + + response = + conn + |> post("/ostatus_subscribe", %{ + "authorization" => %{"name" => user.nickname, "password" => "42", "id" => user2.id} + }) + |> response(200) + + assert response =~ "Wrong username or password" + end + + test "returns error when user is blocked", %{conn: conn} do + Pleroma.Config.put([:user, :deny_follow_blocked], true) + user = insert(:user) + user2 = insert(:user) + {:ok, _user_block} = Pleroma.User.block(user2, user) + + response = + conn + |> post("/ostatus_subscribe", %{ + "authorization" => %{"name" => user.nickname, "password" => "test", "id" => user2.id} + }) + |> response(200) + + assert response =~ "Error following account" + end + end +end diff --git a/test/web/twitter_api/util_controller_test.exs b/test/web/twitter_api/util_controller_test.exs index 43299e147..e65b251df 100644 --- a/test/web/twitter_api/util_controller_test.exs +++ b/test/web/twitter_api/util_controller_test.exs @@ -9,8 +9,8 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do alias Pleroma.Repo alias Pleroma.Tests.ObanHelpers alias Pleroma.User - alias Pleroma.Web.CommonAPI - import ExUnit.CaptureLog + # alias Pleroma.Web.CommonAPI + # import ExUnit.CaptureLog import Pleroma.Factory import Mock @@ -328,196 +328,6 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do end end - describe "GET /ostatus_subscribe - remote_follow/2" do - test "adds status to pleroma instance if the `acct` is a status", %{conn: conn} do - conn = - get( - conn, - "/ostatus_subscribe?acct=https://mastodon.social/users/emelie/statuses/101849165031453009" - ) - - assert redirected_to(conn) =~ "/notice/" - end - - test "show follow account page if the `acct` is a account link", %{conn: conn} do - response = - get( - conn, - "/ostatus_subscribe?acct=https://mastodon.social/users/emelie" - ) - - assert html_response(response, 200) =~ "Log in to follow" - end - - test "show follow page if the `acct` is a account link", %{conn: conn} do - user = insert(:user) - - response = - conn - |> assign(:user, user) - |> get("/ostatus_subscribe?acct=https://mastodon.social/users/emelie") - - assert html_response(response, 200) =~ "Remote follow" - end - - test "show follow page with error when user cannot fecth by `acct` link", %{conn: conn} do - user = insert(:user) - - assert capture_log(fn -> - response = - conn - |> assign(:user, user) - |> get("/ostatus_subscribe?acct=https://mastodon.social/users/not_found") - - assert html_response(response, 200) =~ "Error fetching user" - end) =~ "Object has been deleted" - end - end - - describe "POST /ostatus_subscribe - do_remote_follow/2 with assigned user " do - test "follows user", %{conn: conn} do - user = insert(:user) - user2 = insert(:user) - - response = - conn - |> assign(:user, user) - |> post("/ostatus_subscribe", %{"user" => %{"id" => user2.id}}) - |> response(200) - - assert response =~ "Account followed!" - assert user2.follower_address in User.following(user) - end - - test "returns error when user is deactivated", %{conn: conn} do - user = insert(:user, deactivated: true) - user2 = insert(:user) - - response = - conn - |> assign(:user, user) - |> post("/ostatus_subscribe", %{"user" => %{"id" => user2.id}}) - |> response(200) - - assert response =~ "Error following account" - end - - test "returns error when user is blocked", %{conn: conn} do - Pleroma.Config.put([:user, :deny_follow_blocked], true) - user = insert(:user) - user2 = insert(:user) - - {:ok, _user_block} = Pleroma.User.block(user2, user) - - response = - conn - |> assign(:user, user) - |> post("/ostatus_subscribe", %{"user" => %{"id" => user2.id}}) - |> response(200) - - assert response =~ "Error following account" - end - - test "returns error when followee not found", %{conn: conn} do - user = insert(:user) - - response = - conn - |> assign(:user, user) - |> post("/ostatus_subscribe", %{"user" => %{"id" => "jimm"}}) - |> response(200) - - assert response =~ "Error following account" - end - - test "returns success result when user already in followers", %{conn: conn} do - user = insert(:user) - user2 = insert(:user) - {:ok, _, _, _} = CommonAPI.follow(user, user2) - - response = - conn - |> assign(:user, refresh_record(user)) - |> post("/ostatus_subscribe", %{"user" => %{"id" => user2.id}}) - |> response(200) - - assert response =~ "Account followed!" - end - end - - describe "POST /ostatus_subscribe - do_remote_follow/2 without assigned user " do - test "follows", %{conn: conn} do - user = insert(:user) - user2 = insert(:user) - - response = - conn - |> post("/ostatus_subscribe", %{ - "authorization" => %{"name" => user.nickname, "password" => "test", "id" => user2.id} - }) - |> response(200) - - assert response =~ "Account followed!" - assert user2.follower_address in User.following(user) - end - - test "returns error when followee not found", %{conn: conn} do - user = insert(:user) - - response = - conn - |> post("/ostatus_subscribe", %{ - "authorization" => %{"name" => user.nickname, "password" => "test", "id" => "jimm"} - }) - |> response(200) - - assert response =~ "Error following account" - end - - test "returns error when login invalid", %{conn: conn} do - user = insert(:user) - - response = - conn - |> post("/ostatus_subscribe", %{ - "authorization" => %{"name" => "jimm", "password" => "test", "id" => user.id} - }) - |> response(200) - - assert response =~ "Wrong username or password" - end - - test "returns error when password invalid", %{conn: conn} do - user = insert(:user) - user2 = insert(:user) - - response = - conn - |> post("/ostatus_subscribe", %{ - "authorization" => %{"name" => user.nickname, "password" => "42", "id" => user2.id} - }) - |> response(200) - - assert response =~ "Wrong username or password" - end - - test "returns error when user is blocked", %{conn: conn} do - Pleroma.Config.put([:user, :deny_follow_blocked], true) - user = insert(:user) - user2 = insert(:user) - {:ok, _user_block} = Pleroma.User.block(user2, user) - - response = - conn - |> post("/ostatus_subscribe", %{ - "authorization" => %{"name" => user.nickname, "password" => "test", "id" => user2.id} - }) - |> response(200) - - assert response =~ "Error following account" - end - end - describe "GET /api/pleroma/healthcheck" do clear_config([:instance, :healthcheck]) From c9a44ec4a6f7b98145e2b192519dfa6933f430d0 Mon Sep 17 00:00:00 2001 From: Maksim Date: Sun, 22 Dec 2019 17:58:45 +0000 Subject: [PATCH 07/24] Apply suggestion to lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex --- .../web/twitter_api/controllers/remote_follow_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex b/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex index 460a42566..e5e52a7e8 100644 --- a/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex @@ -30,7 +30,7 @@ defmodule Pleroma.Web.TwitterAPI.RemoteFollowController do defp follow_status(conn, _user, acct) do with {:ok, object} <- Fetcher.fetch_object_from_id(acct), %Activity{id: activity_id} <- Activity.get_create_by_object_ap_id(object.data["id"]) do - redirect(conn, to: "/notice/#{activity_id}") + redirect(conn, to: o_status_path(conn, :notice, activity_id)) else error -> handle_follow_error(conn, error) From 4c505bc615b0e698db4f6d16c3b1f0b159f30e02 Mon Sep 17 00:00:00 2001 From: Maksim Date: Sun, 22 Dec 2019 17:58:54 +0000 Subject: [PATCH 08/24] Apply suggestion to lib/pleroma/web/twitter_api/views/remote_follow_view.ex --- lib/pleroma/web/twitter_api/views/remote_follow_view.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/twitter_api/views/remote_follow_view.ex b/lib/pleroma/web/twitter_api/views/remote_follow_view.ex index 8f1f21bce..fb6109906 100644 --- a/lib/pleroma/web/twitter_api/views/remote_follow_view.ex +++ b/lib/pleroma/web/twitter_api/views/remote_follow_view.ex @@ -6,5 +6,5 @@ defmodule Pleroma.Web.TwitterAPI.RemoteFollowView do use Pleroma.Web, :view import Phoenix.HTML.Form - def avatar_url(user), do: Pleroma.User.avatar_url(user) + defdelegate avatar_url(user), to: Pleroma.User.avatar_url end From bdd71669da43698716be6494528b6e1813d0cd3d Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Sun, 22 Dec 2019 21:17:19 +0300 Subject: [PATCH 09/24] update test --- .../twitter_api/views/remote_follow_view.ex | 2 +- .../remote_follow_controller_test.exs | 47 ++++++++++--------- 2 files changed, 27 insertions(+), 22 deletions(-) diff --git a/lib/pleroma/web/twitter_api/views/remote_follow_view.ex b/lib/pleroma/web/twitter_api/views/remote_follow_view.ex index fb6109906..d469c4726 100644 --- a/lib/pleroma/web/twitter_api/views/remote_follow_view.ex +++ b/lib/pleroma/web/twitter_api/views/remote_follow_view.ex @@ -6,5 +6,5 @@ defmodule Pleroma.Web.TwitterAPI.RemoteFollowView do use Pleroma.Web, :view import Phoenix.HTML.Form - defdelegate avatar_url(user), to: Pleroma.User.avatar_url + defdelegate avatar_url(user), to: Pleroma.User end diff --git a/test/web/twitter_api/remote_follow_controller_test.exs b/test/web/twitter_api/remote_follow_controller_test.exs index a828253b2..3f26a889d 100644 --- a/test/web/twitter_api/remote_follow_controller_test.exs +++ b/test/web/twitter_api/remote_follow_controller_test.exs @@ -21,19 +21,19 @@ defmodule Pleroma.Web.TwitterAPI.RemoteFollowControllerTest do describe "GET /ostatus_subscribe - remote_follow/2" do test "adds status to pleroma instance if the `acct` is a status", %{conn: conn} do - conn = - get( - conn, - "/ostatus_subscribe?acct=https://mastodon.social/users/emelie/statuses/101849165031453009" - ) - - assert redirected_to(conn) =~ "/notice/" + assert conn + |> get( + remote_follow_path(conn, :follow, %{ + acct: "https://mastodon.social/users/emelie/statuses/101849165031453009" + }) + ) + |> redirected_to() =~ "/notice/" end test "show follow account page if the `acct` is a account link", %{conn: conn} do response = conn - |> get("/ostatus_subscribe?acct=https://mastodon.social/users/emelie") + |> get(remote_follow_path(conn, :follow, %{acct: "https://mastodon.social/users/emelie"})) |> html_response(200) assert response =~ "Log in to follow" @@ -45,7 +45,7 @@ defmodule Pleroma.Web.TwitterAPI.RemoteFollowControllerTest do response = conn |> assign(:user, user) - |> get("/ostatus_subscribe?acct=https://mastodon.social/users/emelie") + |> get(remote_follow_path(conn, :follow, %{acct: "https://mastodon.social/users/emelie"})) |> html_response(200) assert response =~ "Remote follow" @@ -58,9 +58,14 @@ defmodule Pleroma.Web.TwitterAPI.RemoteFollowControllerTest do response = conn |> assign(:user, user) - |> get("/ostatus_subscribe?acct=https://mastodon.social/users/not_found") + |> get( + remote_follow_path(conn, :follow, %{ + acct: "https://mastodon.social/users/not_found" + }) + ) + |> html_response(200) - assert html_response(response, 200) =~ "Error fetching user" + assert response =~ "Error fetching user" end) =~ "Object has been deleted" end end @@ -73,7 +78,7 @@ defmodule Pleroma.Web.TwitterAPI.RemoteFollowControllerTest do response = conn |> assign(:user, user) - |> post("/ostatus_subscribe", %{"user" => %{"id" => user2.id}}) + |> post(remote_follow_path(conn, :do_follow), %{"user" => %{"id" => user2.id}}) |> response(200) assert response =~ "Account followed!" @@ -87,7 +92,7 @@ defmodule Pleroma.Web.TwitterAPI.RemoteFollowControllerTest do response = conn |> assign(:user, user) - |> post("/ostatus_subscribe", %{"user" => %{"id" => user2.id}}) + |> post(remote_follow_path(conn, :do_follow), %{"user" => %{"id" => user2.id}}) |> response(200) assert response =~ "Error following account" @@ -103,7 +108,7 @@ defmodule Pleroma.Web.TwitterAPI.RemoteFollowControllerTest do response = conn |> assign(:user, user) - |> post("/ostatus_subscribe", %{"user" => %{"id" => user2.id}}) + |> post(remote_follow_path(conn, :do_follow), %{"user" => %{"id" => user2.id}}) |> response(200) assert response =~ "Error following account" @@ -115,7 +120,7 @@ defmodule Pleroma.Web.TwitterAPI.RemoteFollowControllerTest do response = conn |> assign(:user, user) - |> post("/ostatus_subscribe", %{"user" => %{"id" => "jimm"}}) + |> post(remote_follow_path(conn, :do_follow), %{"user" => %{"id" => "jimm"}}) |> response(200) assert response =~ "Error following account" @@ -129,7 +134,7 @@ defmodule Pleroma.Web.TwitterAPI.RemoteFollowControllerTest do response = conn |> assign(:user, refresh_record(user)) - |> post("/ostatus_subscribe", %{"user" => %{"id" => user2.id}}) + |> post(remote_follow_path(conn, :do_follow), %{"user" => %{"id" => user2.id}}) |> response(200) assert response =~ "Account followed!" @@ -143,7 +148,7 @@ defmodule Pleroma.Web.TwitterAPI.RemoteFollowControllerTest do response = conn - |> post("/ostatus_subscribe", %{ + |> post(remote_follow_path(conn, :do_follow), %{ "authorization" => %{"name" => user.nickname, "password" => "test", "id" => user2.id} }) |> response(200) @@ -157,7 +162,7 @@ defmodule Pleroma.Web.TwitterAPI.RemoteFollowControllerTest do response = conn - |> post("/ostatus_subscribe", %{ + |> post(remote_follow_path(conn, :do_follow), %{ "authorization" => %{"name" => user.nickname, "password" => "test", "id" => "jimm"} }) |> response(200) @@ -170,7 +175,7 @@ defmodule Pleroma.Web.TwitterAPI.RemoteFollowControllerTest do response = conn - |> post("/ostatus_subscribe", %{ + |> post(remote_follow_path(conn, :do_follow), %{ "authorization" => %{"name" => "jimm", "password" => "test", "id" => user.id} }) |> response(200) @@ -184,7 +189,7 @@ defmodule Pleroma.Web.TwitterAPI.RemoteFollowControllerTest do response = conn - |> post("/ostatus_subscribe", %{ + |> post(remote_follow_path(conn, :do_follow), %{ "authorization" => %{"name" => user.nickname, "password" => "42", "id" => user2.id} }) |> response(200) @@ -200,7 +205,7 @@ defmodule Pleroma.Web.TwitterAPI.RemoteFollowControllerTest do response = conn - |> post("/ostatus_subscribe", %{ + |> post(remote_follow_path(conn, :do_follow), %{ "authorization" => %{"name" => user.nickname, "password" => "test", "id" => user2.id} }) |> response(200) From 933dc120438d14502e4bc4c29db904114fb6e438 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Wed, 25 Dec 2019 15:12:43 +0300 Subject: [PATCH 10/24] added code of mr#2067 --- .../controllers/remote_follow_controller.ex | 28 +++++++++++++------ .../remote_follow_controller_test.exs | 21 ++++++++++++-- 2 files changed, 38 insertions(+), 11 deletions(-) diff --git a/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex b/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex index e5e52a7e8..e0d4d5632 100644 --- a/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex @@ -16,7 +16,12 @@ defmodule Pleroma.Web.TwitterAPI.RemoteFollowController do @status_types ["Article", "Event", "Note", "Video", "Page", "Question"] - plug(OAuthScopesPlug, %{scopes: ["follow", "write:follows"]} when action in [:do_follow]) + # Note: follower can submit the form (with password auth) not being signed in (having no token) + plug( + OAuthScopesPlug, + %{fallback: :proceed_unauthenticated, scopes: ["follow", "write:follows"]} + when action in [:do_follow] + ) # GET /ostatus_subscribe # @@ -61,6 +66,16 @@ defmodule Pleroma.Web.TwitterAPI.RemoteFollowController do # POST /ostatus_subscribe # + def do_follow(%{assigns: %{user: %User{} = user}} = conn, %{"user" => %{"id" => id}}) do + with {:fetch_user, %User{} = followee} <- {:fetch_user, User.get_cached_by_id(id)}, + {:ok, _, _, _} <- CommonAPI.follow(user, followee) do + render(conn, "followed.html", %{error: false}) + else + error -> + handle_follow_error(conn, error) + end + end + def do_follow(conn, %{"authorization" => %{"name" => _, "password" => _, "id" => id}}) do with {:fetch_user, %User{} = followee} <- {:fetch_user, User.get_cached_by_id(id)}, {_, {:ok, user}, _} <- {:auth, Authenticator.get_user(conn), followee}, @@ -72,14 +87,9 @@ defmodule Pleroma.Web.TwitterAPI.RemoteFollowController do end end - def do_follow(%{assigns: %{user: user}} = conn, %{"user" => %{"id" => id}}) do - with {:fetch_user, %User{} = followee} <- {:fetch_user, User.get_cached_by_id(id)}, - {:ok, _, _, _} <- CommonAPI.follow(user, followee) do - render(conn, "followed.html", %{error: false}) - else - error -> - handle_follow_error(conn, error) - end + def do_follow(%{assigns: %{user: nil}} = conn, _) do + Logger.debug("Insufficient permissions: follow | write:follows.") + render(conn, "followed.html", %{error: "Insufficient permissions: follow | write:follows."}) end defp handle_follow_error(conn, {:auth, _, followee} = _) do diff --git a/test/web/twitter_api/remote_follow_controller_test.exs b/test/web/twitter_api/remote_follow_controller_test.exs index 3f26a889d..dd2f00dfe 100644 --- a/test/web/twitter_api/remote_follow_controller_test.exs +++ b/test/web/twitter_api/remote_follow_controller_test.exs @@ -70,7 +70,24 @@ defmodule Pleroma.Web.TwitterAPI.RemoteFollowControllerTest do end end - describe "POST /ostatus_subscribe - do_remote_follow/2 with assigned user " do + describe "POST /ostatus_subscribe - do_follow/2 with assigned user " do + test "required `follow | write:follows` scope", %{conn: conn} do + user = insert(:user) + user2 = insert(:user) + read_token = insert(:oauth_token, user: user, scopes: ["read"]) + + assert capture_log(fn -> + response = + conn + |> assign(:user, user) + |> assign(:token, read_token) + |> post(remote_follow_path(conn, :do_follow), %{"user" => %{"id" => user2.id}}) + |> response(200) + + assert response =~ "Error following account" + end) =~ "Insufficient permissions: follow | write:follows." + end + test "follows user", %{conn: conn} do user = insert(:user) user2 = insert(:user) @@ -141,7 +158,7 @@ defmodule Pleroma.Web.TwitterAPI.RemoteFollowControllerTest do end end - describe "POST /ostatus_subscribe - do_remote_follow/2 without assigned user " do + describe "POST /ostatus_subscribe - follow/2 without assigned user " do test "follows", %{conn: conn} do user = insert(:user) user2 = insert(:user) From fa7d8e77e64abbbd488152d8063fe9d012c8ac06 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Fri, 3 Jan 2020 16:21:52 +0300 Subject: [PATCH 11/24] fixed Metadata.Utils.scrub_html_and_truncate --- lib/pleroma/web/metadata/utils.ex | 2 ++ test/web/metadata/utils_test.exs | 32 +++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 test/web/metadata/utils_test.exs diff --git a/lib/pleroma/web/metadata/utils.ex b/lib/pleroma/web/metadata/utils.ex index 382ecf426..589d11901 100644 --- a/lib/pleroma/web/metadata/utils.ex +++ b/lib/pleroma/web/metadata/utils.ex @@ -15,6 +15,7 @@ defmodule Pleroma.Web.Metadata.Utils do |> String.replace(~r//, " ") |> HTML.get_cached_stripped_html_for_activity(object, "metadata") |> Emoji.Formatter.demojify() + |> HtmlEntities.decode() |> Formatter.truncate() end @@ -25,6 +26,7 @@ defmodule Pleroma.Web.Metadata.Utils do |> String.replace(~r//, " ") |> HTML.strip_tags() |> Emoji.Formatter.demojify() + |> HtmlEntities.decode() |> Formatter.truncate(max_length) end diff --git a/test/web/metadata/utils_test.exs b/test/web/metadata/utils_test.exs new file mode 100644 index 000000000..7547f2932 --- /dev/null +++ b/test/web/metadata/utils_test.exs @@ -0,0 +1,32 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Metadata.UtilsTest do + use Pleroma.DataCase + import Pleroma.Factory + alias Pleroma.Web.Metadata.Utils + + describe "scrub_html_and_truncate/1" do + test "it returns text without encode HTML" do + user = insert(:user) + + note = + insert(:note, %{ + data: %{ + "actor" => user.ap_id, + "id" => "https://pleroma.gov/objects/whatever", + "content" => "Pleroma's really cool!" + } + }) + + assert Utils.scrub_html_and_truncate(note) == "Pleroma's really cool!" + end + end + + describe "scrub_html_and_truncate/2" do + test "it returns text without encode HTML" do + assert Utils.scrub_html_and_truncate("Pleroma's really cool!") == "Pleroma's really cool!" + end + end +end From 0b6d1292d29e1f376566fe75aca60c612e9233dc Mon Sep 17 00:00:00 2001 From: eugenijm Date: Fri, 20 Dec 2019 16:38:21 +0300 Subject: [PATCH 12/24] Fix mark-as-read (`POST /api/v1/conversations/:id/read`) refreshing updated_at and bringing conversation to the top in the user's direct conversation list --- CHANGELOG.md | 1 + lib/pleroma/conversation/participation.ex | 10 ++++++---- test/conversation/participation_test.exs | 5 +++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 22f199b3d..efa3518e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -104,6 +104,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Inability to get some local users by nickname in `/api/v1/accounts/:id_or_nickname` - AdminAPI: If some status received reports both in the "new" format and "old" format it was considered reports on two different statuses (in the context of grouped reports) - Admin API: Error when trying to update reports in the "old" format +- Mastodon API: Marking a conversation as read (`POST /api/v1/conversations/:id/read`) brings it to the top in the user's direct conversation list ## [1.1.6] - 2019-11-19 diff --git a/lib/pleroma/conversation/participation.ex b/lib/pleroma/conversation/participation.ex index aafe57280..e5d28ebff 100644 --- a/lib/pleroma/conversation/participation.ex +++ b/lib/pleroma/conversation/participation.ex @@ -64,11 +64,13 @@ defmodule Pleroma.Conversation.Participation do end def mark_as_read(participation) do - participation - |> read_cng(%{read: true}) - |> Repo.update() + __MODULE__ + |> where(id: ^participation.id) + |> update(set: [read: true]) + |> select([p], p) + |> Repo.update_all([]) |> case do - {:ok, participation} -> + {1, [participation]} -> participation = Repo.preload(participation, :user) User.set_unread_conversation_count(participation.user) {:ok, participation} diff --git a/test/conversation/participation_test.exs b/test/conversation/participation_test.exs index ba81c0d4b..ab9f27b2f 100644 --- a/test/conversation/participation_test.exs +++ b/test/conversation/participation_test.exs @@ -125,9 +125,10 @@ defmodule Pleroma.Conversation.ParticipationTest do test "it marks a participation as read" do participation = insert(:participation, %{read: false}) - {:ok, participation} = Participation.mark_as_read(participation) + {:ok, updated_participation} = Participation.mark_as_read(participation) - assert participation.read + assert updated_participation.read + assert updated_participation.updated_at == participation.updated_at end test "it marks a participation as unread" do From 180f257ced4ace9467d1946a582a5f6f962d0163 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 6 Jan 2020 14:10:07 +0000 Subject: [PATCH 13/24] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index efa3518e4..80f0d98af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -104,7 +104,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Inability to get some local users by nickname in `/api/v1/accounts/:id_or_nickname` - AdminAPI: If some status received reports both in the "new" format and "old" format it was considered reports on two different statuses (in the context of grouped reports) - Admin API: Error when trying to update reports in the "old" format -- Mastodon API: Marking a conversation as read (`POST /api/v1/conversations/:id/read`) brings it to the top in the user's direct conversation list +- Mastodon API: Marking a conversation as read (`POST /api/v1/conversations/:id/read`) now no longer brings it to the top in the user's direct conversation list ## [1.1.6] - 2019-11-19 From 2ef8f0be6c1d852be1b91690e0702c69a225c147 Mon Sep 17 00:00:00 2001 From: jp Date: Fri, 10 Jan 2020 14:09:14 -0500 Subject: [PATCH 14/24] Update Dockerfile with labels. Update gitlab-ci for registry usage --- .gitlab-ci.yml | 27 +++++++++++++++++++++++++-- Dockerfile | 14 ++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dc85eaba2..b34c7e98d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,18 +6,24 @@ variables: &global_variables POSTGRES_PASSWORD: postgres DB_HOST: postgres MIX_ENV: test + DOCKER_DRIVER: overlay2 + DOCKER_HOST: unix:///var/run/docker.sock + DOCKER_IMAGE: $CI_REGISTRY_IMAGE:latest + DOCKER_IMAGE_SHA: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA cache: &global_cache_policy key: ${CI_COMMIT_REF_SLUG} paths: - - deps - - _build + - deps + - _build + stages: - build - test - benchmark - deploy - release + - docker before_script: - mix local.hex --force @@ -264,3 +270,20 @@ arm64-musl: variables: *release-variables before_script: *before-release-musl script: *release + +docker: + stage: docker + image: docker:latest + tags: + - dind + before_script: &before-docker + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + - export CI_JOB_TIMESTAMP=$(date --utc -Iseconds) + - export CI_VCS_REF=$CI_COMMIT_SHORT_SHA + script: + - docker pull $DOCKER_IMAGE || true + - docker build --cache-from $DOCKER_IMAGE --build-arg VCS_REF=$CI_VCS_REF --build-arg BUILD_DATE=$CI_JOB_TIMESTAMP -t $DOCKER_IMAGE_SHA -t $DOCKER_IMAGE . + - docker push $DOCKER_IMAGE_SHA + - docker push $DOCKER_IMAGE + only: + - develop diff --git a/Dockerfile b/Dockerfile index c61dcfde9..4f7f12716 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,6 +14,20 @@ RUN apk add git gcc g++ musl-dev make &&\ FROM alpine:3.9 +ARG BUILD_DATE +ARG VCS_REF + +LABEL maintainer="ops@pleroma.social" \ + org.opencontainers.image.title="pleroma" \ + org.opencontainers.image.description="Pleroma for Docker" \ + org.opencontainers.image.authors="ops@pleroma.social" \ + org.opencontainers.image.vendor="pleroma.social" \ + org.opencontainers.image.documentation="https://git.pleroma.social/pleroma/pleroma" \ + org.opencontainers.image.licenses="AGPL-3.0" \ + org.opencontainers.image.url="https://pleroma.social" \ + org.opencontainers.image.revision=$VCS_REF \ + org.opencontainers.image.created=$BUILD_DATE + ARG HOME=/opt/pleroma ARG DATA=/var/lib/pleroma From e1308f10bd13d404769a4765af3b870e2779e90e Mon Sep 17 00:00:00 2001 From: Alibek Omarov Date: Sat, 11 Jan 2020 05:06:40 +0000 Subject: [PATCH 15/24] docs: API: update Mastodon API link --- docs/API/differences_in_mastoapi_responses.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/API/differences_in_mastoapi_responses.md b/docs/API/differences_in_mastoapi_responses.md index 7f5d7681d..bb62ed5f2 100644 --- a/docs/API/differences_in_mastoapi_responses.md +++ b/docs/API/differences_in_mastoapi_responses.md @@ -46,7 +46,7 @@ The `id` parameter can also be the `nickname` of the user. This only works in th Has these additional fields under the `pleroma` object: - `tags`: Lists an array of tags for the user -- `relationship{}`: Includes fields as documented for Mastodon API https://docs.joinmastodon.org/api/entities/#relationship +- `relationship{}`: Includes fields as documented for Mastodon API https://docs.joinmastodon.org/entities/relationship/ - `is_moderator`: boolean, nullable, true if user is a moderator - `is_admin`: boolean, nullable, true if user is an admin - `confirmation_pending`: boolean, true if a new user account is waiting on email confirmation to be activated From 0c9c62509d10c19e2e6d796cb431f1560e9e88b1 Mon Sep 17 00:00:00 2001 From: Hakaba Hitoyo Date: Sat, 11 Jan 2020 17:19:54 +0000 Subject: [PATCH 16/24] Remove MDII uploader --- CHANGELOG.md | 1 + config/config.exs | 4 --- config/description.exs | 17 ------------ lib/pleroma/uploaders/mdii.ex | 37 -------------------------- test/uploaders/mdii_test.exs | 50 ----------------------------------- 5 files changed, 1 insertion(+), 108 deletions(-) delete mode 100644 lib/pleroma/uploaders/mdii.ex delete mode 100644 test/uploaders/mdii_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index 22f199b3d..04efc97c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Removed - **Breaking**: Removed 1.0+ deprecated configurations `Pleroma.Upload, :strip_exif` and `:instance, :dedupe_media` - **Breaking**: OStatus protocol support +- **Breaking**: MDII uploader ### Changed - **Breaking:** Elixir >=1.8 is now required (was >= 1.7) diff --git a/config/config.exs b/config/config.exs index 103361b29..d41abf090 100644 --- a/config/config.exs +++ b/config/config.exs @@ -108,10 +108,6 @@ config :pleroma, Pleroma.Uploaders.S3, streaming_enabled: true, public_endpoint: "https://s3.amazonaws.com" -config :pleroma, Pleroma.Uploaders.MDII, - cgi: "https://mdii.sakura.ne.jp/mdii-post.cgi", - files: "https://mdii.sakura.ne.jp" - config :pleroma, :emoji, shortcode_globs: ["/emoji/custom/**/*.png"], pack_extensions: [".png", ".gif"], diff --git a/config/description.exs b/config/description.exs index 45e4b43f1..1089fd86c 100644 --- a/config/description.exs +++ b/config/description.exs @@ -2557,23 +2557,6 @@ config :pleroma, :config_description, [ } ] }, - %{ - group: :pleroma, - key: Pleroma.Uploaders.MDII, - type: :group, - children: [ - %{ - key: :cgi, - type: :string, - suggestions: ["https://mdii.sakura.ne.jp/mdii-post.cgi"] - }, - %{ - key: :files, - type: :string, - suggestions: ["https://mdii.sakura.ne.jp"] - } - ] - }, %{ group: :pleroma, key: :http, diff --git a/lib/pleroma/uploaders/mdii.ex b/lib/pleroma/uploaders/mdii.ex deleted file mode 100644 index c36f3d61d..000000000 --- a/lib/pleroma/uploaders/mdii.ex +++ /dev/null @@ -1,37 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Uploaders.MDII do - @moduledoc "Represents uploader for https://github.com/hakaba-hitoyo/minimal-digital-image-infrastructure" - - alias Pleroma.Config - alias Pleroma.HTTP - - @behaviour Pleroma.Uploaders.Uploader - - # MDII-hosted images are never passed through the MediaPlug; only local media. - # Delegate to Pleroma.Uploaders.Local - def get_file(file) do - Pleroma.Uploaders.Local.get_file(file) - end - - def put_file(upload) do - cgi = Config.get([Pleroma.Uploaders.MDII, :cgi]) - files = Config.get([Pleroma.Uploaders.MDII, :files]) - - {:ok, file_data} = File.read(upload.tempfile) - - extension = String.split(upload.name, ".") |> List.last() - query = "#{cgi}?#{extension}" - - with {:ok, %{status: 200, body: body}} <- - HTTP.post(query, file_data, [], adapter: [pool: :default]) do - remote_file_name = String.split(body) |> List.first() - public_url = "#{files}/#{remote_file_name}.#{extension}" - {:ok, {:url, public_url}} - else - _ -> Pleroma.Uploaders.Local.put_file(upload) - end - end -end diff --git a/test/uploaders/mdii_test.exs b/test/uploaders/mdii_test.exs deleted file mode 100644 index d432d40f0..000000000 --- a/test/uploaders/mdii_test.exs +++ /dev/null @@ -1,50 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Uploaders.MDIITest do - use Pleroma.DataCase - alias Pleroma.Uploaders.MDII - import Tesla.Mock - - describe "get_file/1" do - test "it returns path to local folder for files" do - assert MDII.get_file("") == {:ok, {:static_dir, "test/uploads"}} - end - end - - describe "put_file/1" do - setup do - file_upload = %Pleroma.Upload{ - name: "mdii-image.jpg", - content_type: "image/jpg", - path: "test_folder/mdii-image.jpg", - tempfile: Path.absname("test/fixtures/image_tmp.jpg") - } - - [file_upload: file_upload] - end - - test "save file", %{file_upload: file_upload} do - mock(fn - %{method: :post, url: "https://mdii.sakura.ne.jp/mdii-post.cgi?jpg"} -> - %Tesla.Env{status: 200, body: "mdii-image"} - end) - - assert MDII.put_file(file_upload) == - {:ok, {:url, "https://mdii.sakura.ne.jp/mdii-image.jpg"}} - end - - test "save file to local if MDII isn`t available", %{file_upload: file_upload} do - mock(fn - %{method: :post, url: "https://mdii.sakura.ne.jp/mdii-post.cgi?jpg"} -> - %Tesla.Env{status: 500} - end) - - assert MDII.put_file(file_upload) == :ok - - assert Path.join([Pleroma.Uploaders.Local.upload_path(), file_upload.path]) - |> File.exists?() - end - end -end From e64059d218871ae3910dd00ba5bcffaafb96d74b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 12 Jan 2020 12:14:09 -0600 Subject: [PATCH 17/24] Assign token that can write to follows --- test/web/twitter_api/remote_follow_controller_test.exs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/web/twitter_api/remote_follow_controller_test.exs b/test/web/twitter_api/remote_follow_controller_test.exs index dd2f00dfe..444949375 100644 --- a/test/web/twitter_api/remote_follow_controller_test.exs +++ b/test/web/twitter_api/remote_follow_controller_test.exs @@ -95,6 +95,7 @@ defmodule Pleroma.Web.TwitterAPI.RemoteFollowControllerTest do response = conn |> assign(:user, user) + |> assign(:token, insert(:oauth_token, user: user, scopes: ["write:follows"])) |> post(remote_follow_path(conn, :do_follow), %{"user" => %{"id" => user2.id}}) |> response(200) @@ -151,6 +152,7 @@ defmodule Pleroma.Web.TwitterAPI.RemoteFollowControllerTest do response = conn |> assign(:user, refresh_record(user)) + |> assign(:token, insert(:oauth_token, user: user, scopes: ["write:follows"])) |> post(remote_follow_path(conn, :do_follow), %{"user" => %{"id" => user2.id}}) |> response(200) From 88f0eed0f24cb05949edcea49215ee939babac58 Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Sun, 12 Jan 2020 18:48:58 +0000 Subject: [PATCH 18/24] Delete attachments when status is deleted --- CHANGELOG.md | 1 + lib/pleroma/object.ex | 88 +++++++++++++++++++++++++++++++ lib/pleroma/uploaders/local.ex | 13 +++++ lib/pleroma/uploaders/s3.ex | 14 +++++ lib/pleroma/uploaders/uploader.ex | 3 +- test/object_test.exs | 68 ++++++++++++++++++++++++ test/uploaders/local_test.exs | 21 ++++++++ test/uploaders/s3_test.exs | 7 +++ 8 files changed, 214 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0907fbd53..397348304 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - **Breaking**: MDII uploader ### Changed +- **Breaking:** attachments are removed along with statuses when there are no other references to it - **Breaking:** Elixir >=1.8 is now required (was >= 1.7) - **Breaking:** attachment links (`config :pleroma, :instance, no_attachment_links` and `config :pleroma, Pleroma.Upload, link_name`) disabled by default - Replaced [pleroma_job_queue](https://git.pleroma.social/pleroma/pleroma_job_queue) and `Pleroma.Web.Federator.RetryQueue` with [Oban](https://github.com/sorentwo/oban) (see [`docs/config.md`](docs/config.md) on migrating customized worker / retry settings) diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index eb37b95a6..2452a7389 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -17,6 +17,8 @@ defmodule Pleroma.Object do require Logger + @type t() :: %__MODULE__{} + schema "objects" do field(:data, :map) @@ -79,6 +81,20 @@ defmodule Pleroma.Object do Repo.one(from(object in Object, where: fragment("(?)->>'id' = ?", object.data, ^ap_id))) end + @doc """ + Get a single attachment by it's name and href + """ + @spec get_attachment_by_name_and_href(String.t(), String.t()) :: Object.t() | nil + def get_attachment_by_name_and_href(name, href) do + query = + from(o in Object, + where: fragment("(?)->>'name' = ?", o.data, ^name), + where: fragment("(?)->>'href' = ?", o.data, ^href) + ) + + Repo.one(query) + end + defp warn_on_no_object_preloaded(ap_id) do "Object.normalize() called without preloaded object (#{inspect(ap_id)}). Consider preloading the object" |> Logger.debug() @@ -164,6 +180,7 @@ defmodule Pleroma.Object do def delete(%Object{data: %{"id" => id}} = object) do with {:ok, _obj} = swap_object_with_tombstone(object), + :ok <- delete_attachments(object), deleted_activity = Activity.delete_all_by_object_ap_id(id), {:ok, true} <- Cachex.del(:object_cache, "object:#{id}"), {:ok, _} <- Cachex.del(:web_resp_cache, URI.parse(id).path) do @@ -171,6 +188,77 @@ defmodule Pleroma.Object do end end + defp delete_attachments(%{data: %{"attachment" => [_ | _] = attachments, "actor" => actor}}) do + hrefs = + Enum.flat_map(attachments, fn attachment -> + Enum.map(attachment["url"], & &1["href"]) + end) + + names = Enum.map(attachments, & &1["name"]) + + uploader = Pleroma.Config.get([Pleroma.Upload, :uploader]) + + # find all objects for copies of the attachments, name and actor doesn't matter here + delete_ids = + from(o in Object, + where: + fragment( + "to_jsonb(array(select jsonb_array_elements((?)#>'{url}') ->> 'href'))::jsonb \\?| (?)", + o.data, + ^hrefs + ) + ) + |> Repo.all() + # we should delete 1 object for any given attachment, but don't delete files if + # there are more than 1 object for it + |> Enum.reduce(%{}, fn %{ + id: id, + data: %{ + "url" => [%{"href" => href}], + "actor" => obj_actor, + "name" => name + } + }, + acc -> + Map.update(acc, href, %{id: id, count: 1}, fn val -> + case obj_actor == actor and name in names do + true -> + # set id of the actor's object that will be deleted + %{val | id: id, count: val.count + 1} + + false -> + # another actor's object, just increase count to not delete file + %{val | count: val.count + 1} + end + end) + end) + |> Enum.map(fn {href, %{id: id, count: count}} -> + # only delete files that have single instance + with 1 <- count do + prefix = + case Pleroma.Config.get([Pleroma.Upload, :base_url]) do + nil -> "media" + _ -> "" + end + + base_url = Pleroma.Config.get([__MODULE__, :base_url], Pleroma.Web.base_url()) + + file_path = String.trim_leading(href, "#{base_url}/#{prefix}") + + uploader.delete_file(file_path) + end + + id + end) + + from(o in Object, where: o.id in ^delete_ids) + |> Repo.delete_all() + + :ok + end + + defp delete_attachments(%{data: _data}), do: :ok + def prune(%Object{data: %{"id" => id}} = object) do with {:ok, object} <- Repo.delete(object), {:ok, true} <- Cachex.del(:object_cache, "object:#{id}"), diff --git a/lib/pleroma/uploaders/local.ex b/lib/pleroma/uploaders/local.ex index 36b3c35ec..2e6fe3292 100644 --- a/lib/pleroma/uploaders/local.ex +++ b/lib/pleroma/uploaders/local.ex @@ -5,10 +5,12 @@ defmodule Pleroma.Uploaders.Local do @behaviour Pleroma.Uploaders.Uploader + @impl true def get_file(_) do {:ok, {:static_dir, upload_path()}} end + @impl true def put_file(upload) do {local_path, file} = case Enum.reverse(Path.split(upload.path)) do @@ -33,4 +35,15 @@ defmodule Pleroma.Uploaders.Local do def upload_path do Pleroma.Config.get!([__MODULE__, :uploads]) end + + @impl true + def delete_file(path) do + upload_path() + |> Path.join(path) + |> File.rm() + |> case do + :ok -> :ok + {:error, posix_error} -> {:error, to_string(posix_error)} + end + end end diff --git a/lib/pleroma/uploaders/s3.ex b/lib/pleroma/uploaders/s3.ex index 9876b6398..feb89cea6 100644 --- a/lib/pleroma/uploaders/s3.ex +++ b/lib/pleroma/uploaders/s3.ex @@ -10,6 +10,7 @@ defmodule Pleroma.Uploaders.S3 do # The file name is re-encoded with S3's constraints here to comply with previous # links with less strict filenames + @impl true def get_file(file) do config = Config.get([__MODULE__]) bucket = Keyword.fetch!(config, :bucket) @@ -35,6 +36,7 @@ defmodule Pleroma.Uploaders.S3 do ])}} end + @impl true def put_file(%Pleroma.Upload{} = upload) do config = Config.get([__MODULE__]) bucket = Keyword.get(config, :bucket) @@ -69,6 +71,18 @@ defmodule Pleroma.Uploaders.S3 do end end + @impl true + def delete_file(file) do + [__MODULE__, :bucket] + |> Config.get() + |> ExAws.S3.delete_object(file) + |> ExAws.request() + |> case do + {:ok, %{status_code: 204}} -> :ok + error -> {:error, inspect(error)} + end + end + @regex Regex.compile!("[^0-9a-zA-Z!.*/'()_-]") def strict_encode(name) do String.replace(name, @regex, "-") diff --git a/lib/pleroma/uploaders/uploader.ex b/lib/pleroma/uploaders/uploader.ex index c0b22c28a..d71e213d2 100644 --- a/lib/pleroma/uploaders/uploader.ex +++ b/lib/pleroma/uploaders/uploader.ex @@ -36,6 +36,8 @@ defmodule Pleroma.Uploaders.Uploader do @callback put_file(Pleroma.Upload.t()) :: :ok | {:ok, file_spec()} | {:error, String.t()} | :wait_callback + @callback delete_file(file :: String.t()) :: :ok | {:error, String.t()} + @callback http_callback(Plug.Conn.t(), Map.t()) :: {:ok, Plug.Conn.t()} | {:ok, Plug.Conn.t(), file_spec()} @@ -43,7 +45,6 @@ defmodule Pleroma.Uploaders.Uploader do @optional_callbacks http_callback: 2 @spec put_file(module(), Pleroma.Upload.t()) :: {:ok, file_spec()} | {:error, String.t()} - def put_file(uploader, upload) do case uploader.put_file(upload) do :ok -> {:ok, {:file, upload.path}} diff --git a/test/object_test.exs b/test/object_test.exs index 9247a6d84..b002c2bae 100644 --- a/test/object_test.exs +++ b/test/object_test.exs @@ -71,6 +71,74 @@ defmodule Pleroma.ObjectTest do end end + describe "delete attachments" do + clear_config([Pleroma.Upload]) + + test "in subdirectories" do + Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) + + file = %Plug.Upload{ + content_type: "image/jpg", + path: Path.absname("test/fixtures/image.jpg"), + filename: "an_image.jpg" + } + + user = insert(:user) + + {:ok, %Object{} = attachment} = + Pleroma.Web.ActivityPub.ActivityPub.upload(file, actor: user.ap_id) + + %{data: %{"attachment" => [%{"url" => [%{"href" => href}]}]}} = + note = insert(:note, %{user: user, data: %{"attachment" => [attachment.data]}}) + + uploads_dir = Pleroma.Config.get!([Pleroma.Uploaders.Local, :uploads]) + + path = href |> Path.dirname() |> Path.basename() + + assert {:ok, ["an_image.jpg"]} == File.ls("#{uploads_dir}/#{path}") + + Object.delete(note) + + assert Object.get_by_id(attachment.id) == nil + + assert {:ok, []} == File.ls("#{uploads_dir}/#{path}") + end + + test "with dedupe enabled" do + Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) + Pleroma.Config.put([Pleroma.Upload, :filters], [Pleroma.Upload.Filter.Dedupe]) + + uploads_dir = Pleroma.Config.get!([Pleroma.Uploaders.Local, :uploads]) + + File.mkdir_p!(uploads_dir) + + file = %Plug.Upload{ + content_type: "image/jpg", + path: Path.absname("test/fixtures/image.jpg"), + filename: "an_image.jpg" + } + + user = insert(:user) + + {:ok, %Object{} = attachment} = + Pleroma.Web.ActivityPub.ActivityPub.upload(file, actor: user.ap_id) + + %{data: %{"attachment" => [%{"url" => [%{"href" => href}]}]}} = + note = insert(:note, %{user: user, data: %{"attachment" => [attachment.data]}}) + + filename = Path.basename(href) + + assert {:ok, files} = File.ls(uploads_dir) + assert filename in files + + Object.delete(note) + + assert Object.get_by_id(attachment.id) == nil + assert {:ok, files} = File.ls(uploads_dir) + refute filename in files + end + end + describe "normalizer" do test "fetches unknown objects by default" do %Object{} = diff --git a/test/uploaders/local_test.exs b/test/uploaders/local_test.exs index fc442d0f1..1963dac23 100644 --- a/test/uploaders/local_test.exs +++ b/test/uploaders/local_test.exs @@ -29,4 +29,25 @@ defmodule Pleroma.Uploaders.LocalTest do |> File.exists?() end end + + describe "delete_file/1" do + test "deletes local file" do + file_path = "local_upload/files/image.jpg" + + file = %Pleroma.Upload{ + name: "image.jpg", + content_type: "image/jpg", + path: file_path, + tempfile: Path.absname("test/fixtures/image_tmp.jpg") + } + + :ok = Local.put_file(file) + local_path = Path.join([Local.upload_path(), file_path]) + assert File.exists?(local_path) + + Local.delete_file(file_path) + + refute File.exists?(local_path) + end + end end diff --git a/test/uploaders/s3_test.exs b/test/uploaders/s3_test.exs index 171316340..ab7795c3b 100644 --- a/test/uploaders/s3_test.exs +++ b/test/uploaders/s3_test.exs @@ -79,4 +79,11 @@ defmodule Pleroma.Uploaders.S3Test do end end end + + describe "delete_file/1" do + test_with_mock "deletes file", ExAws, request: fn _req -> {:ok, %{status_code: 204}} end do + assert :ok = S3.delete_file("image.jpg") + assert_called(ExAws.request(:_)) + end + end end From 0d331f3d23963790b63f5b45b56adfe71651c7ee Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 14 Jan 2020 15:49:52 +0300 Subject: [PATCH 19/24] compilation warnings fix --- test/web/twitter_api/util_controller_test.exs | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/web/twitter_api/util_controller_test.exs b/test/web/twitter_api/util_controller_test.exs index 8418fd071..5d60c0d51 100644 --- a/test/web/twitter_api/util_controller_test.exs +++ b/test/web/twitter_api/util_controller_test.exs @@ -8,9 +8,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do alias Pleroma.Tests.ObanHelpers alias Pleroma.User - alias Pleroma.Web.CommonAPI - import ExUnit.CaptureLog import Pleroma.Factory import Mock From fe57e5139fb8396e2b929998850710d73de587ff Mon Sep 17 00:00:00 2001 From: jp Date: Tue, 14 Jan 2020 12:32:36 -0500 Subject: [PATCH 20/24] Remove cache from docker jobs. Split devlop and stable branches into their own jobs --- .gitlab-ci.yml | 43 +++++++++++++++++++++++++++++++++---------- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b34c7e98d..ba6e41aef 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,10 +6,6 @@ variables: &global_variables POSTGRES_PASSWORD: postgres DB_HOST: postgres MIX_ENV: test - DOCKER_DRIVER: overlay2 - DOCKER_HOST: unix:///var/run/docker.sock - DOCKER_IMAGE: $CI_REGISTRY_IMAGE:latest - DOCKER_IMAGE_SHA: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA cache: &global_cache_policy key: ${CI_COMMIT_REF_SLUG} @@ -274,16 +270,43 @@ arm64-musl: docker: stage: docker image: docker:latest - tags: - - dind + cache: {} + variables: &docker-variables + DOCKER_DRIVER: overlay2 + DOCKER_HOST: unix:///var/run/docker.sock + IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA + IMAGE_TAG_SLUG: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG + IMAGE_TAG_LATEST: $CI_REGISTRY_IMAGE:latest + IMAGE_TAG_LATEST_STABLE: $CI_REGISTRY_IMAGE:latest-stable before_script: &before-docker - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - export CI_JOB_TIMESTAMP=$(date --utc -Iseconds) - export CI_VCS_REF=$CI_COMMIT_SHORT_SHA script: - - docker pull $DOCKER_IMAGE || true - - docker build --cache-from $DOCKER_IMAGE --build-arg VCS_REF=$CI_VCS_REF --build-arg BUILD_DATE=$CI_JOB_TIMESTAMP -t $DOCKER_IMAGE_SHA -t $DOCKER_IMAGE . - - docker push $DOCKER_IMAGE_SHA - - docker push $DOCKER_IMAGE + - docker pull $IMAGE_TAG_SLUG || true + - docker build --cache-from $IMAGE_TAG_SLUG --build-arg VCS_REF=$CI_VCS_REF --build-arg BUILD_DATE=$CI_JOB_TIMESTAMP -t $IMAGE_TAG -t $IMAGE_TAG_SLUG -t $IMAGE_TAG_LATEST . + - docker push $IMAGE_TAG + - docker push $IMAGE_TAG_SLUG + - docker push $IMAGE_TAG_LATEST + tags: + - dind only: - develop + - features/docker-updates + +docker-stable: + stage: docker + image: docker:latest + cache: {} + variables: *docker-variables + before_script: *before-docker + script: + - docker pull $IMAGE_TAG_SLUG || true + - docker build --cache-from $IMAGE_TAG_SLUG --build-arg VCS_REF=$CI_VCS_REF --build-arg BUILD_DATE=$CI_JOB_TIMESTAMP -t $IMAGE_TAG -t $IMAGE_TAG_SLUG -t $IMAGE_TAG_LATEST_STABLE . + - docker push $IMAGE_TAG + - docker push $IMAGE_TAG_SLUG + - docker push $IMAGE_TAG_LATEST_STABLE + tags: + - dind + only: + - stable From 964d188a96f8d746b6d1c0ca06c9abc5bc682a11 Mon Sep 17 00:00:00 2001 From: jp Date: Tue, 14 Jan 2020 12:43:28 -0500 Subject: [PATCH 21/24] Add allow_failure to docker jobs --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ba6e41aef..b47bfb598 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -282,6 +282,7 @@ docker: - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - export CI_JOB_TIMESTAMP=$(date --utc -Iseconds) - export CI_VCS_REF=$CI_COMMIT_SHORT_SHA + allow_failure: true script: - docker pull $IMAGE_TAG_SLUG || true - docker build --cache-from $IMAGE_TAG_SLUG --build-arg VCS_REF=$CI_VCS_REF --build-arg BUILD_DATE=$CI_JOB_TIMESTAMP -t $IMAGE_TAG -t $IMAGE_TAG_SLUG -t $IMAGE_TAG_LATEST . @@ -292,7 +293,6 @@ docker: - dind only: - develop - - features/docker-updates docker-stable: stage: docker @@ -300,6 +300,7 @@ docker-stable: cache: {} variables: *docker-variables before_script: *before-docker + allow_failure: true script: - docker pull $IMAGE_TAG_SLUG || true - docker build --cache-from $IMAGE_TAG_SLUG --build-arg VCS_REF=$CI_VCS_REF --build-arg BUILD_DATE=$CI_JOB_TIMESTAMP -t $IMAGE_TAG -t $IMAGE_TAG_SLUG -t $IMAGE_TAG_LATEST_STABLE . From ff15d6ef139a397002954da565b15861fdc65ca1 Mon Sep 17 00:00:00 2001 From: jp Date: Tue, 14 Jan 2020 13:45:47 -0500 Subject: [PATCH 22/24] Remove artifacts passing by setting `dependencies: []` --- .gitlab-ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b47bfb598..af9cf5f24 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -271,6 +271,7 @@ docker: stage: docker image: docker:latest cache: {} + dependencies: [] variables: &docker-variables DOCKER_DRIVER: overlay2 DOCKER_HOST: unix:///var/run/docker.sock @@ -293,11 +294,13 @@ docker: - dind only: - develop + - features/docker-updates docker-stable: stage: docker image: docker:latest cache: {} + dependencies: [] variables: *docker-variables before_script: *before-docker allow_failure: true From a58a0a7b1b85f900c52c63fc9a4c55a84431d0b0 Mon Sep 17 00:00:00 2001 From: jp Date: Tue, 14 Jan 2020 13:47:36 -0500 Subject: [PATCH 23/24] Remove forked test branch matching --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index af9cf5f24..2b2601082 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -294,7 +294,6 @@ docker: - dind only: - develop - - features/docker-updates docker-stable: stage: docker From 023b7f605b5736b561f5b3a59de4d602933d7c71 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 15 Jan 2020 16:51:09 +0400 Subject: [PATCH 24/24] Fix notification controller test --- .../controllers/notification_controller_test.exs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/web/mastodon_api/controllers/notification_controller_test.exs b/test/web/mastodon_api/controllers/notification_controller_test.exs index 38161982a..6f0606250 100644 --- a/test/web/mastodon_api/controllers/notification_controller_test.exs +++ b/test/web/mastodon_api/controllers/notification_controller_test.exs @@ -458,8 +458,9 @@ defmodule Pleroma.Web.MastodonAPI.NotificationControllerTest do end describe "from specified user" do - test "account_id", %{conn: conn} do - user = insert(:user) + test "account_id" do + %{user: user, conn: conn} = oauth_access(["read:notifications"]) + %{id: account_id} = other_user1 = insert(:user) other_user2 = insert(:user)