From 17bfd000d7d44ff13cf7becbfd9ce08b896d66eb Mon Sep 17 00:00:00 2001 From: Sachin Joshi Date: Wed, 22 May 2019 06:39:19 +0200 Subject: [PATCH 01/43] Ability to reset avatar, profile banner and backgroud --- CHANGELOG.md | 1 + .../web/twitter_api/twitter_api_controller.ex | 35 ++++++++++++++++++ .../twitter_api_controller_test.exs | 37 +++++++++++++++++++ 3 files changed, 73 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 12c439135..d4a74d30c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Metadata: RelMe provider - OAuth: added support for refresh tokens - Emoji packs and emoji pack manager +- Ability to reset avatar, profile banner and backgroud ### Changed - **Breaking:** Configuration: move from Pleroma.Mailer to Pleroma.Emails.Mailer diff --git a/lib/pleroma/web/twitter_api/twitter_api_controller.ex b/lib/pleroma/web/twitter_api/twitter_api_controller.ex index 31e86685a..a796e38fd 100644 --- a/lib/pleroma/web/twitter_api/twitter_api_controller.ex +++ b/lib/pleroma/web/twitter_api/twitter_api_controller.ex @@ -456,6 +456,16 @@ defmodule Pleroma.Web.TwitterAPI.Controller do end end + def update_avatar(%{assigns: %{user: user}} = conn, %{"img" => ""}) do + change = Changeset.change(user, %{avatar: nil}) + {:ok, user} = User.update_and_set_cache(change) + CommonAPI.update(user) + + conn + |> put_view(UserView) + |> render("show.json", %{user: user, for: user}) + end + def update_avatar(%{assigns: %{user: user}} = conn, params) do {:ok, object} = ActivityPub.upload(params, type: :avatar) change = Changeset.change(user, %{avatar: object.data}) @@ -467,6 +477,19 @@ defmodule Pleroma.Web.TwitterAPI.Controller do |> render("show.json", %{user: user, for: user}) end + def update_banner(%{assigns: %{user: user}} = conn, %{"banner" => ""}) do + with new_info <- %{"banner" => %{}}, + info_cng <- User.Info.profile_update(user.info, new_info), + changeset <- Ecto.Changeset.change(user) |> Ecto.Changeset.put_embed(:info, info_cng), + {:ok, user} <- User.update_and_set_cache(changeset) do + CommonAPI.update(user) + response = %{url: nil} |> Jason.encode!() + + conn + |> json_reply(200, response) + end + end + def update_banner(%{assigns: %{user: user}} = conn, params) do with {:ok, object} <- ActivityPub.upload(%{"img" => params["banner"]}, type: :banner), new_info <- %{"banner" => object.data}, @@ -482,6 +505,18 @@ defmodule Pleroma.Web.TwitterAPI.Controller do end end + def update_background(%{assigns: %{user: user}} = conn, %{"img" => ""}) do + with new_info <- %{"background" => %{}}, + info_cng <- User.Info.profile_update(user.info, new_info), + changeset <- Ecto.Changeset.change(user) |> Ecto.Changeset.put_embed(:info, info_cng), + {:ok, _user} <- User.update_and_set_cache(changeset) do + response = %{url: nil} |> Jason.encode!() + + conn + |> json_reply(200, response) + end + end + def update_background(%{assigns: %{user: user}} = conn, params) do with {:ok, object} <- ActivityPub.upload(params, type: :background), new_info <- %{"background" => object.data}, diff --git a/test/web/twitter_api/twitter_api_controller_test.exs b/test/web/twitter_api/twitter_api_controller_test.exs index e194f14fb..373efa639 100644 --- a/test/web/twitter_api/twitter_api_controller_test.exs +++ b/test/web/twitter_api/twitter_api_controller_test.exs @@ -40,6 +40,18 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do user = refresh_record(user) assert user.info.banner["type"] == "Image" end + + test "profile banner can be reset", %{conn: conn} do + user = insert(:user) + + conn + |> assign(:user, user) + |> post(authenticated_twitter_api__path(conn, :update_banner), %{"banner" => ""}) + |> json_response(200) + + user = refresh_record(user) + assert user.info.banner == %{} + end end describe "POST /api/qvitter/update_background_image" do @@ -54,6 +66,18 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do user = refresh_record(user) assert user.info.background["type"] == "Image" end + + test "background can be reset", %{conn: conn} do + user = insert(:user) + + conn + |> assign(:user, user) + |> post(authenticated_twitter_api__path(conn, :update_background), %{"img" => ""}) + |> json_response(200) + + user = refresh_record(user) + assert user.info.background == %{} + end end describe "POST /api/account/verify_credentials" do @@ -853,6 +877,19 @@ defmodule Pleroma.Web.TwitterAPI.ControllerTest do assert json_response(conn, 200) == UserView.render("show.json", %{user: current_user, for: current_user}) end + + test "user avatar can be reset", %{conn: conn, user: current_user} do + conn = + conn + |> with_credentials(current_user.nickname, "test") + |> post("/api/qvitter/update_avatar.json", %{img: ""}) + + current_user = User.get_cached_by_id(current_user.id) + assert current_user.avatar == nil + + assert json_response(conn, 200) == + UserView.render("show.json", %{user: current_user, for: current_user}) + end end describe "GET /api/qvitter/mutes.json" do From 1452a96ad6cfd7d250e3f7c10805994cc92016a7 Mon Sep 17 00:00:00 2001 From: Sachin Joshi Date: Mon, 27 May 2019 15:31:01 +0545 Subject: [PATCH 02/43] ability to set and reset avatar, profile banner and backgroud in Mastodon API --- .../mastodon_api/mastodon_api_controller.ex | 63 ++++++++++++ lib/pleroma/web/router.ex | 4 + .../mastodon_api_controller_test.exs | 97 +++++++++++++++++++ 3 files changed, 164 insertions(+) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 1ec0f30a1..1ff839e9e 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -152,6 +152,69 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do end end + def update_avatar(%{assigns: %{user: user}} = conn, %{"img" => ""}) do + change = Changeset.change(user, %{avatar: nil}) + {:ok, user} = User.update_and_set_cache(change) + CommonAPI.update(user) + + json(conn, %{url: nil}) + end + + def update_avatar(%{assigns: %{user: user}} = conn, params) do + {:ok, object} = ActivityPub.upload(params, type: :avatar) + change = Changeset.change(user, %{avatar: object.data}) + {:ok, user} = User.update_and_set_cache(change) + CommonAPI.update(user) + %{"url" => [%{"href" => href} | _]} = object.data + + json(conn, %{url: href}) + end + + def update_banner(%{assigns: %{user: user}} = conn, %{"banner" => ""}) do + with new_info <- %{"banner" => %{}}, + info_cng <- User.Info.profile_update(user.info, new_info), + changeset <- Ecto.Changeset.change(user) |> Ecto.Changeset.put_embed(:info, info_cng), + {:ok, user} <- User.update_and_set_cache(changeset) do + CommonAPI.update(user) + + json(conn, %{url: nil}) + end + end + + def update_banner(%{assigns: %{user: user}} = conn, params) do + with {:ok, object} <- ActivityPub.upload(%{"img" => params["banner"]}, type: :banner), + new_info <- %{"banner" => object.data}, + info_cng <- User.Info.profile_update(user.info, new_info), + changeset <- Ecto.Changeset.change(user) |> Ecto.Changeset.put_embed(:info, info_cng), + {:ok, user} <- User.update_and_set_cache(changeset) do + CommonAPI.update(user) + %{"url" => [%{"href" => href} | _]} = object.data + + json(conn, %{url: href}) + end + end + + def update_background(%{assigns: %{user: user}} = conn, %{"img" => ""}) do + with new_info <- %{"background" => %{}}, + info_cng <- User.Info.profile_update(user.info, new_info), + changeset <- Ecto.Changeset.change(user) |> Ecto.Changeset.put_embed(:info, info_cng), + {:ok, _user} <- User.update_and_set_cache(changeset) do + json(conn, %{url: nil}) + end + end + + def update_background(%{assigns: %{user: user}} = conn, params) do + with {:ok, object} <- ActivityPub.upload(params, type: :background), + new_info <- %{"background" => object.data}, + info_cng <- User.Info.profile_update(user.info, new_info), + changeset <- Ecto.Changeset.change(user) |> Ecto.Changeset.put_embed(:info, info_cng), + {:ok, _user} <- User.update_and_set_cache(changeset) do + %{"url" => [%{"href" => href} | _]} = object.data + + json(conn, %{url: href}) + end + end + def verify_credentials(%{assigns: %{user: user}} = conn, _) do account = AccountView.render("account.json", %{user: user, for: user}) json(conn, account) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 352268b96..42ef64c4f 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -318,6 +318,10 @@ defmodule Pleroma.Web.Router do patch("/accounts/update_credentials", MastodonAPIController, :update_credentials) + patch("/accounts/update_avatar", MastodonAPIController, :update_avatar) + patch("/accounts/update_banner", MastodonAPIController, :update_banner) + patch("/accounts/update_background", MastodonAPIController, :update_background) + post("/statuses", MastodonAPIController, :post_status) delete("/statuses/:id", MastodonAPIController, :delete_status) diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 1d9f5a816..90ca26441 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -24,6 +24,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do import ExUnit.CaptureLog import Tesla.Mock + @image "data:image/gif;base64,R0lGODlhEAAQAMQAAORHHOVSKudfOulrSOp3WOyDZu6QdvCchPGolfO0o/XBs/fNwfjZ0frl3/zy7////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACH5BAkAABAALAAAAAAQABAAAAVVICSOZGlCQAosJ6mu7fiyZeKqNKToQGDsM8hBADgUXoGAiqhSvp5QAnQKGIgUhwFUYLCVDFCrKUE1lBavAViFIDlTImbKC5Gm2hB0SlBCBMQiB0UjIQA7" + setup do mock(fn env -> apply(HttpRequestMock, :request, [env]) end) :ok @@ -477,6 +479,101 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do assert expected == json_response(conn, 200) end + test "user avatar can be set", %{conn: conn} do + user = insert(:user) + avatar_image = File.read!("test/fixtures/avatar_data_uri") + + conn = + conn + |> assign(:user, user) + |> patch("/api/v1/accounts/update_avatar", %{img: avatar_image}) + + user = refresh_record(user) + + assert %{ + "name" => _, + "type" => _, + "url" => [ + %{ + "href" => _, + "mediaType" => _, + "type" => _ + } + ] + } = user.avatar + + assert %{"url" => _} = json_response(conn, 200) + end + + test "user avatar can be reset", %{conn: conn} do + user = insert(:user) + + conn = + conn + |> assign(:user, user) + |> patch("/api/v1/accounts/update_avatar", %{img: ""}) + + user = User.get_cached_by_id(user.id) + + assert user.avatar == nil + + assert %{"url" => nil} = json_response(conn, 200) + end + + test "can set profile banner", %{conn: conn} do + user = insert(:user) + + conn = + conn + |> assign(:user, user) + |> patch("/api/v1/accounts/update_banner", %{"banner" => @image}) + + user = refresh_record(user) + assert user.info.banner["type"] == "Image" + + assert %{"url" => _} = json_response(conn, 200) + end + + test "can reset profile banner", %{conn: conn} do + user = insert(:user) + + conn = + conn + |> assign(:user, user) + |> patch("/api/v1/accounts/update_banner", %{"banner" => ""}) + + user = refresh_record(user) + assert user.info.banner == %{} + + assert %{"url" => nil} = json_response(conn, 200) + end + + test "background image can be set", %{conn: conn} do + user = insert(:user) + + conn = + conn + |> assign(:user, user) + |> patch("/api/v1/accounts/update_background", %{"img" => @image}) + + user = refresh_record(user) + assert user.info.background["type"] == "Image" + assert %{"url" => _} = json_response(conn, 200) + end + + test "background image can be reset", %{conn: conn} do + user = insert(:user) + + conn = + conn + |> assign(:user, user) + |> patch("/api/v1/accounts/update_background", %{"img" => ""}) + + user = refresh_record(user) + assert user.info.background == %{} + assert %{"url" => nil} = json_response(conn, 200) + end + test "creates an oauth app", %{conn: conn} do user = insert(:user) app_attrs = build(:oauth_app) From bdc1592377d19da67040a78faf360770c54d4e6a Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sat, 29 Jun 2019 14:25:47 +0300 Subject: [PATCH 03/43] Add PLEROMA_BUILD_ENV to override env for the version generator and use it in the CI for building docs This is needed because if we run with MIX_ENV=dev we will get a "+dev" suffix to the version in the docs, but if we run with MIX_ENV=prod the docs build will fail because ex_doc is dev-only dependency. --- .gitlab-ci.yml | 1 + mix.exs | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f8711f299..d0c540b16 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -35,6 +35,7 @@ docs-build: - develop@pleroma/pleroma variables: MIX_ENV: dev + PLEROMA_BUILD_ENV: prod script: - mix deps.get - mix compile diff --git a/mix.exs b/mix.exs index 19a80ffb2..c2618d2b2 100644 --- a/mix.exs +++ b/mix.exs @@ -207,6 +207,15 @@ defmodule Pleroma.Mixfile do env_name = if Mix.env() != :prod, do: to_string(Mix.env()) + env_override = System.get_env("PLEROMA_BUILD_ENV") + + env_name = + if env_override do + if env_override != "prod", do: env_override + else + env_name + end + build = [build_name, env_name] |> Enum.filter(fn string -> string && string != "" end) From 5b7b1040b38d262b1815276f86036b50847851c7 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sat, 29 Jun 2019 20:04:50 +0300 Subject: [PATCH 04/43] [#161] Limited replies depth on incoming federation in order to prevent memory leaks on recursive replies fetching. --- lib/pleroma/object.ex | 24 ++-- lib/pleroma/object/fetcher.ex | 8 +- .../web/activity_pub/transmogrifier.ex | 127 +++++++++++------- lib/pleroma/web/federator/federator.ex | 6 + .../web/ostatus/handlers/note_handler.ex | 13 +- lib/pleroma/web/ostatus/ostatus.ex | 22 +-- test/web/activity_pub/transmogrifier_test.exs | 37 ++++- test/web/ostatus/ostatus_test.exs | 28 +++- 8 files changed, 181 insertions(+), 84 deletions(-) diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 4b181ec59..b8647dd26 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -44,20 +44,20 @@ defmodule Pleroma.Object do Repo.one(from(object in Object, where: fragment("(?)->>'id' = ?", object.data, ^ap_id))) end - def normalize(_, fetch_remote \\ true) + def normalize(_, fetch_remote \\ true, options \\ []) # If we pass an Activity to Object.normalize(), we can try to use the preloaded object. # Use this whenever possible, especially when walking graphs in an O(N) loop! - def normalize(%Object{} = object, _), do: object - def normalize(%Activity{object: %Object{} = object}, _), do: object + def normalize(%Object{} = object, _, _), do: object + def normalize(%Activity{object: %Object{} = object}, _, _), do: object # A hack for fake activities - def normalize(%Activity{data: %{"object" => %{"fake" => true} = data}}, _) do + def normalize(%Activity{data: %{"object" => %{"fake" => true} = data}}, _, _) do %Object{id: "pleroma:fake_object_id", data: data} end # Catch and log Object.normalize() calls where the Activity's child object is not # preloaded. - def normalize(%Activity{data: %{"object" => %{"id" => ap_id}}}, fetch_remote) do + def normalize(%Activity{data: %{"object" => %{"id" => ap_id}}}, fetch_remote, _) do Logger.debug( "Object.normalize() called without preloaded object (#{ap_id}). Consider preloading the object!" ) @@ -67,7 +67,7 @@ defmodule Pleroma.Object do normalize(ap_id, fetch_remote) end - def normalize(%Activity{data: %{"object" => ap_id}}, fetch_remote) do + def normalize(%Activity{data: %{"object" => ap_id}}, fetch_remote, _) do Logger.debug( "Object.normalize() called without preloaded object (#{ap_id}). Consider preloading the object!" ) @@ -78,10 +78,14 @@ defmodule Pleroma.Object do end # Old way, try fetching the object through cache. - def normalize(%{"id" => ap_id}, fetch_remote), do: normalize(ap_id, fetch_remote) - def normalize(ap_id, false) when is_binary(ap_id), do: get_cached_by_ap_id(ap_id) - def normalize(ap_id, true) when is_binary(ap_id), do: Fetcher.fetch_object_from_id!(ap_id) - def normalize(_, _), do: nil + def normalize(%{"id" => ap_id}, fetch_remote, _), do: normalize(ap_id, fetch_remote) + def normalize(ap_id, false, _) when is_binary(ap_id), do: get_cached_by_ap_id(ap_id) + + def normalize(ap_id, true, options) when is_binary(ap_id) do + Fetcher.fetch_object_from_id!(ap_id, options) + end + + def normalize(_, _, _), do: nil # Owned objects can only be mutated by their owner def authorize_mutation(%Object{data: %{"actor" => actor}}, %User{ap_id: ap_id}), diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index c422490ac..fffbf2bbb 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -22,7 +22,7 @@ defmodule Pleroma.Object.Fetcher do # TODO: # This will create a Create activity, which we need internally at the moment. - def fetch_object_from_id(id) do + def fetch_object_from_id(id, options \\ []) do if object = Object.get_cached_by_ap_id(id) do {:ok, object} else @@ -38,7 +38,7 @@ defmodule Pleroma.Object.Fetcher do "object" => data }, :ok <- Containment.contain_origin(id, params), - {:ok, activity} <- Transmogrifier.handle_incoming(params), + {:ok, activity} <- Transmogrifier.handle_incoming(params, options), {:object, _data, %Object{} = object} <- {:object, data, Object.normalize(activity, false)} do {:ok, object} @@ -63,8 +63,8 @@ defmodule Pleroma.Object.Fetcher do end end - def fetch_object_from_id!(id) do - with {:ok, object} <- fetch_object_from_id(id) do + def fetch_object_from_id!(id, options \\ []) do + with {:ok, object} <- fetch_object_from_id(id, options) do object else _e -> diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 3bb8b40b5..d5ced2d1d 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -14,6 +14,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.ActivityPub.Visibility + alias Pleroma.Web.Federator import Ecto.Query @@ -22,20 +23,20 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do @doc """ Modifies an incoming AP object (mastodon format) to our internal format. """ - def fix_object(object) do + def fix_object(object, options \\ []) do object |> fix_actor |> fix_url |> fix_attachments |> fix_context - |> fix_in_reply_to + |> fix_in_reply_to(options) |> fix_emoji |> fix_tag |> fix_content_map |> fix_likes |> fix_addressing |> fix_summary - |> fix_type + |> fix_type(options) end def fix_summary(%{"summary" => nil} = object) do @@ -164,7 +165,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do object end - def fix_in_reply_to(%{"inReplyTo" => in_reply_to} = object) + def fix_in_reply_to(object, options \\ []) + + def fix_in_reply_to(%{"inReplyTo" => in_reply_to} = object, options) when not is_nil(in_reply_to) do in_reply_to_id = cond do @@ -182,28 +185,34 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do "" end - case get_obj_helper(in_reply_to_id) do - {:ok, replied_object} -> - with %Activity{} = _activity <- - Activity.get_create_by_object_ap_id(replied_object.data["id"]) do - object - |> Map.put("inReplyTo", replied_object.data["id"]) - |> Map.put("inReplyToAtomUri", object["inReplyToAtomUri"] || in_reply_to_id) - |> Map.put("conversation", replied_object.data["context"] || object["conversation"]) - |> Map.put("context", replied_object.data["context"] || object["conversation"]) - else - e -> - Logger.error("Couldn't fetch \"#{inspect(in_reply_to_id)}\", error: #{inspect(e)}") - object - end + object = Map.put(object, "inReplyToAtomUri", in_reply_to_id) - e -> - Logger.error("Couldn't fetch \"#{inspect(in_reply_to_id)}\", error: #{inspect(e)}") - object + if (options[:depth] || 1) <= Federator.max_replies_depth() do + case get_obj_helper(in_reply_to_id, options) do + {:ok, replied_object} -> + with %Activity{} = _activity <- + Activity.get_create_by_object_ap_id(replied_object.data["id"]) do + object + |> Map.put("inReplyTo", replied_object.data["id"]) + |> Map.put("inReplyToAtomUri", object["inReplyToAtomUri"] || in_reply_to_id) + |> Map.put("conversation", replied_object.data["context"] || object["conversation"]) + |> Map.put("context", replied_object.data["context"] || object["conversation"]) + else + e -> + Logger.error("Couldn't fetch \"#{inspect(in_reply_to_id)}\", error: #{inspect(e)}") + object + end + + e -> + Logger.error("Couldn't fetch \"#{inspect(in_reply_to_id)}\", error: #{inspect(e)}") + object + end + else + object end end - def fix_in_reply_to(object), do: object + def fix_in_reply_to(object, _options), do: object def fix_context(object) do context = object["context"] || object["conversation"] || Utils.generate_context_id() @@ -336,8 +345,15 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def fix_content_map(object), do: object - def fix_type(%{"inReplyTo" => reply_id} = object) when is_binary(reply_id) do - reply = Object.normalize(reply_id) + def fix_type(object, options \\ []) + + def fix_type(%{"inReplyTo" => reply_id} = object, options) when is_binary(reply_id) do + reply = + if (options[:depth] || 1) <= Federator.max_replies_depth() do + Object.normalize(reply_id, true) + else + nil + end if reply && (reply.data["type"] == "Question" and object["name"]) do Map.put(object, "type", "Answer") @@ -346,7 +362,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end end - def fix_type(object), do: object + def fix_type(object, _), do: object defp mastodon_follow_hack(%{"id" => id, "actor" => follower_id}, followed) do with true <- id =~ "follows", @@ -374,9 +390,11 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end end + def handle_incoming(data, options \\ []) + # Flag objects are placed ahead of the ID check because Mastodon 2.8 and earlier send them # with nil ID. - def handle_incoming(%{"type" => "Flag", "object" => objects, "actor" => actor} = data) do + def handle_incoming(%{"type" => "Flag", "object" => objects, "actor" => actor} = data, _options) do with context <- data["context"] || Utils.generate_context_id(), content <- data["content"] || "", %User{} = actor <- User.get_cached_by_ap_id(actor), @@ -409,15 +427,19 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end # disallow objects with bogus IDs - def handle_incoming(%{"id" => nil}), do: :error - def handle_incoming(%{"id" => ""}), do: :error + def handle_incoming(%{"id" => nil}, _options), do: :error + def handle_incoming(%{"id" => ""}, _options), do: :error # length of https:// = 8, should validate better, but good enough for now. - def handle_incoming(%{"id" => id}) when not (is_binary(id) and length(id) > 8), do: :error + def handle_incoming(%{"id" => id}, _options) when not (is_binary(id) and length(id) > 8), + do: :error # TODO: validate those with a Ecto scheme # - tags # - emoji - def handle_incoming(%{"type" => "Create", "object" => %{"type" => objtype} = object} = data) + def handle_incoming( + %{"type" => "Create", "object" => %{"type" => objtype} = object} = data, + options + ) when objtype in ["Article", "Note", "Video", "Page", "Question", "Answer"] do actor = Containment.get_actor(data) @@ -427,7 +449,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do with nil <- Activity.get_create_by_object_ap_id(object["id"]), {:ok, %User{} = user} <- User.get_or_fetch_by_ap_id(data["actor"]) do - object = fix_object(data["object"]) + options = Keyword.put(options, :depth, (options[:depth] || 0) + 1) + object = fix_object(data["object"], options) params = %{ to: data["to"], @@ -452,7 +475,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end def handle_incoming( - %{"type" => "Follow", "object" => followed, "actor" => follower, "id" => id} = data + %{"type" => "Follow", "object" => followed, "actor" => follower, "id" => id} = data, + _options ) do with %User{local: true} = followed <- User.get_cached_by_ap_id(followed), {:ok, %User{} = follower} <- User.get_or_fetch_by_ap_id(follower), @@ -503,7 +527,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end def handle_incoming( - %{"type" => "Accept", "object" => follow_object, "actor" => _actor, "id" => _id} = data + %{"type" => "Accept", "object" => follow_object, "actor" => _actor, "id" => _id} = data, + _options ) do with actor <- Containment.get_actor(data), {:ok, %User{} = followed} <- User.get_or_fetch_by_ap_id(actor), @@ -524,7 +549,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end def handle_incoming( - %{"type" => "Reject", "object" => follow_object, "actor" => _actor, "id" => _id} = data + %{"type" => "Reject", "object" => follow_object, "actor" => _actor, "id" => _id} = data, + _options ) do with actor <- Containment.get_actor(data), {:ok, %User{} = followed} <- User.get_or_fetch_by_ap_id(actor), @@ -548,7 +574,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end def handle_incoming( - %{"type" => "Like", "object" => object_id, "actor" => _actor, "id" => id} = data + %{"type" => "Like", "object" => object_id, "actor" => _actor, "id" => id} = data, + _options ) do with actor <- Containment.get_actor(data), {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor), @@ -561,7 +588,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end def handle_incoming( - %{"type" => "Announce", "object" => object_id, "actor" => _actor, "id" => id} = data + %{"type" => "Announce", "object" => object_id, "actor" => _actor, "id" => id} = data, + _options ) do with actor <- Containment.get_actor(data), {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor), @@ -576,7 +604,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def handle_incoming( %{"type" => "Update", "object" => %{"type" => object_type} = object, "actor" => actor_id} = - data + data, + _options ) when object_type in ["Person", "Application", "Service", "Organization"] do with %User{ap_id: ^actor_id} = actor <- User.get_cached_by_ap_id(object["id"]) do @@ -614,7 +643,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do # an error or a tombstone. This would allow us to verify that a deletion actually took # place. def handle_incoming( - %{"type" => "Delete", "object" => object_id, "actor" => _actor, "id" => _id} = data + %{"type" => "Delete", "object" => object_id, "actor" => _actor, "id" => _id} = data, + _options ) do object_id = Utils.get_ap_id(object_id) @@ -635,7 +665,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do "object" => %{"type" => "Announce", "object" => object_id}, "actor" => _actor, "id" => id - } = data + } = data, + _options ) do with actor <- Containment.get_actor(data), {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor), @@ -653,7 +684,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do "object" => %{"type" => "Follow", "object" => followed}, "actor" => follower, "id" => id - } = _data + } = _data, + _options ) do with %User{local: true} = followed <- User.get_cached_by_ap_id(followed), {:ok, %User{} = follower} <- User.get_or_fetch_by_ap_id(follower), @@ -671,7 +703,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do "object" => %{"type" => "Block", "object" => blocked}, "actor" => blocker, "id" => id - } = _data + } = _data, + _options ) do with true <- Pleroma.Config.get([:activitypub, :accept_blocks]), %User{local: true} = blocked <- User.get_cached_by_ap_id(blocked), @@ -685,7 +718,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end def handle_incoming( - %{"type" => "Block", "object" => blocked, "actor" => blocker, "id" => id} = _data + %{"type" => "Block", "object" => blocked, "actor" => blocker, "id" => id} = _data, + _options ) do with true <- Pleroma.Config.get([:activitypub, :accept_blocks]), %User{local: true} = blocked = User.get_cached_by_ap_id(blocked), @@ -705,7 +739,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do "object" => %{"type" => "Like", "object" => object_id}, "actor" => _actor, "id" => id - } = data + } = data, + _options ) do with actor <- Containment.get_actor(data), {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor), @@ -717,10 +752,10 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end end - def handle_incoming(_), do: :error + def handle_incoming(_, _), do: :error - def get_obj_helper(id) do - if object = Object.normalize(id), do: {:ok, object}, else: nil + def get_obj_helper(id, options \\ []) do + if object = Object.normalize(id, true, options), do: {:ok, object}, else: nil end def set_reply_to_uri(%{"inReplyTo" => in_reply_to} = object) when is_binary(in_reply_to) do diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index f4c9fe284..7c13ff323 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -22,6 +22,12 @@ defmodule Pleroma.Web.Federator do refresh_subscriptions() end + @max_replies_depth 100 + + @doc "Addresses [memory leaks on recursive replies fetching](https://git.pleroma.social/pleroma/pleroma/issues/161)" + # credo:disable-for-previous-line Credo.Check.Readability.MaxLineLength + def max_replies_depth, do: @max_replies_depth + # Client API def incoming_doc(doc) do diff --git a/lib/pleroma/web/ostatus/handlers/note_handler.ex b/lib/pleroma/web/ostatus/handlers/note_handler.ex index ec6e5cfaf..6f8f3ddcb 100644 --- a/lib/pleroma/web/ostatus/handlers/note_handler.ex +++ b/lib/pleroma/web/ostatus/handlers/note_handler.ex @@ -10,6 +10,7 @@ defmodule Pleroma.Web.OStatus.NoteHandler do alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.CommonAPI + alias Pleroma.Web.Federator alias Pleroma.Web.OStatus alias Pleroma.Web.XML @@ -88,14 +89,15 @@ defmodule Pleroma.Web.OStatus.NoteHandler do Map.put(note, "external_url", url) end - def fetch_replied_to_activity(entry, in_reply_to) do + def fetch_replied_to_activity(entry, in_reply_to, options \\ []) do with %Activity{} = activity <- Activity.get_create_by_object_ap_id(in_reply_to) do activity else _e -> - with in_reply_to_href when not is_nil(in_reply_to_href) <- + with true <- (options[:depth] || 1) <= Federator.max_replies_depth(), + in_reply_to_href when not is_nil(in_reply_to_href) <- XML.string_from_xpath("//thr:in-reply-to[1]/@href", entry), - {:ok, [activity | _]} <- OStatus.fetch_activity_from_url(in_reply_to_href) do + {:ok, [activity | _]} <- OStatus.fetch_activity_from_url(in_reply_to_href, options) do activity else _e -> nil @@ -104,7 +106,7 @@ defmodule Pleroma.Web.OStatus.NoteHandler do end # TODO: Clean this up a bit. - def handle_note(entry, doc \\ nil) do + def handle_note(entry, doc \\ nil, options \\ []) do with id <- XML.string_from_xpath("//id", entry), activity when is_nil(activity) <- Activity.get_create_by_object_ap_id_with_object(id), [author] <- :xmerl_xpath.string('//author[1]', doc), @@ -112,7 +114,8 @@ defmodule Pleroma.Web.OStatus.NoteHandler do content_html <- OStatus.get_content(entry), cw <- OStatus.get_cw(entry), in_reply_to <- XML.string_from_xpath("//thr:in-reply-to[1]/@ref", entry), - in_reply_to_activity <- fetch_replied_to_activity(entry, in_reply_to), + options <- Keyword.put(options, :depth, (options[:depth] || 0) + 1), + in_reply_to_activity <- fetch_replied_to_activity(entry, in_reply_to, options), in_reply_to_object <- (in_reply_to_activity && Object.normalize(in_reply_to_activity)) || nil, in_reply_to <- (in_reply_to_object && in_reply_to_object.data["id"]) || in_reply_to, diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 6ed089d84..502410c83 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -54,7 +54,7 @@ defmodule Pleroma.Web.OStatus do "#{Web.base_url()}/ostatus_subscribe?acct={uri}" end - def handle_incoming(xml_string) do + def handle_incoming(xml_string, options \\ []) do with doc when doc != :error <- parse_document(xml_string) do with {:ok, actor_user} <- find_make_or_update_user(doc), do: Pleroma.Instances.set_reachable(actor_user.ap_id) @@ -91,10 +91,12 @@ defmodule Pleroma.Web.OStatus do _ -> case object_type do 'http://activitystrea.ms/schema/1.0/note' -> - with {:ok, activity} <- NoteHandler.handle_note(entry, doc), do: activity + with {:ok, activity} <- NoteHandler.handle_note(entry, doc, options), + do: activity 'http://activitystrea.ms/schema/1.0/comment' -> - with {:ok, activity} <- NoteHandler.handle_note(entry, doc), do: activity + with {:ok, activity} <- NoteHandler.handle_note(entry, doc, options), + do: activity _ -> Logger.error("Couldn't parse incoming document") @@ -359,7 +361,7 @@ defmodule Pleroma.Web.OStatus do end end - def fetch_activity_from_atom_url(url) do + def fetch_activity_from_atom_url(url, options \\ []) do with true <- String.starts_with?(url, "http"), {:ok, %{body: body, status: code}} when code in 200..299 <- HTTP.get( @@ -367,7 +369,7 @@ defmodule Pleroma.Web.OStatus do [{:Accept, "application/atom+xml"}] ) do Logger.debug("Got document from #{url}, handling...") - handle_incoming(body) + handle_incoming(body, options) else e -> Logger.debug("Couldn't get #{url}: #{inspect(e)}") @@ -375,13 +377,13 @@ defmodule Pleroma.Web.OStatus do end end - def fetch_activity_from_html_url(url) do + def fetch_activity_from_html_url(url, options \\ []) do Logger.debug("Trying to fetch #{url}") with true <- String.starts_with?(url, "http"), {:ok, %{body: body}} <- HTTP.get(url, []), {:ok, atom_url} <- get_atom_url(body) do - fetch_activity_from_atom_url(atom_url) + fetch_activity_from_atom_url(atom_url, options) else e -> Logger.debug("Couldn't get #{url}: #{inspect(e)}") @@ -389,11 +391,11 @@ defmodule Pleroma.Web.OStatus do end end - def fetch_activity_from_url(url) do - with {:ok, [_ | _] = activities} <- fetch_activity_from_atom_url(url) do + def fetch_activity_from_url(url, options \\ []) do + with {:ok, [_ | _] = activities} <- fetch_activity_from_atom_url(url, options) do {:ok, activities} else - _e -> fetch_activity_from_html_url(url) + _e -> fetch_activity_from_html_url(url, options) end rescue e -> diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 68ec03c33..fc8703ca9 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -11,12 +11,13 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Transmogrifier + alias Pleroma.Web.CommonAPI alias Pleroma.Web.OStatus alias Pleroma.Web.Websub.WebsubClientSubscription + import Mock import Pleroma.Factory import ExUnit.CaptureLog - alias Pleroma.Web.CommonAPI setup_all do Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) @@ -46,12 +47,10 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do data["object"] |> Map.put("inReplyTo", "https://shitposter.club/notice/2827873") - data = - data - |> Map.put("object", object) - + data = Map.put(data, "object", object) {:ok, returned_activity} = Transmogrifier.handle_incoming(data) - returned_object = Object.normalize(returned_activity.data["object"]) + + returned_object = Object.normalize(returned_activity.data["object"], false) assert activity = Activity.get_create_by_object_ap_id( @@ -61,6 +60,32 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do assert returned_object.data["inReplyToAtomUri"] == "https://shitposter.club/notice/2827873" end + test "it does not fetch replied-to activities beyond max_replies_depth" do + data = + File.read!("test/fixtures/mastodon-post-activity.json") + |> Poison.decode!() + + object = + data["object"] + |> Map.put("inReplyTo", "https://shitposter.club/notice/2827873") + + data = Map.put(data, "object", object) + + with_mock Pleroma.Web.Federator, + max_replies_depth: fn -> 0 end do + {:ok, returned_activity} = Transmogrifier.handle_incoming(data) + + returned_object = Object.normalize(returned_activity.data["object"], false) + + refute Activity.get_create_by_object_ap_id( + "tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment" + ) + + assert returned_object.data["inReplyToAtomUri"] == + "https://shitposter.club/notice/2827873" + end + end + test "it does not crash if the object in inReplyTo can't be fetched" do data = File.read!("test/fixtures/mastodon-post-activity.json") diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index f6be16862..12627356c 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -11,8 +11,10 @@ defmodule Pleroma.Web.OStatusTest do alias Pleroma.User alias Pleroma.Web.OStatus alias Pleroma.Web.XML - import Pleroma.Factory + import ExUnit.CaptureLog + import Mock + import Pleroma.Factory setup_all do Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) @@ -266,10 +268,13 @@ defmodule Pleroma.Web.OStatusTest do assert favorited_activity.local end - test "handle incoming replies" do + test_with_mock "handle incoming replies, fetching replied-to activities if we don't have them", + OStatus, + [:passthrough], + [] do incoming = File.read!("test/fixtures/incoming_note_activity_answer.xml") {:ok, [activity]} = OStatus.handle_incoming(incoming) - object = Object.normalize(activity.data["object"]) + object = Object.normalize(activity.data["object"], false) assert activity.data["type"] == "Create" assert object.data["type"] == "Note" @@ -282,6 +287,23 @@ defmodule Pleroma.Web.OStatusTest do assert object.data["id"] == "tag:gs.example.org:4040,2017-04-25:noticeId=55:objectType=note" assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["to"] + + assert called(OStatus.fetch_activity_from_url(object.data["inReplyTo"], :_)) + end + + test_with_mock "handle incoming replies, not fetching replied-to activities beyond max_replies_depth", + OStatus, + [:passthrough], + [] do + incoming = File.read!("test/fixtures/incoming_note_activity_answer.xml") + + with_mock Pleroma.Web.Federator, + max_replies_depth: fn -> 0 end do + {:ok, [activity]} = OStatus.handle_incoming(incoming) + object = Object.normalize(activity.data["object"], false) + + refute called(OStatus.fetch_activity_from_url(object.data["inReplyTo"], :_)) + end end test "handle incoming follows" do From 044eb039c22f0cdececee85922b0c194f0944f42 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sun, 30 Jun 2019 13:06:10 +0300 Subject: [PATCH 05/43] OTP Release install docs: Remove --dry-run in cron certbot command --- docs/installation/otp_en.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/installation/otp_en.md b/docs/installation/otp_en.md index fa71f23e1..9b851e395 100644 --- a/docs/installation/otp_en.md +++ b/docs/installation/otp_en.md @@ -207,7 +207,7 @@ certbot renew --cert-name yourinstance.tld --webroot -w /var/lib/letsencrypt/ -- # Add it to the daily cron echo '#!/bin/sh -certbot renew --cert-name yourinstance.tld --webroot -w /var/lib/letsencrypt/ --dry-run --post-hook "systemctl reload nginx" +certbot renew --cert-name yourinstance.tld --webroot -w /var/lib/letsencrypt/ --post-hook "systemctl reload nginx" ' > /etc/cron.daily/renew-pleroma-cert chmod +x /etc/cron.daily/renew-pleroma-cert @@ -228,7 +228,7 @@ certbot renew --cert-name yourinstance.tld --webroot -w /var/lib/letsencrypt/ -- # Add it to the daily cron echo '#!/bin/sh -certbot renew --cert-name yourinstance.tld --webroot -w /var/lib/letsencrypt/ --dry-run --post-hook "rc-service nginx reload" +certbot renew --cert-name yourinstance.tld --webroot -w /var/lib/letsencrypt/ --post-hook "rc-service nginx reload" ' > /etc/periodic/daily/renew-pleroma-cert chmod +x /etc/periodic/daily/renew-pleroma-cert From 2b9d914089755297f6ac24ffbb81934cf3c70cdd Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sun, 30 Jun 2019 15:58:50 +0300 Subject: [PATCH 06/43] [#161] Refactoring, documentation. --- CHANGELOG.md | 1 + config/config.exs | 1 + docs/config.md | 1 + lib/pleroma/web/activity_pub/transmogrifier.ex | 6 ++---- lib/pleroma/web/federator/federator.ex | 12 +++++++++--- lib/pleroma/web/ostatus/handlers/note_handler.ex | 2 +- test/web/activity_pub/transmogrifier_test.exs | 2 +- test/web/ostatus/ostatus_test.exs | 2 +- 8 files changed, 17 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a6ec8674d..85d077e3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [Unreleased] ### Added - MRF: Support for priming the mediaproxy cache (`Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy`) +- Federation: Support for restricting max. reply-to depth on fetching ## [1.0.0] - 2019-06-29 ### Security diff --git a/config/config.exs b/config/config.exs index e337f00aa..65f239e31 100644 --- a/config/config.exs +++ b/config/config.exs @@ -218,6 +218,7 @@ config :pleroma, :instance, }, registrations_open: true, federating: true, + federation_incoming_replies_max_depth: 100, federation_reachability_timeout_days: 7, federation_publisher_modules: [ Pleroma.Web.ActivityPub.Publisher, diff --git a/docs/config.md b/docs/config.md index feef43ba9..b40147481 100644 --- a/docs/config.md +++ b/docs/config.md @@ -87,6 +87,7 @@ config :pleroma, Pleroma.Emails.Mailer, * `invites_enabled`: Enable user invitations for admins (depends on `registrations_open: false`). * `account_activation_required`: Require users to confirm their emails before signing in. * `federating`: Enable federation with other instances +* `federation_incoming_replies_max_depth`: Max. depth of reply-to activities fetching on incoming federation (to prevent memory leakage on extremely nested incoming threads). If set to `nil`, threads of any depth will be fetched. * `federation_reachability_timeout_days`: Timeout (in days) of each external federation target being unreachable prior to pausing federating to it. * `allow_relay`: Enable Pleroma’s Relay, which makes it possible to follow a whole instance * `rewrite_policy`: Message Rewrite Policy, either one or a list. Here are the ones available by default: diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index d5ced2d1d..543d4bb7d 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -187,7 +187,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do object = Map.put(object, "inReplyToAtomUri", in_reply_to_id) - if (options[:depth] || 1) <= Federator.max_replies_depth() do + if Federator.allowed_incoming_reply_depth?(options[:depth]) do case get_obj_helper(in_reply_to_id, options) do {:ok, replied_object} -> with %Activity{} = _activity <- @@ -349,10 +349,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def fix_type(%{"inReplyTo" => reply_id} = object, options) when is_binary(reply_id) do reply = - if (options[:depth] || 1) <= Federator.max_replies_depth() do + if Federator.allowed_incoming_reply_depth?(options[:depth]) do Object.normalize(reply_id, true) - else - nil end if reply && (reply.data["type"] == "Question" and object["name"]) do diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index 7c13ff323..f4f9e83e0 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -22,11 +22,17 @@ defmodule Pleroma.Web.Federator do refresh_subscriptions() end - @max_replies_depth 100 - @doc "Addresses [memory leaks on recursive replies fetching](https://git.pleroma.social/pleroma/pleroma/issues/161)" # credo:disable-for-previous-line Credo.Check.Readability.MaxLineLength - def max_replies_depth, do: @max_replies_depth + def allowed_incoming_reply_depth?(depth) do + max_replies_depth = Pleroma.Config.get([:instance, :federation_incoming_replies_max_depth]) + + if max_replies_depth do + (depth || 1) <= max_replies_depth + else + true + end + end # Client API diff --git a/lib/pleroma/web/ostatus/handlers/note_handler.ex b/lib/pleroma/web/ostatus/handlers/note_handler.ex index 6f8f3ddcb..8e0adad91 100644 --- a/lib/pleroma/web/ostatus/handlers/note_handler.ex +++ b/lib/pleroma/web/ostatus/handlers/note_handler.ex @@ -94,7 +94,7 @@ defmodule Pleroma.Web.OStatus.NoteHandler do activity else _e -> - with true <- (options[:depth] || 1) <= Federator.max_replies_depth(), + with true <- Federator.allowed_incoming_reply_depth?(options[:depth]), in_reply_to_href when not is_nil(in_reply_to_href) <- XML.string_from_xpath("//thr:in-reply-to[1]/@href", entry), {:ok, [activity | _]} <- OStatus.fetch_activity_from_url(in_reply_to_href, options) do diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index fc8703ca9..a914d3c4c 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -72,7 +72,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do data = Map.put(data, "object", object) with_mock Pleroma.Web.Federator, - max_replies_depth: fn -> 0 end do + allowed_incoming_reply_depth?: fn _ -> false end do {:ok, returned_activity} = Transmogrifier.handle_incoming(data) returned_object = Object.normalize(returned_activity.data["object"], false) diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 12627356c..acce33008 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -298,7 +298,7 @@ defmodule Pleroma.Web.OStatusTest do incoming = File.read!("test/fixtures/incoming_note_activity_answer.xml") with_mock Pleroma.Web.Federator, - max_replies_depth: fn -> 0 end do + allowed_incoming_reply_depth?: fn _ -> false end do {:ok, [activity]} = OStatus.handle_incoming(incoming) object = Object.normalize(activity.data["object"], false) From 611f6665ac8c9553ceeb3cb731f6a3f0482af829 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Mon, 1 Jul 2019 03:36:42 +0300 Subject: [PATCH 07/43] Update changelog and admin API docs with tags, display_name, avatar additions --- CHANGELOG.md | 2 ++ docs/api/admin_api.md | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 663d99ffd..ae7e4c0ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Admin API: Endpoints for managing reports - Admin API: Endpoints for deleting and changing the scope of individual reported statuses - Admin API: Endpoints to view and change config settings. +- Admin API: Return users' tags when querying reports +- Admin API: Return avatar and display name when querying users - AdminFE: initial release with basic user management accessible at /pleroma/admin/ - Mastodon API: Add chat token to `verify_credentials` response - Mastodon API: Add background image setting to `update_credentials` diff --git a/docs/api/admin_api.md b/docs/api/admin_api.md index 4be0ab0f8..74bde3ece 100644 --- a/docs/api/admin_api.md +++ b/docs/api/admin_api.md @@ -38,7 +38,9 @@ Authentication is required and the user must be an admin. "moderator": bool }, "local": bool, - "tags": array + "tags": array, + "avatar": string, + "display_name": string }, ... ] @@ -331,6 +333,7 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret "pleroma": {}, "sensitive": false }, + "tags": ["force_unlisted"], "statuses_count": 3, "url": "https://pleroma.example.org/users/user", "username": "user" @@ -366,6 +369,7 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret "pleroma": {}, "sensitive": false }, + "tags": ["force_unlisted"], "statuses_count": 1, "url": "https://pleroma.example.org/users/lain", "username": "lain" From 85c7916c94017c6a3657a3540f1e6c3afcb78225 Mon Sep 17 00:00:00 2001 From: Maksim Date: Mon, 1 Jul 2019 01:08:07 +0000 Subject: [PATCH 08/43] [#1033] Fix database migrations --- priv/repo/migrations/20170320193800_create_user.exs | 2 +- .../repo/migrations/20170321074828_create_activity.exs | 4 ++-- priv/repo/migrations/20170321074832_create_object.exs | 2 +- .../migrations/20170330153447_add_index_to_objects.exs | 2 +- ...15141210_add_unique_index_to_email_and_nickname.exs | 4 ++-- ...170418200143_create_webssub_server_subscription.exs | 2 +- ...0170426154155_create_websub_client_subscription.exs | 2 +- ...d_contraints_to_activities_and_objects_part_two.exs | 8 +++++--- .../20170502083023_add_local_field_to_activities.exs | 2 +- .../20170506222027_add_unique_index_to_apid.exs | 2 +- priv/repo/migrations/20170529093232_longer_bios.exs | 9 ++++++++- .../20170620095947_remove_activities_index.exs | 2 +- .../20170620133028_add_object_activity_index.exs | 2 +- ...170620142420_add_object_activity_index_part_two.exs | 2 +- .../20170701142005_add_actor_index_to_activity.exs | 2 +- .../migrations/20170906120646_add_mastodon_apps.exs | 2 +- .../20170906143140_create_o_auth_authorizations.exs | 2 +- .../migrations/20170906152508_create_o_auth_token.exs | 2 +- .../migrations/20170911123607_create_notifications.exs | 4 ++-- .../20171019141706_create_password_reset_tokens.exs | 2 +- .../20171109091239_add_actor_to_activity.exs | 2 +- .../20171130135819_add_local_index_to_user.exs | 2 +- .../20171212163643_add_recipients_to_activities.exs | 2 +- .../20171212164525_fill_recipients_in_activities.exs | 2 ++ .../20180221210540_make_following_postgres_array.exs | 4 +++- .../20180327174350_drop_local_index_on_activities.exs | 2 +- .../20180327175831_actually_drop_local_index.exs | 2 +- priv/repo/migrations/20180429094642_create_lists.exs | 4 ++-- .../20180516154905_create_user_trigram_index.exs | 2 +- .../20180530123448_add_list_follow_index.exs | 2 +- .../20180612110515_create_user_invite_tokens.exs | 2 +- priv/repo/migrations/20180813003722_create_filters.exs | 6 +++--- ...6_add_recipients_to_and_cc_fields_to_activities.exs | 4 ++-- .../20180829182612_activities_add_to_cc_indices.exs | 4 ++-- ...ove_recipients_to_and_cc_fields_from_activities.exs | 9 ++++++++- .../20180903114437_users_add_is_moderator_index.exs | 2 +- .../20180918182427_create_push_subscriptions.exs | 4 ++-- .../migrations/20181201104428_add_uuid_extension.exs | 4 +++- .../20181201105617_add_uui_ds_to_user_info.exs | 4 +++- .../migrations/20181206125616_add_tags_to_users.exs | 2 +- .../20181218172826_users_and_activities_flake_id.exs | 4 +++- .../20190109152453_add_visibility_function.exs | 4 ++-- .../20190115085500_create_user_fts_index.exs | 2 +- .../20190118074940_fix_user_trigram_index.exs | 4 ++-- .../20190123092341_users_add_is_admin_index.exs | 2 +- .../migrations/20190123125546_create_instances.exs | 6 +++--- priv/repo/migrations/20190123125839_fix_info_ids.exs | 4 +++- ...0190126160540_change_push_subscriptions_varchar.exs | 8 +++++++- .../migrations/20190204200237_add_correct_dm_index.exs | 2 +- .../migrations/20190205114625_create_thread_mutes.exs | 4 ++-- .../migrations/20190315101315_create_registrations.exs | 6 +++--- .../20190325185009_create_notification_id_index.exs | 2 +- .../20190328053912_create_scheduled_activities.exs | 6 +++--- .../20190403131720_add_oauth_token_indexes.exs | 6 +++--- .../20190405160700_add_index_on_subscribers.exs | 2 +- .../migrations/20190408123347_create_conversations.exs | 10 +++++----- ...190410152859_add_participation_updated_at_index.exs | 2 +- ...190411094120_add_index_on_user_info_deactivated.exs | 2 +- .../migrations/20190413082658_create_bookmarks.exs | 4 ++-- .../20190414125034_migrate_old_bookmarks.exs | 10 ++++++++-- .../20190501125843_add_fts_index_to_objects.exs | 2 +- ...20190501133552_add_refresh_token_index_to_token.exs | 2 +- ...190513175809_change_hide_column_in_filter_table.exs | 8 +++++++- .../20190515222404_add_thread_visibility_function.exs | 2 +- priv/repo/migrations/20190518032627_create_config.exs | 4 ++-- ...d_non_followers_fields_to_notification_settings.exs | 4 +++- .../20190603115238_add_index_on_activities_local.exs | 2 +- .../20190603173419_add_tag_index_to_objects.exs | 2 +- .../20190622151019_add_group_key_to_config.exs | 4 ++-- .../20190510135645_add_fts_index_to_objects_two.exs | 10 +++++----- 70 files changed, 151 insertions(+), 103 deletions(-) diff --git a/priv/repo/migrations/20170320193800_create_user.exs b/priv/repo/migrations/20170320193800_create_user.exs index 2d2f7fbf0..089964a26 100644 --- a/priv/repo/migrations/20170320193800_create_user.exs +++ b/priv/repo/migrations/20170320193800_create_user.exs @@ -2,7 +2,7 @@ defmodule Pleroma.Repo.Migrations.CreatePleroma.User do use Ecto.Migration def change do - create table(:users) do + create_if_not_exists table(:users) do add :email, :string add :password_hash, :string add :name, :string diff --git a/priv/repo/migrations/20170321074828_create_activity.exs b/priv/repo/migrations/20170321074828_create_activity.exs index 6e875ae43..f5c872721 100644 --- a/priv/repo/migrations/20170321074828_create_activity.exs +++ b/priv/repo/migrations/20170321074828_create_activity.exs @@ -2,13 +2,13 @@ defmodule Pleroma.Repo.Migrations.CreatePleroma.Activity do use Ecto.Migration def change do - create table(:activities) do + create_if_not_exists table(:activities) do add :data, :map timestamps() end - create index(:activities, [:data], using: :gin) + create_if_not_exists index(:activities, [:data], using: :gin) end end diff --git a/priv/repo/migrations/20170321074832_create_object.exs b/priv/repo/migrations/20170321074832_create_object.exs index b8bd49747..b184672ad 100644 --- a/priv/repo/migrations/20170321074832_create_object.exs +++ b/priv/repo/migrations/20170321074832_create_object.exs @@ -2,7 +2,7 @@ defmodule Pleroma.Repo.Migrations.CreatePleroma.Object do use Ecto.Migration def change do - create table(:objects) do + create_if_not_exists table(:objects) do add :data, :map timestamps() diff --git a/priv/repo/migrations/20170330153447_add_index_to_objects.exs b/priv/repo/migrations/20170330153447_add_index_to_objects.exs index 30ed61f51..25e308533 100644 --- a/priv/repo/migrations/20170330153447_add_index_to_objects.exs +++ b/priv/repo/migrations/20170330153447_add_index_to_objects.exs @@ -2,6 +2,6 @@ defmodule Pleroma.Repo.Migrations.AddIndexToObjects do use Ecto.Migration def change do - create index(:objects, [:data], using: :gin) + create_if_not_exists index(:objects, [:data], using: :gin) end end diff --git a/priv/repo/migrations/20170415141210_add_unique_index_to_email_and_nickname.exs b/priv/repo/migrations/20170415141210_add_unique_index_to_email_and_nickname.exs index 361ca04da..42da88954 100644 --- a/priv/repo/migrations/20170415141210_add_unique_index_to_email_and_nickname.exs +++ b/priv/repo/migrations/20170415141210_add_unique_index_to_email_and_nickname.exs @@ -2,7 +2,7 @@ defmodule Pleroma.Repo.Migrations.AddUniqueIndexToEmailAndNickname do use Ecto.Migration def change do - create unique_index(:users, [:email]) - create unique_index(:users, [:nickname]) + create_if_not_exists unique_index(:users, [:email]) + create_if_not_exists unique_index(:users, [:nickname]) end end diff --git a/priv/repo/migrations/20170418200143_create_webssub_server_subscription.exs b/priv/repo/migrations/20170418200143_create_webssub_server_subscription.exs index fe2fa2304..243280378 100644 --- a/priv/repo/migrations/20170418200143_create_webssub_server_subscription.exs +++ b/priv/repo/migrations/20170418200143_create_webssub_server_subscription.exs @@ -2,7 +2,7 @@ defmodule Pleroma.Repo.Migrations.CreateWebsubServerSubscription do use Ecto.Migration def change do - create table(:websub_server_subscriptions) do + create_if_not_exists table(:websub_server_subscriptions) do add :topic, :string add :callback, :string add :secret, :string diff --git a/priv/repo/migrations/20170426154155_create_websub_client_subscription.exs b/priv/repo/migrations/20170426154155_create_websub_client_subscription.exs index 89d3af7ae..4b79d7506 100644 --- a/priv/repo/migrations/20170426154155_create_websub_client_subscription.exs +++ b/priv/repo/migrations/20170426154155_create_websub_client_subscription.exs @@ -2,7 +2,7 @@ defmodule Pleroma.Repo.Migrations.CreateWebsubClientSubscription do use Ecto.Migration def change do - create table(:websub_client_subscriptions) do + create_if_not_exists table(:websub_client_subscriptions) do add :topic, :string add :secret, :string add :valid_until, :naive_datetime_usec diff --git a/priv/repo/migrations/20170501133231_add_id_contraints_to_activities_and_objects_part_two.exs b/priv/repo/migrations/20170501133231_add_id_contraints_to_activities_and_objects_part_two.exs index 1472b60b4..f5e5cd269 100644 --- a/priv/repo/migrations/20170501133231_add_id_contraints_to_activities_and_objects_part_two.exs +++ b/priv/repo/migrations/20170501133231_add_id_contraints_to_activities_and_objects_part_two.exs @@ -1,10 +1,12 @@ defmodule Pleroma.Repo.Migrations.AddIdContraintsToActivitiesAndObjectsPartTwo do use Ecto.Migration - def change do + def up do drop_if_exists index(:objects, ["(data->>\"id\")"], name: :objects_unique_apid_index) drop_if_exists index(:activities, ["(data->>\"id\")"], name: :activities_unique_apid_index) - create unique_index(:objects, ["(data->>'id')"], name: :objects_unique_apid_index) - create unique_index(:activities, ["(data->>'id')"], name: :activities_unique_apid_index) + create_if_not_exists unique_index(:objects, ["(data->>'id')"], name: :objects_unique_apid_index) + create_if_not_exists unique_index(:activities, ["(data->>'id')"], name: :activities_unique_apid_index) end + + def down, do: :ok end diff --git a/priv/repo/migrations/20170502083023_add_local_field_to_activities.exs b/priv/repo/migrations/20170502083023_add_local_field_to_activities.exs index 088d68f67..cebc11d21 100644 --- a/priv/repo/migrations/20170502083023_add_local_field_to_activities.exs +++ b/priv/repo/migrations/20170502083023_add_local_field_to_activities.exs @@ -6,6 +6,6 @@ defmodule Pleroma.Repo.Migrations.AddLocalFieldToActivities do add :local, :boolean, default: true end - create index(:activities, [:local]) + create_if_not_exists index(:activities, [:local]) end end diff --git a/priv/repo/migrations/20170506222027_add_unique_index_to_apid.exs b/priv/repo/migrations/20170506222027_add_unique_index_to_apid.exs index 864b5e47d..1b7e33b70 100644 --- a/priv/repo/migrations/20170506222027_add_unique_index_to_apid.exs +++ b/priv/repo/migrations/20170506222027_add_unique_index_to_apid.exs @@ -2,6 +2,6 @@ defmodule Pleroma.Repo.Migrations.AddUniqueIndexToAPID do use Ecto.Migration def change do - create unique_index(:users, [:ap_id]) + create_if_not_exists unique_index(:users, [:ap_id]) end end diff --git a/priv/repo/migrations/20170529093232_longer_bios.exs b/priv/repo/migrations/20170529093232_longer_bios.exs index f8c14b117..9188f4bee 100644 --- a/priv/repo/migrations/20170529093232_longer_bios.exs +++ b/priv/repo/migrations/20170529093232_longer_bios.exs @@ -1,9 +1,16 @@ defmodule Pleroma.Repo.Migrations.LongerBios do use Ecto.Migration - def change do + def up do alter table(:users) do modify :bio, :text end end + + def down do + alter table(:users) do + modify :bio, :string + end + end + end diff --git a/priv/repo/migrations/20170620095947_remove_activities_index.exs b/priv/repo/migrations/20170620095947_remove_activities_index.exs index 85feabeeb..e7d41eac4 100644 --- a/priv/repo/migrations/20170620095947_remove_activities_index.exs +++ b/priv/repo/migrations/20170620095947_remove_activities_index.exs @@ -2,6 +2,6 @@ defmodule Pleroma.Repo.Migrations.RemoveActivitiesIndex do use Ecto.Migration def change do - drop index(:activities, [:data]) + drop_if_exists index(:activities, [:data]) end end diff --git a/priv/repo/migrations/20170620133028_add_object_activity_index.exs b/priv/repo/migrations/20170620133028_add_object_activity_index.exs index df0b8391e..5c312b8f4 100644 --- a/priv/repo/migrations/20170620133028_add_object_activity_index.exs +++ b/priv/repo/migrations/20170620133028_add_object_activity_index.exs @@ -3,6 +3,6 @@ defmodule Pleroma.Repo.Migrations.AddObjectActivityIndex do def change do # This was wrong, now a noop - # create index(:objects, ["(data->'object'->>'id')", "(data->>'type')"], name: :activities_create_objects_index) + # create_if_not_exists index(:objects, ["(data->'object'->>'id')", "(data->>'type')"], name: :activities_create_objects_index) end end diff --git a/priv/repo/migrations/20170620142420_add_object_activity_index_part_two.exs b/priv/repo/migrations/20170620142420_add_object_activity_index_part_two.exs index 8d5a95c45..c95218fad 100644 --- a/priv/repo/migrations/20170620142420_add_object_activity_index_part_two.exs +++ b/priv/repo/migrations/20170620142420_add_object_activity_index_part_two.exs @@ -3,6 +3,6 @@ defmodule Pleroma.Repo.Migrations.AddObjectActivityIndexPartTwo do def change do drop_if_exists index(:objects, ["(data->'object'->>'id')", "(data->>'type')"], name: :activities_create_objects_index) - create index(:activities, ["(data->'object'->>'id')", "(data->>'type')"], name: :activities_create_objects_index) + create_if_not_exists index(:activities, ["(data->'object'->>'id')", "(data->>'type')"], name: :activities_create_objects_index) end end diff --git a/priv/repo/migrations/20170701142005_add_actor_index_to_activity.exs b/priv/repo/migrations/20170701142005_add_actor_index_to_activity.exs index 82c64396f..807fe3728 100644 --- a/priv/repo/migrations/20170701142005_add_actor_index_to_activity.exs +++ b/priv/repo/migrations/20170701142005_add_actor_index_to_activity.exs @@ -2,6 +2,6 @@ defmodule Pleroma.Repo.Migrations.AddActorIndexToActivity do use Ecto.Migration def change do - create index(:activities, ["(data->>'actor')", "inserted_at desc"], name: :activities_actor_index) + create_if_not_exists index(:activities, ["(data->>'actor')", "inserted_at desc"], name: :activities_actor_index) end end diff --git a/priv/repo/migrations/20170906120646_add_mastodon_apps.exs b/priv/repo/migrations/20170906120646_add_mastodon_apps.exs index d3dd317dd..ccd5e3fe2 100644 --- a/priv/repo/migrations/20170906120646_add_mastodon_apps.exs +++ b/priv/repo/migrations/20170906120646_add_mastodon_apps.exs @@ -2,7 +2,7 @@ defmodule Pleroma.Repo.Migrations.AddMastodonApps do use Ecto.Migration def change do - create table(:apps) do + create_if_not_exists table(:apps) do add :client_name, :string add :redirect_uris, :string add :scopes, :string diff --git a/priv/repo/migrations/20170906143140_create_o_auth_authorizations.exs b/priv/repo/migrations/20170906143140_create_o_auth_authorizations.exs index ead1d023e..63b25c537 100644 --- a/priv/repo/migrations/20170906143140_create_o_auth_authorizations.exs +++ b/priv/repo/migrations/20170906143140_create_o_auth_authorizations.exs @@ -2,7 +2,7 @@ defmodule Pleroma.Repo.Migrations.CreateOAuthAuthorizations do use Ecto.Migration def change do - create table(:oauth_authorizations) do + create_if_not_exists table(:oauth_authorizations) do add :app_id, references(:apps) add :user_id, references(:users) add :token, :string diff --git a/priv/repo/migrations/20170906152508_create_o_auth_token.exs b/priv/repo/migrations/20170906152508_create_o_auth_token.exs index ed56bbf36..08471bbf8 100644 --- a/priv/repo/migrations/20170906152508_create_o_auth_token.exs +++ b/priv/repo/migrations/20170906152508_create_o_auth_token.exs @@ -2,7 +2,7 @@ defmodule Pleroma.Repo.Migrations.CreateOAuthToken do use Ecto.Migration def change do - create table(:oauth_tokens) do + create_if_not_exists table(:oauth_tokens) do add :app_id, references(:apps) add :user_id, references(:users) add :token, :string diff --git a/priv/repo/migrations/20170911123607_create_notifications.exs b/priv/repo/migrations/20170911123607_create_notifications.exs index 5be809fb8..50de9c5f1 100644 --- a/priv/repo/migrations/20170911123607_create_notifications.exs +++ b/priv/repo/migrations/20170911123607_create_notifications.exs @@ -2,7 +2,7 @@ defmodule Pleroma.Repo.Migrations.CreateNotifications do use Ecto.Migration def change do - create table(:notifications) do + create_if_not_exists table(:notifications) do add :user_id, references(:users, on_delete: :delete_all) add :activity_id, references(:activities, on_delete: :delete_all) add :seen, :boolean, default: false @@ -10,6 +10,6 @@ defmodule Pleroma.Repo.Migrations.CreateNotifications do timestamps() end - create index(:notifications, [:user_id]) + create_if_not_exists index(:notifications, [:user_id]) end end diff --git a/priv/repo/migrations/20171019141706_create_password_reset_tokens.exs b/priv/repo/migrations/20171019141706_create_password_reset_tokens.exs index 2d9be3aab..dde0f945f 100644 --- a/priv/repo/migrations/20171019141706_create_password_reset_tokens.exs +++ b/priv/repo/migrations/20171019141706_create_password_reset_tokens.exs @@ -2,7 +2,7 @@ defmodule Pleroma.Repo.Migrations.CreatePasswordResetTokens do use Ecto.Migration def change do - create table(:password_reset_tokens) do + create_if_not_exists table(:password_reset_tokens) do add :token, :string add :user_id, references(:users) add :used, :boolean, default: false diff --git a/priv/repo/migrations/20171109091239_add_actor_to_activity.exs b/priv/repo/migrations/20171109091239_add_actor_to_activity.exs index 2d8b60a91..fb5f80c98 100644 --- a/priv/repo/migrations/20171109091239_add_actor_to_activity.exs +++ b/priv/repo/migrations/20171109091239_add_actor_to_activity.exs @@ -12,7 +12,7 @@ defmodule Pleroma.Repo.Migrations.AddActorToActivity do end def down do - drop index(:activities, [:actor, "id DESC NULLS LAST"]) + drop_if_exists index(:activities, [:actor, "id DESC NULLS LAST"]) alter table(:activities) do remove :actor end diff --git a/priv/repo/migrations/20171130135819_add_local_index_to_user.exs b/priv/repo/migrations/20171130135819_add_local_index_to_user.exs index 25716be21..3438bbbc4 100644 --- a/priv/repo/migrations/20171130135819_add_local_index_to_user.exs +++ b/priv/repo/migrations/20171130135819_add_local_index_to_user.exs @@ -2,6 +2,6 @@ defmodule Pleroma.Repo.Migrations.AddLocalIndexToUser do use Ecto.Migration def change do - create index(:users, [:local]) + create_if_not_exists index(:users, [:local]) end end diff --git a/priv/repo/migrations/20171212163643_add_recipients_to_activities.exs b/priv/repo/migrations/20171212163643_add_recipients_to_activities.exs index 7bce78108..4520b398e 100644 --- a/priv/repo/migrations/20171212163643_add_recipients_to_activities.exs +++ b/priv/repo/migrations/20171212163643_add_recipients_to_activities.exs @@ -6,6 +6,6 @@ defmodule Pleroma.Repo.Migrations.AddRecipientsToActivities do add :recipients, {:array, :string} end - create index(:activities, [:recipients], using: :gin) + create_if_not_exists index(:activities, [:recipients], using: :gin) end end diff --git a/priv/repo/migrations/20171212164525_fill_recipients_in_activities.exs b/priv/repo/migrations/20171212164525_fill_recipients_in_activities.exs index 1fcc0dabb..87de64ca5 100644 --- a/priv/repo/migrations/20171212164525_fill_recipients_in_activities.exs +++ b/priv/repo/migrations/20171212164525_fill_recipients_in_activities.exs @@ -18,4 +18,6 @@ defmodule Pleroma.Repo.Migrations.FillRecipientsInActivities do end) end end + + def down, do: :ok end diff --git a/priv/repo/migrations/20180221210540_make_following_postgres_array.exs b/priv/repo/migrations/20180221210540_make_following_postgres_array.exs index 98ca7d9d7..5a8f8f669 100644 --- a/priv/repo/migrations/20180221210540_make_following_postgres_array.exs +++ b/priv/repo/migrations/20180221210540_make_following_postgres_array.exs @@ -1,7 +1,7 @@ defmodule Pleroma.Repo.Migrations.MakeFollowingPostgresArray do use Ecto.Migration - def change do + def up do alter table(:users) do add :following_temp, {:array, :string} end @@ -15,4 +15,6 @@ defmodule Pleroma.Repo.Migrations.MakeFollowingPostgresArray do end rename table(:users), :following_temp, to: :following end + + def down, do: :ok end diff --git a/priv/repo/migrations/20180327174350_drop_local_index_on_activities.exs b/priv/repo/migrations/20180327174350_drop_local_index_on_activities.exs index 70f2c9fe6..35c4ce62f 100644 --- a/priv/repo/migrations/20180327174350_drop_local_index_on_activities.exs +++ b/priv/repo/migrations/20180327174350_drop_local_index_on_activities.exs @@ -2,6 +2,6 @@ defmodule Pleroma.Repo.Migrations.DropLocalIndexOnActivities do use Ecto.Migration def change do - drop index(:users, [:local]) + drop_if_exists index(:users, [:local]) end end diff --git a/priv/repo/migrations/20180327175831_actually_drop_local_index.exs b/priv/repo/migrations/20180327175831_actually_drop_local_index.exs index 2da65689c..7556336ed 100644 --- a/priv/repo/migrations/20180327175831_actually_drop_local_index.exs +++ b/priv/repo/migrations/20180327175831_actually_drop_local_index.exs @@ -2,7 +2,7 @@ defmodule Pleroma.Repo.Migrations.ActuallyDropLocalIndex do use Ecto.Migration def change do - create index(:users, [:local]) + create_if_not_exists index(:users, [:local]) drop_if_exists index("activities", :local) end end diff --git a/priv/repo/migrations/20180429094642_create_lists.exs b/priv/repo/migrations/20180429094642_create_lists.exs index 64c62250e..9d3ce50b3 100644 --- a/priv/repo/migrations/20180429094642_create_lists.exs +++ b/priv/repo/migrations/20180429094642_create_lists.exs @@ -2,7 +2,7 @@ defmodule Pleroma.Repo.Migrations.CreateLists do use Ecto.Migration def change do - create table(:lists) do + create_if_not_exists table(:lists) do add :user_id, references(:users, on_delete: :delete_all) add :title, :string add :following, {:array, :string} @@ -10,6 +10,6 @@ defmodule Pleroma.Repo.Migrations.CreateLists do timestamps() end - create index(:lists, [:user_id]) + create_if_not_exists index(:lists, [:user_id]) end end diff --git a/priv/repo/migrations/20180516154905_create_user_trigram_index.exs b/priv/repo/migrations/20180516154905_create_user_trigram_index.exs index abfa4b3cc..58622a87e 100644 --- a/priv/repo/migrations/20180516154905_create_user_trigram_index.exs +++ b/priv/repo/migrations/20180516154905_create_user_trigram_index.exs @@ -2,6 +2,6 @@ defmodule Pleroma.Repo.Migrations.CreateUserTrigramIndex do use Ecto.Migration def change do - create index(:users, ["(nickname || name) gist_trgm_ops"], name: :users_trigram_index, using: :gist) + create_if_not_exists index(:users, ["(nickname || name) gist_trgm_ops"], name: :users_trigram_index, using: :gist) end end diff --git a/priv/repo/migrations/20180530123448_add_list_follow_index.exs b/priv/repo/migrations/20180530123448_add_list_follow_index.exs index d6603e916..86b8de30a 100644 --- a/priv/repo/migrations/20180530123448_add_list_follow_index.exs +++ b/priv/repo/migrations/20180530123448_add_list_follow_index.exs @@ -2,6 +2,6 @@ defmodule Pleroma.Repo.Migrations.AddListFollowIndex do use Ecto.Migration def change do - create index(:lists, [:following]) + create_if_not_exists index(:lists, [:following]) end end diff --git a/priv/repo/migrations/20180612110515_create_user_invite_tokens.exs b/priv/repo/migrations/20180612110515_create_user_invite_tokens.exs index d0a1cf784..faee379f0 100644 --- a/priv/repo/migrations/20180612110515_create_user_invite_tokens.exs +++ b/priv/repo/migrations/20180612110515_create_user_invite_tokens.exs @@ -2,7 +2,7 @@ defmodule Pleroma.Repo.Migrations.CreateUserInviteTokens do use Ecto.Migration def change do - create table(:user_invite_tokens) do + create_if_not_exists table(:user_invite_tokens) do add :token, :string add :used, :boolean, default: false diff --git a/priv/repo/migrations/20180813003722_create_filters.exs b/priv/repo/migrations/20180813003722_create_filters.exs index 8e7129f34..541cf46a1 100644 --- a/priv/repo/migrations/20180813003722_create_filters.exs +++ b/priv/repo/migrations/20180813003722_create_filters.exs @@ -2,7 +2,7 @@ defmodule Pleroma.Repo.Migrations.CreateFilters do use Ecto.Migration def change do - create table(:filters) do + create_if_not_exists table(:filters) do add :user_id, references(:users, on_delete: :delete_all) add :filter_id, :integer add :hide, :boolean @@ -14,7 +14,7 @@ defmodule Pleroma.Repo.Migrations.CreateFilters do timestamps() end - create index(:filters, [:user_id]) - create index(:filters, [:phrase], where: "hide = true", name: :hided_phrases_index) + create_if_not_exists index(:filters, [:user_id]) + create_if_not_exists index(:filters, [:phrase], where: "hide = true", name: :hided_phrases_index) end end diff --git a/priv/repo/migrations/20180829082446_add_recipients_to_and_cc_fields_to_activities.exs b/priv/repo/migrations/20180829082446_add_recipients_to_and_cc_fields_to_activities.exs index 96af412f0..af9d521c0 100644 --- a/priv/repo/migrations/20180829082446_add_recipients_to_and_cc_fields_to_activities.exs +++ b/priv/repo/migrations/20180829082446_add_recipients_to_and_cc_fields_to_activities.exs @@ -7,7 +7,7 @@ defmodule Pleroma.Repo.Migrations.AddRecipientsToAndCcFieldsToActivities do add :recipients_cc, {:array, :string} end - create index(:activities, [:recipients_to], using: :gin) - create index(:activities, [:recipients_cc], using: :gin) + create_if_not_exists index(:activities, [:recipients_to], using: :gin) + create_if_not_exists index(:activities, [:recipients_cc], using: :gin) end end diff --git a/priv/repo/migrations/20180829182612_activities_add_to_cc_indices.exs b/priv/repo/migrations/20180829182612_activities_add_to_cc_indices.exs index f6c622e3e..9d31f6779 100644 --- a/priv/repo/migrations/20180829182612_activities_add_to_cc_indices.exs +++ b/priv/repo/migrations/20180829182612_activities_add_to_cc_indices.exs @@ -2,7 +2,7 @@ defmodule Pleroma.Repo.Migrations.ActivitiesAddToCcIndices do use Ecto.Migration def change do - create index(:activities, ["(data->'to')"], name: :activities_to_index, using: :gin) - create index(:activities, ["(data->'cc')"], name: :activities_cc_index, using: :gin) + create_if_not_exists index(:activities, ["(data->'to')"], name: :activities_to_index, using: :gin) + create_if_not_exists index(:activities, ["(data->'cc')"], name: :activities_cc_index, using: :gin) end end diff --git a/priv/repo/migrations/20180829183529_remove_recipients_to_and_cc_fields_from_activities.exs b/priv/repo/migrations/20180829183529_remove_recipients_to_and_cc_fields_from_activities.exs index ed4f5af30..017ef161f 100644 --- a/priv/repo/migrations/20180829183529_remove_recipients_to_and_cc_fields_from_activities.exs +++ b/priv/repo/migrations/20180829183529_remove_recipients_to_and_cc_fields_from_activities.exs @@ -1,10 +1,17 @@ defmodule Pleroma.Repo.Migrations.RemoveRecipientsToAndCcFieldsFromActivities do use Ecto.Migration - def change do + def up do alter table(:activities) do remove :recipients_to remove :recipients_cc end end + + def down do + alter table(:activities) do + add :recipients_to, {:array, :string} + add :recipients_cc, {:array, :string} + end + end end diff --git a/priv/repo/migrations/20180903114437_users_add_is_moderator_index.exs b/priv/repo/migrations/20180903114437_users_add_is_moderator_index.exs index ba6b90ea9..adce28bdf 100644 --- a/priv/repo/migrations/20180903114437_users_add_is_moderator_index.exs +++ b/priv/repo/migrations/20180903114437_users_add_is_moderator_index.exs @@ -2,6 +2,6 @@ defmodule Pleroma.Repo.Migrations.UsersAddIsModeratorIndex do use Ecto.Migration def change do - create index(:users, ["(info->'is_moderator')"], name: :users_is_moderator_index, using: :gin) + create_if_not_exists index(:users, ["(info->'is_moderator')"], name: :users_is_moderator_index, using: :gin) end end diff --git a/priv/repo/migrations/20180918182427_create_push_subscriptions.exs b/priv/repo/migrations/20180918182427_create_push_subscriptions.exs index 0cc7afa54..36bdf322a 100644 --- a/priv/repo/migrations/20180918182427_create_push_subscriptions.exs +++ b/priv/repo/migrations/20180918182427_create_push_subscriptions.exs @@ -2,7 +2,7 @@ defmodule Pleroma.Repo.Migrations.CreatePushSubscriptions do use Ecto.Migration def change do - create table("push_subscriptions") do + create_if_not_exists table("push_subscriptions") do add :user_id, references("users", on_delete: :delete_all) add :token_id, references("oauth_tokens", on_delete: :delete_all) add :endpoint, :string @@ -13,6 +13,6 @@ defmodule Pleroma.Repo.Migrations.CreatePushSubscriptions do timestamps() end - create index("push_subscriptions", [:user_id, :token_id], unique: true) + create_if_not_exists index("push_subscriptions", [:user_id, :token_id], unique: true) end end diff --git a/priv/repo/migrations/20181201104428_add_uuid_extension.exs b/priv/repo/migrations/20181201104428_add_uuid_extension.exs index 2509e558d..99fcb957c 100644 --- a/priv/repo/migrations/20181201104428_add_uuid_extension.exs +++ b/priv/repo/migrations/20181201104428_add_uuid_extension.exs @@ -1,7 +1,9 @@ defmodule Pleroma.Repo.Migrations.AddUUIDExtension do use Ecto.Migration - def change do + def up do execute("create extension if not exists \"uuid-ossp\"") end + + def down, do: :ok end diff --git a/priv/repo/migrations/20181201105617_add_uui_ds_to_user_info.exs b/priv/repo/migrations/20181201105617_add_uui_ds_to_user_info.exs index 9571a1e4d..964383668 100644 --- a/priv/repo/migrations/20181201105617_add_uui_ds_to_user_info.exs +++ b/priv/repo/migrations/20181201105617_add_uui_ds_to_user_info.exs @@ -1,7 +1,9 @@ defmodule Pleroma.Repo.Migrations.AddUUIDsToUserInfo do use Ecto.Migration - def change do + def up do execute("update users set info = jsonb_set(info, '{\"id\"}', to_jsonb(uuid_generate_v4()))") end + + def down, do: :ok end diff --git a/priv/repo/migrations/20181206125616_add_tags_to_users.exs b/priv/repo/migrations/20181206125616_add_tags_to_users.exs index 1502f63b6..7d42a0fba 100644 --- a/priv/repo/migrations/20181206125616_add_tags_to_users.exs +++ b/priv/repo/migrations/20181206125616_add_tags_to_users.exs @@ -6,6 +6,6 @@ defmodule Pleroma.Repo.Migrations.AddTagsToUsers do add :tags, {:array, :string} end - create index(:users, [:tags], using: :gin) + create_if_not_exists index(:users, [:tags], using: :gin) end end diff --git a/priv/repo/migrations/20181218172826_users_and_activities_flake_id.exs b/priv/repo/migrations/20181218172826_users_and_activities_flake_id.exs index 47d2d02da..a5b4c543d 100644 --- a/priv/repo/migrations/20181218172826_users_and_activities_flake_id.exs +++ b/priv/repo/migrations/20181218172826_users_and_activities_flake_id.exs @@ -12,7 +12,7 @@ defmodule Pleroma.Repo.Migrations.UsersAndActivitiesFlakeId do # 4- update relation pkeys with the new ids # 5- rename the temporary column to id # 6- re-create the constraints - def change do + def up do # Old serial int ids are transformed to 128bits with extra padding. # The application (in `Pleroma.FlakeId`) handles theses IDs properly as integers; to keep compatibility # with previously issued ids. @@ -75,6 +75,8 @@ defmodule Pleroma.Repo.Migrations.UsersAndActivitiesFlakeId do stop_clippy_heartbeats(clippy) end + def down, do: :ok + defp start_clippy_heartbeats() do count = from(a in "activities", select: count(a.id)) |> Repo.one! diff --git a/priv/repo/migrations/20190109152453_add_visibility_function.exs b/priv/repo/migrations/20190109152453_add_visibility_function.exs index 3aadabcd7..b6a4e752b 100644 --- a/priv/repo/migrations/20190109152453_add_visibility_function.exs +++ b/priv/repo/migrations/20190109152453_add_visibility_function.exs @@ -37,12 +37,12 @@ defmodule Pleroma.Repo.Migrations.AddVisibilityFunction do end def down do - drop( + drop_if_exists( index(:activities, ["activity_visibility(actor, recipients, data)"], name: :activities_visibility_index ) ) - execute("drop function activity_visibility(actor varchar, recipients varchar[], data jsonb)") + execute("drop function if exists activity_visibility(actor varchar, recipients varchar[], data jsonb)") end end diff --git a/priv/repo/migrations/20190115085500_create_user_fts_index.exs b/priv/repo/migrations/20190115085500_create_user_fts_index.exs index 499d67113..cff975318 100644 --- a/priv/repo/migrations/20190115085500_create_user_fts_index.exs +++ b/priv/repo/migrations/20190115085500_create_user_fts_index.exs @@ -2,7 +2,7 @@ defmodule Pleroma.Repo.Migrations.CreateUserFtsIndex do use Ecto.Migration def change do - create index( + create_if_not_exists index( :users, [ """ diff --git a/priv/repo/migrations/20190118074940_fix_user_trigram_index.exs b/priv/repo/migrations/20190118074940_fix_user_trigram_index.exs index b4e8c984c..b0d272802 100644 --- a/priv/repo/migrations/20190118074940_fix_user_trigram_index.exs +++ b/priv/repo/migrations/20190118074940_fix_user_trigram_index.exs @@ -4,7 +4,7 @@ defmodule Pleroma.Repo.Migrations.FixUserTrigramIndex do def up do drop_if_exists(index(:users, [], name: :users_trigram_index)) - create( + create_if_not_exists( index(:users, ["(trim(nickname || ' ' || coalesce(name, ''))) gist_trgm_ops"], name: :users_trigram_index, using: :gist @@ -15,7 +15,7 @@ defmodule Pleroma.Repo.Migrations.FixUserTrigramIndex do def down do drop_if_exists(index(:users, [], name: :users_trigram_index)) - create( + create_if_not_exists( index(:users, ["(nickname || name) gist_trgm_ops"], name: :users_trigram_index, using: :gist) ) end diff --git a/priv/repo/migrations/20190123092341_users_add_is_admin_index.exs b/priv/repo/migrations/20190123092341_users_add_is_admin_index.exs index ba6ff78b5..25f248c59 100644 --- a/priv/repo/migrations/20190123092341_users_add_is_admin_index.exs +++ b/priv/repo/migrations/20190123092341_users_add_is_admin_index.exs @@ -2,6 +2,6 @@ defmodule Pleroma.Repo.Migrations.UsersAddIsAdminIndex do use Ecto.Migration def change do - create(index(:users, ["(info->'is_admin')"], name: :users_is_admin_index, using: :gin)) + create_if_not_exists(index(:users, ["(info->'is_admin')"], name: :users_is_admin_index, using: :gin)) end end diff --git a/priv/repo/migrations/20190123125546_create_instances.exs b/priv/repo/migrations/20190123125546_create_instances.exs index 3d23b343e..a9b356bc3 100644 --- a/priv/repo/migrations/20190123125546_create_instances.exs +++ b/priv/repo/migrations/20190123125546_create_instances.exs @@ -2,14 +2,14 @@ defmodule Pleroma.Repo.Migrations.CreateInstances do use Ecto.Migration def change do - create table(:instances) do + create_if_not_exists table(:instances) do add :host, :string add :unreachable_since, :naive_datetime_usec timestamps() end - create unique_index(:instances, [:host]) - create index(:instances, [:unreachable_since]) + create_if_not_exists unique_index(:instances, [:host]) + create_if_not_exists index(:instances, [:unreachable_since]) end end diff --git a/priv/repo/migrations/20190123125839_fix_info_ids.exs b/priv/repo/migrations/20190123125839_fix_info_ids.exs index 2b4c2b5a9..5b263b3b3 100644 --- a/priv/repo/migrations/20190123125839_fix_info_ids.exs +++ b/priv/repo/migrations/20190123125839_fix_info_ids.exs @@ -1,9 +1,11 @@ defmodule Pleroma.Repo.Migrations.FixInfoIds do use Ecto.Migration - def change do + def up do execute( "update users set info = jsonb_set(info, '{id}', to_jsonb(uuid_generate_v4())) where info->'id' is null;" ) end + + def down, do: :ok end diff --git a/priv/repo/migrations/20190126160540_change_push_subscriptions_varchar.exs b/priv/repo/migrations/20190126160540_change_push_subscriptions_varchar.exs index 337fed156..8e1c0e630 100644 --- a/priv/repo/migrations/20190126160540_change_push_subscriptions_varchar.exs +++ b/priv/repo/migrations/20190126160540_change_push_subscriptions_varchar.exs @@ -1,9 +1,15 @@ defmodule Pleroma.Repo.Migrations.ChangePushSubscriptionsVarchar do use Ecto.Migration - def change do + def up do alter table(:push_subscriptions) do modify(:endpoint, :varchar) end end + + def down do + alter table(:push_subscriptions) do + modify(:endpoint, :string) + end + end end diff --git a/priv/repo/migrations/20190204200237_add_correct_dm_index.exs b/priv/repo/migrations/20190204200237_add_correct_dm_index.exs index 558732cd2..1facb37b8 100644 --- a/priv/repo/migrations/20190204200237_add_correct_dm_index.exs +++ b/priv/repo/migrations/20190204200237_add_correct_dm_index.exs @@ -19,7 +19,7 @@ defmodule Pleroma.Repo.Migrations.AddCorrectDMIndex do end def down do - drop( + drop_if_exists( index(:activities, ["activity_visibility(actor, recipients, data)", "id DESC"], name: :activities_visibility_index, concurrently: true, diff --git a/priv/repo/migrations/20190205114625_create_thread_mutes.exs b/priv/repo/migrations/20190205114625_create_thread_mutes.exs index 8e9eccbae..7e44db121 100644 --- a/priv/repo/migrations/20190205114625_create_thread_mutes.exs +++ b/priv/repo/migrations/20190205114625_create_thread_mutes.exs @@ -2,11 +2,11 @@ defmodule Pleroma.Repo.Migrations.CreateThreadMutes do use Ecto.Migration def change do - create table(:thread_mutes) do + create_if_not_exists table(:thread_mutes) do add :user_id, references(:users, type: :uuid, on_delete: :delete_all) add :context, :string end - create unique_index(:thread_mutes, [:user_id, :context], name: :unique_index) + create_if_not_exists unique_index(:thread_mutes, [:user_id, :context], name: :unique_index) end end diff --git a/priv/repo/migrations/20190315101315_create_registrations.exs b/priv/repo/migrations/20190315101315_create_registrations.exs index 6b28cbdd3..34a390a93 100644 --- a/priv/repo/migrations/20190315101315_create_registrations.exs +++ b/priv/repo/migrations/20190315101315_create_registrations.exs @@ -2,7 +2,7 @@ defmodule Pleroma.Repo.Migrations.CreateRegistrations do use Ecto.Migration def change do - create table(:registrations, primary_key: false) do + create_if_not_exists table(:registrations, primary_key: false) do add :id, :uuid, primary_key: true add :user_id, references(:users, type: :uuid, on_delete: :delete_all) add :provider, :string @@ -12,7 +12,7 @@ defmodule Pleroma.Repo.Migrations.CreateRegistrations do timestamps() end - create unique_index(:registrations, [:provider, :uid]) - create unique_index(:registrations, [:user_id, :provider, :uid]) + create_if_not_exists unique_index(:registrations, [:provider, :uid]) + create_if_not_exists unique_index(:registrations, [:user_id, :provider, :uid]) end end diff --git a/priv/repo/migrations/20190325185009_create_notification_id_index.exs b/priv/repo/migrations/20190325185009_create_notification_id_index.exs index a6ab38d02..01cb30559 100644 --- a/priv/repo/migrations/20190325185009_create_notification_id_index.exs +++ b/priv/repo/migrations/20190325185009_create_notification_id_index.exs @@ -2,6 +2,6 @@ defmodule Pleroma.Repo.Migrations.CreateNotificationIdIndex do use Ecto.Migration def change do - create index(:notifications, ["id desc nulls last"]) + create_if_not_exists index(:notifications, ["id desc nulls last"]) end end diff --git a/priv/repo/migrations/20190328053912_create_scheduled_activities.exs b/priv/repo/migrations/20190328053912_create_scheduled_activities.exs index dd737e25a..8a01bbdb1 100644 --- a/priv/repo/migrations/20190328053912_create_scheduled_activities.exs +++ b/priv/repo/migrations/20190328053912_create_scheduled_activities.exs @@ -2,7 +2,7 @@ defmodule Pleroma.Repo.Migrations.CreateScheduledActivities do use Ecto.Migration def change do - create table(:scheduled_activities) do + create_if_not_exists table(:scheduled_activities) do add(:user_id, references(:users, type: :uuid, on_delete: :delete_all)) add(:scheduled_at, :naive_datetime, null: false) add(:params, :map, null: false) @@ -10,7 +10,7 @@ defmodule Pleroma.Repo.Migrations.CreateScheduledActivities do timestamps() end - create(index(:scheduled_activities, [:scheduled_at])) - create(index(:scheduled_activities, [:user_id])) + create_if_not_exists(index(:scheduled_activities, [:scheduled_at])) + create_if_not_exists(index(:scheduled_activities, [:user_id])) end end diff --git a/priv/repo/migrations/20190403131720_add_oauth_token_indexes.exs b/priv/repo/migrations/20190403131720_add_oauth_token_indexes.exs index ebcd29389..ab1bf2165 100644 --- a/priv/repo/migrations/20190403131720_add_oauth_token_indexes.exs +++ b/priv/repo/migrations/20190403131720_add_oauth_token_indexes.exs @@ -2,8 +2,8 @@ defmodule Pleroma.Repo.Migrations.AddOauthTokenIndexes do use Ecto.Migration def change do - create(unique_index(:oauth_tokens, [:token])) - create(index(:oauth_tokens, [:app_id])) - create(index(:oauth_tokens, [:user_id])) + create_if_not_exists(unique_index(:oauth_tokens, [:token])) + create_if_not_exists(index(:oauth_tokens, [:app_id])) + create_if_not_exists(index(:oauth_tokens, [:user_id])) end end diff --git a/priv/repo/migrations/20190405160700_add_index_on_subscribers.exs b/priv/repo/migrations/20190405160700_add_index_on_subscribers.exs index 232f75c92..460dafb1b 100644 --- a/priv/repo/migrations/20190405160700_add_index_on_subscribers.exs +++ b/priv/repo/migrations/20190405160700_add_index_on_subscribers.exs @@ -1,6 +1,6 @@ defmodule Pleroma.Repo.Migrations.AddIndexOnSubscribers do use Ecto.Migration - + @disable_ddl_transaction true def change do create index(:users, ["(info->'subscribers')"], name: :users_subscribers_index, using: :gin, concurrently: true) diff --git a/priv/repo/migrations/20190408123347_create_conversations.exs b/priv/repo/migrations/20190408123347_create_conversations.exs index 0e0af30ae..7b7d89da7 100644 --- a/priv/repo/migrations/20190408123347_create_conversations.exs +++ b/priv/repo/migrations/20190408123347_create_conversations.exs @@ -6,12 +6,12 @@ defmodule Pleroma.Repo.Migrations.CreateConversations do use Ecto.Migration def change do - create table(:conversations) do + create_if_not_exists table(:conversations) do add(:ap_id, :string, null: false) timestamps() end - create table(:conversation_participations) do + create_if_not_exists table(:conversation_participations) do add(:user_id, references(:users, type: :uuid, on_delete: :delete_all)) add(:conversation_id, references(:conversations, on_delete: :delete_all)) add(:read, :boolean, default: false) @@ -19,8 +19,8 @@ defmodule Pleroma.Repo.Migrations.CreateConversations do timestamps() end - create index(:conversation_participations, [:conversation_id]) - create unique_index(:conversation_participations, [:user_id, :conversation_id]) - create unique_index(:conversations, [:ap_id]) + create_if_not_exists index(:conversation_participations, [:conversation_id]) + create_if_not_exists unique_index(:conversation_participations, [:user_id, :conversation_id]) + create_if_not_exists unique_index(:conversations, [:ap_id]) end end diff --git a/priv/repo/migrations/20190410152859_add_participation_updated_at_index.exs b/priv/repo/migrations/20190410152859_add_participation_updated_at_index.exs index 1ce688c52..b5ca2fc0f 100644 --- a/priv/repo/migrations/20190410152859_add_participation_updated_at_index.exs +++ b/priv/repo/migrations/20190410152859_add_participation_updated_at_index.exs @@ -2,6 +2,6 @@ defmodule Pleroma.Repo.Migrations.AddParticipationUpdatedAtIndex do use Ecto.Migration def change do - create index(:conversation_participations, ["updated_at desc"]) + create_if_not_exists index(:conversation_participations, ["updated_at desc"]) end end diff --git a/priv/repo/migrations/20190411094120_add_index_on_user_info_deactivated.exs b/priv/repo/migrations/20190411094120_add_index_on_user_info_deactivated.exs index d701dcecc..c19427f12 100644 --- a/priv/repo/migrations/20190411094120_add_index_on_user_info_deactivated.exs +++ b/priv/repo/migrations/20190411094120_add_index_on_user_info_deactivated.exs @@ -2,6 +2,6 @@ defmodule Pleroma.Repo.Migrations.AddIndexOnUserInfoDeactivated do use Ecto.Migration def change do - create(index(:users, ["(info->'deactivated')"], name: :users_deactivated_index, using: :gin)) + create_if_not_exists(index(:users, ["(info->'deactivated')"], name: :users_deactivated_index, using: :gin)) end end diff --git a/priv/repo/migrations/20190413082658_create_bookmarks.exs b/priv/repo/migrations/20190413082658_create_bookmarks.exs index 38b108158..cfd025fc5 100644 --- a/priv/repo/migrations/20190413082658_create_bookmarks.exs +++ b/priv/repo/migrations/20190413082658_create_bookmarks.exs @@ -2,13 +2,13 @@ defmodule Pleroma.Repo.Migrations.CreateBookmarks do use Ecto.Migration def change do - create table(:bookmarks) do + create_if_not_exists table(:bookmarks) do add(:user_id, references(:users, type: :uuid, on_delete: :delete_all)) add(:activity_id, references(:activities, type: :uuid, on_delete: :delete_all)) timestamps() end - create(unique_index(:bookmarks, [:user_id, :activity_id])) + create_if_not_exists(unique_index(:bookmarks, [:user_id, :activity_id])) end end diff --git a/priv/repo/migrations/20190414125034_migrate_old_bookmarks.exs b/priv/repo/migrations/20190414125034_migrate_old_bookmarks.exs index 134b7c6f7..ce4590954 100644 --- a/priv/repo/migrations/20190414125034_migrate_old_bookmarks.exs +++ b/priv/repo/migrations/20190414125034_migrate_old_bookmarks.exs @@ -6,7 +6,7 @@ defmodule Pleroma.Repo.Migrations.MigrateOldBookmarks do alias Pleroma.User alias Pleroma.Repo - def change do + def up do query = from(u in User, where: u.local == true, @@ -18,7 +18,7 @@ defmodule Pleroma.Repo.Migrations.MigrateOldBookmarks do |> Enum.each(fn %{id: user_id, bookmarks: bookmarks} -> Enum.each(bookmarks, fn ap_id -> activity = Activity.get_create_by_object_ap_id(ap_id) - unless is_nil(activity), do: {:ok, _} = Bookmark.create(user_id, activity.id) + unless is_nil(activity), do: {:ok, _} = Bookmark.create(user_id, activity.id) end) end) @@ -26,4 +26,10 @@ defmodule Pleroma.Repo.Migrations.MigrateOldBookmarks do remove(:bookmarks) end end + + def down do + alter table(:users) do + add :bookmarks, {:array, :string}, null: false, default: [] + end + end end diff --git a/priv/repo/migrations/20190501125843_add_fts_index_to_objects.exs b/priv/repo/migrations/20190501125843_add_fts_index_to_objects.exs index 9b274695e..d4de51691 100644 --- a/priv/repo/migrations/20190501125843_add_fts_index_to_objects.exs +++ b/priv/repo/migrations/20190501125843_add_fts_index_to_objects.exs @@ -3,6 +3,6 @@ defmodule Pleroma.Repo.Migrations.AddFTSIndexToObjects do def change do drop_if_exists index(:activities, ["(to_tsvector('english', data->'object'->>'content'))"], using: :gin, name: :activities_fts) - create index(:objects, ["(to_tsvector('english', data->>'content'))"], using: :gin, name: :objects_fts) + create_if_not_exists index(:objects, ["(to_tsvector('english', data->>'content'))"], using: :gin, name: :objects_fts) end end diff --git a/priv/repo/migrations/20190501133552_add_refresh_token_index_to_token.exs b/priv/repo/migrations/20190501133552_add_refresh_token_index_to_token.exs index 449f2a3d4..44a266985 100644 --- a/priv/repo/migrations/20190501133552_add_refresh_token_index_to_token.exs +++ b/priv/repo/migrations/20190501133552_add_refresh_token_index_to_token.exs @@ -2,6 +2,6 @@ defmodule Pleroma.Repo.Migrations.AddRefreshTokenIndexToToken do use Ecto.Migration def change do - create(unique_index(:oauth_tokens, [:refresh_token])) + create_if_not_exists(unique_index(:oauth_tokens, [:refresh_token])) end end diff --git a/priv/repo/migrations/20190513175809_change_hide_column_in_filter_table.exs b/priv/repo/migrations/20190513175809_change_hide_column_in_filter_table.exs index 2ffb88cc9..246b70cfb 100644 --- a/priv/repo/migrations/20190513175809_change_hide_column_in_filter_table.exs +++ b/priv/repo/migrations/20190513175809_change_hide_column_in_filter_table.exs @@ -1,9 +1,15 @@ defmodule Pleroma.Repo.Migrations.ChangeHideColumnInFilterTable do use Ecto.Migration - def change do + def up do alter table(:filters) do modify :hide, :boolean, default: false end end + + def down do + alter table(:filters) do + modify :hide, :boolean + end + end end diff --git a/priv/repo/migrations/20190515222404_add_thread_visibility_function.exs b/priv/repo/migrations/20190515222404_add_thread_visibility_function.exs index dc9abc998..5184c0c48 100644 --- a/priv/repo/migrations/20190515222404_add_thread_visibility_function.exs +++ b/priv/repo/migrations/20190515222404_add_thread_visibility_function.exs @@ -68,6 +68,6 @@ defmodule Pleroma.Repo.Migrations.AddThreadVisibilityFunction do end def down do - execute("drop function thread_visibility(actor varchar, activity_id varchar)") + execute("drop function if exists thread_visibility(actor varchar, activity_id varchar)") end end diff --git a/priv/repo/migrations/20190518032627_create_config.exs b/priv/repo/migrations/20190518032627_create_config.exs index 1e4e3c689..cb65f3939 100644 --- a/priv/repo/migrations/20190518032627_create_config.exs +++ b/priv/repo/migrations/20190518032627_create_config.exs @@ -2,12 +2,12 @@ defmodule Pleroma.Repo.Migrations.CreateConfig do use Ecto.Migration def change do - create table(:config) do + create_if_not_exists table(:config) do add(:key, :string) add(:value, :binary) timestamps() end - create(unique_index(:config, :key)) + create_if_not_exists(unique_index(:config, :key)) end end diff --git a/priv/repo/migrations/20190525071417_add_non_follows_and_non_followers_fields_to_notification_settings.exs b/priv/repo/migrations/20190525071417_add_non_follows_and_non_followers_fields_to_notification_settings.exs index a88b0ea61..505f0acb2 100644 --- a/priv/repo/migrations/20190525071417_add_non_follows_and_non_followers_fields_to_notification_settings.exs +++ b/priv/repo/migrations/20190525071417_add_non_follows_and_non_followers_fields_to_notification_settings.exs @@ -1,10 +1,12 @@ defmodule Pleroma.Repo.Migrations.AddNonFollowsAndNonFollowersFieldsToNotificationSettings do use Ecto.Migration - def change do + def up do execute(""" update users set info = jsonb_set(info, '{notification_settings}', '{"local": true, "remote": true, "follows": true, "followers": true, "non_follows": true, "non_followers": true}') where local=true """) end + + def down, do: :ok end diff --git a/priv/repo/migrations/20190603115238_add_index_on_activities_local.exs b/priv/repo/migrations/20190603115238_add_index_on_activities_local.exs index 89daa9705..7e6643497 100644 --- a/priv/repo/migrations/20190603115238_add_index_on_activities_local.exs +++ b/priv/repo/migrations/20190603115238_add_index_on_activities_local.exs @@ -2,6 +2,6 @@ defmodule Pleroma.Repo.Migrations.AddIndexOnActivitiesLocal do use Ecto.Migration def change do - create(index("activities", [:local])) + create_if_not_exists(index("activities", [:local])) end end diff --git a/priv/repo/migrations/20190603173419_add_tag_index_to_objects.exs b/priv/repo/migrations/20190603173419_add_tag_index_to_objects.exs index c915a0213..93d57a249 100644 --- a/priv/repo/migrations/20190603173419_add_tag_index_to_objects.exs +++ b/priv/repo/migrations/20190603173419_add_tag_index_to_objects.exs @@ -3,6 +3,6 @@ defmodule Pleroma.Repo.Migrations.AddTagIndexToObjects do def change do drop_if_exists index(:activities, ["(data #> '{\"object\",\"tag\"}')"], using: :gin, name: :activities_tags) - create index(:objects, ["(data->'tag')"], using: :gin, name: :objects_tags) + create_if_not_exists index(:objects, ["(data->'tag')"], using: :gin, name: :objects_tags) end end diff --git a/priv/repo/migrations/20190622151019_add_group_key_to_config.exs b/priv/repo/migrations/20190622151019_add_group_key_to_config.exs index d7a3785d0..867dafaf3 100644 --- a/priv/repo/migrations/20190622151019_add_group_key_to_config.exs +++ b/priv/repo/migrations/20190622151019_add_group_key_to_config.exs @@ -6,7 +6,7 @@ defmodule Pleroma.Repo.Migrations.AddGroupKeyToConfig do add(:group, :string) end - drop(unique_index("config", :key)) - create(unique_index("config", [:group, :key])) + drop_if_exists(unique_index("config", :key)) + create_if_not_exists(unique_index("config", [:group, :key])) end end diff --git a/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs b/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs index b6a24441a..6227769dc 100644 --- a/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs +++ b/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs @@ -14,7 +14,7 @@ defmodule Pleroma.Repo.Migrations.AddFtsIndexToObjectsTwo do return new; end $$ LANGUAGE plpgsql") - execute("create index objects_fts on objects using RUM (fts_content rum_tsvector_addon_ops, inserted_at) with (attach = 'inserted_at', to = 'fts_content');") + execute("create index if not exists objects_fts on objects using RUM (fts_content rum_tsvector_addon_ops, inserted_at) with (attach = 'inserted_at', to = 'fts_content');") execute("CREATE TRIGGER tsvectorupdate BEFORE INSERT OR UPDATE ON objects FOR EACH ROW EXECUTE PROCEDURE objects_fts_update()") @@ -23,12 +23,12 @@ defmodule Pleroma.Repo.Migrations.AddFtsIndexToObjectsTwo do end def down do - execute "drop index objects_fts" - execute "drop trigger tsvectorupdate on objects" - execute "drop function objects_fts_update()" + execute "drop index if exists objects_fts" + execute "drop trigger if exists tsvectorupdate on objects" + execute "drop function if exists objects_fts_update()" alter table(:objects) do remove(:fts_content, :tsvector) end - create index(:objects, ["(to_tsvector('english', data->>'content'))"], using: :gin, name: :objects_fts) + create_if_not_exists index(:objects, ["(to_tsvector('english', data->>'content'))"], using: :gin, name: :objects_fts) end end From 0b3561ae72ea1686f940366e41d6bd1a2262d665 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Mon, 1 Jul 2019 14:04:24 +0300 Subject: [PATCH 09/43] Move changelog entries --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ae7e4c0ed..8a0dad453 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Added - MRF: Support for priming the mediaproxy cache (`Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy`) - Mastodon API: Support for the [`tagged` filter](https://github.com/tootsuite/mastodon/pull/9755) in [`GET /api/v1/accounts/:id/statuses`](https://docs.joinmastodon.org/api/rest/accounts/#get-api-v1-accounts-id-statuses) +- Admin API: Return users' tags when querying reports +- Admin API: Return avatar and display name when querying users ### Fixed - Not being able to pin unlisted posts @@ -56,8 +58,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Admin API: Endpoints for managing reports - Admin API: Endpoints for deleting and changing the scope of individual reported statuses - Admin API: Endpoints to view and change config settings. -- Admin API: Return users' tags when querying reports -- Admin API: Return avatar and display name when querying users - AdminFE: initial release with basic user management accessible at /pleroma/admin/ - Mastodon API: Add chat token to `verify_credentials` response - Mastodon API: Add background image setting to `update_credentials` From ab50c6b3ca26c87e4b7fdbe04da5d0d236d2e9ce Mon Sep 17 00:00:00 2001 From: Eugenij Date: Tue, 2 Jul 2019 01:27:00 +0000 Subject: [PATCH 10/43] Add fallback value to the admin report view. This is to avoid 500 error when the reported user no longer exists in the database. --- lib/pleroma/web/admin_api/views/report_view.ex | 4 +++- test/web/admin_api/views/report_view_test.exs | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/admin_api/views/report_view.ex b/lib/pleroma/web/admin_api/views/report_view.ex index 73ccdc582..a25f3f1fe 100644 --- a/lib/pleroma/web/admin_api/views/report_view.ex +++ b/lib/pleroma/web/admin_api/views/report_view.ex @@ -46,8 +46,10 @@ defmodule Pleroma.Web.AdminAPI.ReportView do } end - defp merge_account_views(user) do + defp merge_account_views(%User{} = user) do Pleroma.Web.MastodonAPI.AccountView.render("account.json", %{user: user}) |> Map.merge(Pleroma.Web.AdminAPI.AccountView.render("show.json", %{user: user})) end + + defp merge_account_views(_), do: %{} end diff --git a/test/web/admin_api/views/report_view_test.exs b/test/web/admin_api/views/report_view_test.exs index 75d8bb4b5..a00c9c579 100644 --- a/test/web/admin_api/views/report_view_test.exs +++ b/test/web/admin_api/views/report_view_test.exs @@ -111,4 +111,20 @@ defmodule Pleroma.Web.AdminAPI.ReportViewTest do refute "" == ReportView.render("show.json", %{report: activity})[:content] end + + test "doesn't error out when the user doesn't exists" do + user = insert(:user) + other_user = insert(:user) + + {:ok, activity} = + CommonAPI.report(user, %{ + "account_id" => other_user.id, + "comment" => "" + }) + + Pleroma.User.delete(other_user) + Pleroma.User.invalidate_cache(other_user) + + assert %{} = ReportView.render("show.json", %{report: activity}) + end end From a02f52d12a32d78aa11a86840dfdaf94b5355dfb Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Tue, 2 Jul 2019 09:04:59 +0300 Subject: [PATCH 11/43] [#1048] Resolved violations of SemVer version format. Refactoring. --- mix.exs | 48 +++++++++++++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/mix.exs b/mix.exs index c2618d2b2..22d3d50df 100644 --- a/mix.exs +++ b/mix.exs @@ -174,10 +174,14 @@ defmodule Pleroma.Mixfile do # Builds a version string made of: # * the application version # * a pre-release if ahead of the tag: the describe string (-count-commithash) - # * build info: + # * branch name + # * build metadata: # * a build name if `PLEROMA_BUILD_NAME` or `:pleroma, :build_name` is defined # * the mix environment if different than prod defp version(version) do + identifier_filter = ~r/[^0-9a-z\-]+/i + + # Pre-release version, denoted from patch version with a hyphen {git_tag, git_pre_release} = with {tag, 0} <- System.cmd("git", ["describe", "--tags", "--abbrev=0"], stderr_to_stdout: true), @@ -198,6 +202,19 @@ defmodule Pleroma.Mixfile do ) end + # Branch name as pre-release version component, denoted with a dot + branch_name = + with {branch_name, 0} <- System.cmd("git", ["rev-parse", "--abbrev-ref", "HEAD"]), + branch_name <- System.get_env("PLEROMA_BUILD_BRANCH") || branch_name, + true <- branch_name != "master" do + branch_name = + branch_name + |> String.trim() + |> String.replace(identifier_filter, "-") + + "." <> branch_name + end + build_name = cond do name = Application.get_env(:pleroma, :build_name) -> name @@ -206,37 +223,26 @@ defmodule Pleroma.Mixfile do end env_name = if Mix.env() != :prod, do: to_string(Mix.env()) - env_override = System.get_env("PLEROMA_BUILD_ENV") env_name = - if env_override do - if env_override != "prod", do: env_override - else - env_name + case env_override do + nil -> env_name + env_override when env_override in ["", "prod"] -> nil + env_override -> env_override end - build = + # Build metadata, denoted with a plus sign + build_metadata = [build_name, env_name] |> Enum.filter(fn string -> string && string != "" end) - |> Enum.join("-") + |> Enum.join(".") |> (fn "" -> nil - string -> "+" <> string + string -> "+" <> String.replace(string, identifier_filter, "-") end).() - branch_name = - with {branch_name, 0} <- System.cmd("git", ["rev-parse", "--abbrev-ref", "HEAD"]), - branch_name <- System.get_env("PLEROMA_BUILD_BRANCH") || branch_name, - true <- branch_name != "master" do - branch_name = - String.trim(branch_name) - |> String.replace(~r/[^0-9a-z\-\.]+/i, "-") - - "-" <> branch_name - end - - [version, git_pre_release, branch_name, build] + [version, git_pre_release, branch_name, build_metadata] |> Enum.filter(fn string -> string && string != "" end) |> Enum.join() end From 8c7d7cc9365e996c624d4bdb6c8d578687ff117f Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Tue, 2 Jul 2019 18:47:53 +0300 Subject: [PATCH 12/43] [#1048] Fixed version parsing in pleroma_ctl. Closes #1036. --- rel/files/bin/pleroma_ctl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rel/files/bin/pleroma_ctl b/rel/files/bin/pleroma_ctl index 9c67b209b..e731d20eb 100755 --- a/rel/files/bin/pleroma_ctl +++ b/rel/files/bin/pleroma_ctl @@ -30,12 +30,15 @@ detect_flavour() { detect_branch() { version="$(cut -d' ' -f2 <"$RELEASE_ROOT"/releases/start_erl.data)" - branch="$(echo "$version" | cut -d'-' -f 4)" + # Expected format: major.minor.patch_version(-number_of_commits_ahead_of_tag-gcommit_hash).branch + branch="$(echo "$version" | cut -d'.' -f 4)" if [ "$branch" = "develop" ]; then echo "develop" elif [ "$branch" = "" ]; then echo "master" else + # Note: branch name in version is of SemVer format and may only contain [0-9a-zA-Z-] symbols — + # if supporting releases for more branches, need to ensure they contain only these symbols. echo "Releases are built only for master and develop branches" >&2 exit 1 fi From 291d95dcc3cd85050421d2ab022e01a6d89c0f3d Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Wed, 3 Jul 2019 01:14:40 +0300 Subject: [PATCH 13/43] Fix migration --- .../20170522160642_case_insensivtivity.exs | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/priv/repo/migrations/20170522160642_case_insensivtivity.exs b/priv/repo/migrations/20170522160642_case_insensivtivity.exs index c7565946e..470a545e5 100644 --- a/priv/repo/migrations/20170522160642_case_insensivtivity.exs +++ b/priv/repo/migrations/20170522160642_case_insensivtivity.exs @@ -2,18 +2,24 @@ defmodule Pleroma.Repo.Migrations.CaseInsensivtivity do use Ecto.Migration def up do - execute ("create extension if not exists citext") + execute("create extension if not exists citext") + + drop_if_exists(index(:users, [:email])) + alter table(:users) do - modify :email, :citext - modify :nickname, :citext + modify(:email, :citext) + modify(:nickname, :citext) end + + create_if_not_exists(index(:users, [:email])) end def down do alter table(:users) do - modify :email, :string - modify :nickname, :string + modify(:email, :string) + modify(:nickname, :string) end - execute ("drop extension if exists citext") + + execute("drop extension if exists citext") end end From 4f42093220393a13033fcfd306acf54c8791e98f Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 3 Jul 2019 14:11:04 +0700 Subject: [PATCH 14/43] Remove Uploaders.Swift --- lib/pleroma/uploaders/swift/keystone.ex | 51 ------------------------- lib/pleroma/uploaders/swift/swift.ex | 29 -------------- lib/pleroma/uploaders/swift/uploader.ex | 19 --------- priv/templates/sample_config.eex | 17 --------- 4 files changed, 116 deletions(-) delete mode 100644 lib/pleroma/uploaders/swift/keystone.ex delete mode 100644 lib/pleroma/uploaders/swift/swift.ex delete mode 100644 lib/pleroma/uploaders/swift/uploader.ex diff --git a/lib/pleroma/uploaders/swift/keystone.ex b/lib/pleroma/uploaders/swift/keystone.ex deleted file mode 100644 index dd44c7561..000000000 --- a/lib/pleroma/uploaders/swift/keystone.ex +++ /dev/null @@ -1,51 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Uploaders.Swift.Keystone do - use HTTPoison.Base - - def process_url(url) do - Enum.join( - [Pleroma.Config.get!([Pleroma.Uploaders.Swift, :auth_url]), url], - "/" - ) - end - - def process_response_body(body) do - body - |> Jason.decode!() - end - - def get_token do - settings = Pleroma.Config.get(Pleroma.Uploaders.Swift) - username = Keyword.fetch!(settings, :username) - password = Keyword.fetch!(settings, :password) - tenant_id = Keyword.fetch!(settings, :tenant_id) - - case post( - "/tokens", - make_auth_body(username, password, tenant_id), - ["Content-Type": "application/json"], - hackney: [:insecure] - ) do - {:ok, %Tesla.Env{status: 200, body: body}} -> - body["access"]["token"]["id"] - - {:ok, %Tesla.Env{status: _}} -> - "" - end - end - - def make_auth_body(username, password, tenant) do - Jason.encode!(%{ - :auth => %{ - :passwordCredentials => %{ - :username => username, - :password => password - }, - :tenantId => tenant - } - }) - end -end diff --git a/lib/pleroma/uploaders/swift/swift.ex b/lib/pleroma/uploaders/swift/swift.ex deleted file mode 100644 index 2b0f2ad04..000000000 --- a/lib/pleroma/uploaders/swift/swift.ex +++ /dev/null @@ -1,29 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Uploaders.Swift.Client do - use HTTPoison.Base - - def process_url(url) do - Enum.join( - [Pleroma.Config.get!([Pleroma.Uploaders.Swift, :storage_url]), url], - "/" - ) - end - - def upload_file(filename, body, content_type) do - token = Pleroma.Uploaders.Swift.Keystone.get_token() - - case put("#{filename}", body, "X-Auth-Token": token, "Content-Type": content_type) do - {:ok, %Tesla.Env{status: 201}} -> - {:ok, {:file, filename}} - - {:ok, %Tesla.Env{status: 401}} -> - {:error, "Unauthorized, Bad Token"} - - {:error, _} -> - {:error, "Swift Upload Error"} - end - end -end diff --git a/lib/pleroma/uploaders/swift/uploader.ex b/lib/pleroma/uploaders/swift/uploader.ex deleted file mode 100644 index d122b09e7..000000000 --- a/lib/pleroma/uploaders/swift/uploader.ex +++ /dev/null @@ -1,19 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Uploaders.Swift do - @behaviour Pleroma.Uploaders.Uploader - - def get_file(name) do - {:ok, {:url, Path.join([Pleroma.Config.get!([__MODULE__, :object_url]), name])}} - end - - def put_file(upload) do - Pleroma.Uploaders.Swift.Client.upload_file( - upload.path, - File.read!(upload.tmpfile), - upload.content_type - ) - end -end diff --git a/priv/templates/sample_config.eex b/priv/templates/sample_config.eex index 2d4a49328..5cc31c604 100644 --- a/priv/templates/sample_config.eex +++ b/priv/templates/sample_config.eex @@ -67,20 +67,3 @@ config :pleroma, Pleroma.Uploaders.Local, uploads: "<%= uploads_dir %>" # For using third-party S3 clones like wasabi, also do: # config :ex_aws, :s3, # host: "s3.wasabisys.com" - - -# Configure Openstack Swift support if desired. -# -# Many openstack deployments are different, so config is left very open with -# no assumptions made on which provider you're using. This should allow very -# wide support without needing separate handlers for OVH, Rackspace, etc. -# -# config :pleroma, Pleroma.Uploaders.Swift, -# container: "some-container", -# username: "api-username-yyyy", -# password: "api-key-xxxx", -# tenant_id: "", -# auth_url: "https://keystone-endpoint.provider.com", -# storage_url: "https://swift-endpoint.prodider.com/v1/AUTH_/", -# object_url: "https://cdn-endpoint.provider.com/" -# From 11143c542a012f64db8db0fa7b82f063ef338e94 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 3 Jul 2019 14:42:24 +0700 Subject: [PATCH 15/43] Remove `httpoison` from dependencies --- mix.exs | 1 - .../7369654.atom | 0 .../7369654.html | 0 .../{httpoison_mock => tesla_mock}/7even.json | 0 .../admin@mastdon.example.org.json | 0 .../atarifrosch_feed.xml | 0 .../atarifrosch_webfinger.xml | 0 .../baptiste.gelex.xyz-article.json | 0 .../baptiste.gelex.xyz-user.json | 0 .../eal_sakamoto.xml | 0 .../emelie.atom | 0 .../emelie.json | 0 .../framasoft@framatube.org.json | 0 .../framatube.org_host_meta | 0 .../gerzilla.de_host_meta | 0 .../gnusocial.de_host_meta | 0 .../gs.example.org_host_meta | 0 .../hellpie.json | 0 ...__gs.example.org_4040_index.php_user_1.xml | 0 ...n.example.org_users_admin_status_1234.json | 0 ....php_api_statuses_user_timeline_1.atom.xml | 0 .../https___info.pleroma.site_actor.json | 0 .../https___mamot.fr_users_Skruyb.atom | 0 ...__mastodon.social_users_lambadalambda.atom | 0 ...___mastodon.social_users_lambadalambda.xml | 0 ...ps___osada.macgirvin.com_channel_mike.json | 0 .../https___pawoo.net_users_aqidaqidaqid.xml | 0 .../https___pawoo.net_users_pekorino.atom | 0 .../https___pawoo.net_users_pekorino.xml | 0 .../https___pleroma.soykaf.com_users_lain.xml | 0 ...leroma.soykaf.com_users_lain_feed.atom.xml | 0 .../https___prismo.news__mxb.json | 0 ...er.club_api_statuses_show_2827873.atom.xml | 0 ...club_api_statuses_user_timeline_1.atom.xml | 0 ...ttps___shitposter.club_notice_2827873.html | 0 .../https___shitposter.club_user_1.xml | 0 ..._api_statuses_user_timeline_23211.atom.xml | 0 ..._api_statuses_user_timeline_29191.atom.xml | 0 .../https___social.heldscal.la_user_23211.xml | 0 .../https___social.heldscal.la_user_29191.xml | 0 .../https__info.pleroma.site_activity.json | 0 .../https__info.pleroma.site_activity2.json | 0 .../https__info.pleroma.site_activity3.json | 0 .../https__info.pleroma.site_activity4.json | 0 .../kaniini@gerzilla.de.json | 0 .../kaniini@hubzilla.example.org.json | 0 .../lain_squeet.me_webfinger.xml | 0 .../lucifermysticus.json | 0 .../macgirvin.com_host_meta | 0 .../mamot.fr_host_meta | 0 .../mastodon.social_host_meta | 0 .../mastodon.xyz_host_meta | 0 .../mayumayu.json | 0 .../mayumayupost.json | 0 .../mike@osada.macgirvin.com.json | 0 .../nonexistant@social.heldscal.la.xml | 0 .../pawoo.net_host_meta | 0 .../peertube.moe-vid.json | 0 .../pleroma.soykaf.com_host_meta | 0 .../puckipedia.com.json | 0 .../rinpatch.json | 0 .../{httpoison_mock => tesla_mock}/rye.json | 0 .../sakamoto.atom | 0 .../sakamoto_eal_feed.atom | 0 .../shitposter.club_host_meta | 0 .../shp@pleroma.soykaf.com.feed | 0 .../shp@pleroma.soykaf.com.webfigner | 0 .../shp@social.heldscal.la.xml | 0 .../skruyb@mamot.fr.atom | 0 .../social.heldscal.la_host_meta | 0 .../social.sakamoto.gq_host_meta | 0 ...ial.stopwatchingus-heidelberg.de_host_meta | 0 .../social.wxcafe.net_host_meta | 0 .../spc_5381.atom | 0 .../spc_5381_xrd.xml | 0 .../squeet.me_host_meta | 0 .../status.alpicola.com_host_meta | 0 .../status.emelie.json | 0 .../webfinger_emelie.json | 0 .../winterdienst_webfinger.json | 0 test/support/http_request_mock.ex | 140 +++++++++--------- 81 files changed, 67 insertions(+), 74 deletions(-) rename test/fixtures/{httpoison_mock => tesla_mock}/7369654.atom (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/7369654.html (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/7even.json (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/admin@mastdon.example.org.json (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/atarifrosch_feed.xml (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/atarifrosch_webfinger.xml (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/baptiste.gelex.xyz-article.json (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/baptiste.gelex.xyz-user.json (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/eal_sakamoto.xml (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/emelie.atom (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/emelie.json (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/framasoft@framatube.org.json (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/framatube.org_host_meta (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/gerzilla.de_host_meta (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/gnusocial.de_host_meta (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/gs.example.org_host_meta (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/hellpie.json (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/http___gs.example.org_4040_index.php_user_1.xml (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/http___mastodon.example.org_users_admin_status_1234.json (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/http__gs.example.org_index.php_api_statuses_user_timeline_1.atom.xml (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/https___info.pleroma.site_actor.json (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/https___mamot.fr_users_Skruyb.atom (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/https___mastodon.social_users_lambadalambda.atom (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/https___mastodon.social_users_lambadalambda.xml (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/https___osada.macgirvin.com_channel_mike.json (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/https___pawoo.net_users_aqidaqidaqid.xml (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/https___pawoo.net_users_pekorino.atom (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/https___pawoo.net_users_pekorino.xml (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/https___pleroma.soykaf.com_users_lain.xml (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/https___pleroma.soykaf.com_users_lain_feed.atom.xml (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/https___prismo.news__mxb.json (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/https___shitposter.club_api_statuses_show_2827873.atom.xml (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/https___shitposter.club_api_statuses_user_timeline_1.atom.xml (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/https___shitposter.club_notice_2827873.html (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/https___shitposter.club_user_1.xml (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/https___social.heldscal.la_api_statuses_user_timeline_23211.atom.xml (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/https___social.heldscal.la_api_statuses_user_timeline_29191.atom.xml (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/https___social.heldscal.la_user_23211.xml (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/https___social.heldscal.la_user_29191.xml (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/https__info.pleroma.site_activity.json (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/https__info.pleroma.site_activity2.json (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/https__info.pleroma.site_activity3.json (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/https__info.pleroma.site_activity4.json (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/kaniini@gerzilla.de.json (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/kaniini@hubzilla.example.org.json (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/lain_squeet.me_webfinger.xml (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/lucifermysticus.json (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/macgirvin.com_host_meta (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/mamot.fr_host_meta (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/mastodon.social_host_meta (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/mastodon.xyz_host_meta (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/mayumayu.json (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/mayumayupost.json (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/mike@osada.macgirvin.com.json (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/nonexistant@social.heldscal.la.xml (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/pawoo.net_host_meta (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/peertube.moe-vid.json (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/pleroma.soykaf.com_host_meta (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/puckipedia.com.json (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/rinpatch.json (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/rye.json (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/sakamoto.atom (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/sakamoto_eal_feed.atom (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/shitposter.club_host_meta (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/shp@pleroma.soykaf.com.feed (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/shp@pleroma.soykaf.com.webfigner (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/shp@social.heldscal.la.xml (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/skruyb@mamot.fr.atom (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/social.heldscal.la_host_meta (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/social.sakamoto.gq_host_meta (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/social.stopwatchingus-heidelberg.de_host_meta (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/social.wxcafe.net_host_meta (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/spc_5381.atom (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/spc_5381_xrd.xml (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/squeet.me_host_meta (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/status.alpicola.com_host_meta (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/status.emelie.json (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/webfinger_emelie.json (100%) rename test/fixtures/{httpoison_mock => tesla_mock}/winterdienst_webfinger.json (100%) diff --git a/mix.exs b/mix.exs index c2618d2b2..518de590d 100644 --- a/mix.exs +++ b/mix.exs @@ -109,7 +109,6 @@ defmodule Pleroma.Mixfile do {:phoenix_html, "~> 2.10"}, {:calendar, "~> 0.17.4"}, {:cachex, "~> 3.0.2"}, - {:httpoison, "~> 1.2.0"}, {:poison, "~> 3.0", override: true}, {:tesla, "~> 1.2"}, {:jason, "~> 1.0"}, diff --git a/test/fixtures/httpoison_mock/7369654.atom b/test/fixtures/tesla_mock/7369654.atom similarity index 100% rename from test/fixtures/httpoison_mock/7369654.atom rename to test/fixtures/tesla_mock/7369654.atom diff --git a/test/fixtures/httpoison_mock/7369654.html b/test/fixtures/tesla_mock/7369654.html similarity index 100% rename from test/fixtures/httpoison_mock/7369654.html rename to test/fixtures/tesla_mock/7369654.html diff --git a/test/fixtures/httpoison_mock/7even.json b/test/fixtures/tesla_mock/7even.json similarity index 100% rename from test/fixtures/httpoison_mock/7even.json rename to test/fixtures/tesla_mock/7even.json diff --git a/test/fixtures/httpoison_mock/admin@mastdon.example.org.json b/test/fixtures/tesla_mock/admin@mastdon.example.org.json similarity index 100% rename from test/fixtures/httpoison_mock/admin@mastdon.example.org.json rename to test/fixtures/tesla_mock/admin@mastdon.example.org.json diff --git a/test/fixtures/httpoison_mock/atarifrosch_feed.xml b/test/fixtures/tesla_mock/atarifrosch_feed.xml similarity index 100% rename from test/fixtures/httpoison_mock/atarifrosch_feed.xml rename to test/fixtures/tesla_mock/atarifrosch_feed.xml diff --git a/test/fixtures/httpoison_mock/atarifrosch_webfinger.xml b/test/fixtures/tesla_mock/atarifrosch_webfinger.xml similarity index 100% rename from test/fixtures/httpoison_mock/atarifrosch_webfinger.xml rename to test/fixtures/tesla_mock/atarifrosch_webfinger.xml diff --git a/test/fixtures/httpoison_mock/baptiste.gelex.xyz-article.json b/test/fixtures/tesla_mock/baptiste.gelex.xyz-article.json similarity index 100% rename from test/fixtures/httpoison_mock/baptiste.gelex.xyz-article.json rename to test/fixtures/tesla_mock/baptiste.gelex.xyz-article.json diff --git a/test/fixtures/httpoison_mock/baptiste.gelex.xyz-user.json b/test/fixtures/tesla_mock/baptiste.gelex.xyz-user.json similarity index 100% rename from test/fixtures/httpoison_mock/baptiste.gelex.xyz-user.json rename to test/fixtures/tesla_mock/baptiste.gelex.xyz-user.json diff --git a/test/fixtures/httpoison_mock/eal_sakamoto.xml b/test/fixtures/tesla_mock/eal_sakamoto.xml similarity index 100% rename from test/fixtures/httpoison_mock/eal_sakamoto.xml rename to test/fixtures/tesla_mock/eal_sakamoto.xml diff --git a/test/fixtures/httpoison_mock/emelie.atom b/test/fixtures/tesla_mock/emelie.atom similarity index 100% rename from test/fixtures/httpoison_mock/emelie.atom rename to test/fixtures/tesla_mock/emelie.atom diff --git a/test/fixtures/httpoison_mock/emelie.json b/test/fixtures/tesla_mock/emelie.json similarity index 100% rename from test/fixtures/httpoison_mock/emelie.json rename to test/fixtures/tesla_mock/emelie.json diff --git a/test/fixtures/httpoison_mock/framasoft@framatube.org.json b/test/fixtures/tesla_mock/framasoft@framatube.org.json similarity index 100% rename from test/fixtures/httpoison_mock/framasoft@framatube.org.json rename to test/fixtures/tesla_mock/framasoft@framatube.org.json diff --git a/test/fixtures/httpoison_mock/framatube.org_host_meta b/test/fixtures/tesla_mock/framatube.org_host_meta similarity index 100% rename from test/fixtures/httpoison_mock/framatube.org_host_meta rename to test/fixtures/tesla_mock/framatube.org_host_meta diff --git a/test/fixtures/httpoison_mock/gerzilla.de_host_meta b/test/fixtures/tesla_mock/gerzilla.de_host_meta similarity index 100% rename from test/fixtures/httpoison_mock/gerzilla.de_host_meta rename to test/fixtures/tesla_mock/gerzilla.de_host_meta diff --git a/test/fixtures/httpoison_mock/gnusocial.de_host_meta b/test/fixtures/tesla_mock/gnusocial.de_host_meta similarity index 100% rename from test/fixtures/httpoison_mock/gnusocial.de_host_meta rename to test/fixtures/tesla_mock/gnusocial.de_host_meta diff --git a/test/fixtures/httpoison_mock/gs.example.org_host_meta b/test/fixtures/tesla_mock/gs.example.org_host_meta similarity index 100% rename from test/fixtures/httpoison_mock/gs.example.org_host_meta rename to test/fixtures/tesla_mock/gs.example.org_host_meta diff --git a/test/fixtures/httpoison_mock/hellpie.json b/test/fixtures/tesla_mock/hellpie.json similarity index 100% rename from test/fixtures/httpoison_mock/hellpie.json rename to test/fixtures/tesla_mock/hellpie.json diff --git a/test/fixtures/httpoison_mock/http___gs.example.org_4040_index.php_user_1.xml b/test/fixtures/tesla_mock/http___gs.example.org_4040_index.php_user_1.xml similarity index 100% rename from test/fixtures/httpoison_mock/http___gs.example.org_4040_index.php_user_1.xml rename to test/fixtures/tesla_mock/http___gs.example.org_4040_index.php_user_1.xml diff --git a/test/fixtures/httpoison_mock/http___mastodon.example.org_users_admin_status_1234.json b/test/fixtures/tesla_mock/http___mastodon.example.org_users_admin_status_1234.json similarity index 100% rename from test/fixtures/httpoison_mock/http___mastodon.example.org_users_admin_status_1234.json rename to test/fixtures/tesla_mock/http___mastodon.example.org_users_admin_status_1234.json diff --git a/test/fixtures/httpoison_mock/http__gs.example.org_index.php_api_statuses_user_timeline_1.atom.xml b/test/fixtures/tesla_mock/http__gs.example.org_index.php_api_statuses_user_timeline_1.atom.xml similarity index 100% rename from test/fixtures/httpoison_mock/http__gs.example.org_index.php_api_statuses_user_timeline_1.atom.xml rename to test/fixtures/tesla_mock/http__gs.example.org_index.php_api_statuses_user_timeline_1.atom.xml diff --git a/test/fixtures/httpoison_mock/https___info.pleroma.site_actor.json b/test/fixtures/tesla_mock/https___info.pleroma.site_actor.json similarity index 100% rename from test/fixtures/httpoison_mock/https___info.pleroma.site_actor.json rename to test/fixtures/tesla_mock/https___info.pleroma.site_actor.json diff --git a/test/fixtures/httpoison_mock/https___mamot.fr_users_Skruyb.atom b/test/fixtures/tesla_mock/https___mamot.fr_users_Skruyb.atom similarity index 100% rename from test/fixtures/httpoison_mock/https___mamot.fr_users_Skruyb.atom rename to test/fixtures/tesla_mock/https___mamot.fr_users_Skruyb.atom diff --git a/test/fixtures/httpoison_mock/https___mastodon.social_users_lambadalambda.atom b/test/fixtures/tesla_mock/https___mastodon.social_users_lambadalambda.atom similarity index 100% rename from test/fixtures/httpoison_mock/https___mastodon.social_users_lambadalambda.atom rename to test/fixtures/tesla_mock/https___mastodon.social_users_lambadalambda.atom diff --git a/test/fixtures/httpoison_mock/https___mastodon.social_users_lambadalambda.xml b/test/fixtures/tesla_mock/https___mastodon.social_users_lambadalambda.xml similarity index 100% rename from test/fixtures/httpoison_mock/https___mastodon.social_users_lambadalambda.xml rename to test/fixtures/tesla_mock/https___mastodon.social_users_lambadalambda.xml diff --git a/test/fixtures/httpoison_mock/https___osada.macgirvin.com_channel_mike.json b/test/fixtures/tesla_mock/https___osada.macgirvin.com_channel_mike.json similarity index 100% rename from test/fixtures/httpoison_mock/https___osada.macgirvin.com_channel_mike.json rename to test/fixtures/tesla_mock/https___osada.macgirvin.com_channel_mike.json diff --git a/test/fixtures/httpoison_mock/https___pawoo.net_users_aqidaqidaqid.xml b/test/fixtures/tesla_mock/https___pawoo.net_users_aqidaqidaqid.xml similarity index 100% rename from test/fixtures/httpoison_mock/https___pawoo.net_users_aqidaqidaqid.xml rename to test/fixtures/tesla_mock/https___pawoo.net_users_aqidaqidaqid.xml diff --git a/test/fixtures/httpoison_mock/https___pawoo.net_users_pekorino.atom b/test/fixtures/tesla_mock/https___pawoo.net_users_pekorino.atom similarity index 100% rename from test/fixtures/httpoison_mock/https___pawoo.net_users_pekorino.atom rename to test/fixtures/tesla_mock/https___pawoo.net_users_pekorino.atom diff --git a/test/fixtures/httpoison_mock/https___pawoo.net_users_pekorino.xml b/test/fixtures/tesla_mock/https___pawoo.net_users_pekorino.xml similarity index 100% rename from test/fixtures/httpoison_mock/https___pawoo.net_users_pekorino.xml rename to test/fixtures/tesla_mock/https___pawoo.net_users_pekorino.xml diff --git a/test/fixtures/httpoison_mock/https___pleroma.soykaf.com_users_lain.xml b/test/fixtures/tesla_mock/https___pleroma.soykaf.com_users_lain.xml similarity index 100% rename from test/fixtures/httpoison_mock/https___pleroma.soykaf.com_users_lain.xml rename to test/fixtures/tesla_mock/https___pleroma.soykaf.com_users_lain.xml diff --git a/test/fixtures/httpoison_mock/https___pleroma.soykaf.com_users_lain_feed.atom.xml b/test/fixtures/tesla_mock/https___pleroma.soykaf.com_users_lain_feed.atom.xml similarity index 100% rename from test/fixtures/httpoison_mock/https___pleroma.soykaf.com_users_lain_feed.atom.xml rename to test/fixtures/tesla_mock/https___pleroma.soykaf.com_users_lain_feed.atom.xml diff --git a/test/fixtures/httpoison_mock/https___prismo.news__mxb.json b/test/fixtures/tesla_mock/https___prismo.news__mxb.json similarity index 100% rename from test/fixtures/httpoison_mock/https___prismo.news__mxb.json rename to test/fixtures/tesla_mock/https___prismo.news__mxb.json diff --git a/test/fixtures/httpoison_mock/https___shitposter.club_api_statuses_show_2827873.atom.xml b/test/fixtures/tesla_mock/https___shitposter.club_api_statuses_show_2827873.atom.xml similarity index 100% rename from test/fixtures/httpoison_mock/https___shitposter.club_api_statuses_show_2827873.atom.xml rename to test/fixtures/tesla_mock/https___shitposter.club_api_statuses_show_2827873.atom.xml diff --git a/test/fixtures/httpoison_mock/https___shitposter.club_api_statuses_user_timeline_1.atom.xml b/test/fixtures/tesla_mock/https___shitposter.club_api_statuses_user_timeline_1.atom.xml similarity index 100% rename from test/fixtures/httpoison_mock/https___shitposter.club_api_statuses_user_timeline_1.atom.xml rename to test/fixtures/tesla_mock/https___shitposter.club_api_statuses_user_timeline_1.atom.xml diff --git a/test/fixtures/httpoison_mock/https___shitposter.club_notice_2827873.html b/test/fixtures/tesla_mock/https___shitposter.club_notice_2827873.html similarity index 100% rename from test/fixtures/httpoison_mock/https___shitposter.club_notice_2827873.html rename to test/fixtures/tesla_mock/https___shitposter.club_notice_2827873.html diff --git a/test/fixtures/httpoison_mock/https___shitposter.club_user_1.xml b/test/fixtures/tesla_mock/https___shitposter.club_user_1.xml similarity index 100% rename from test/fixtures/httpoison_mock/https___shitposter.club_user_1.xml rename to test/fixtures/tesla_mock/https___shitposter.club_user_1.xml diff --git a/test/fixtures/httpoison_mock/https___social.heldscal.la_api_statuses_user_timeline_23211.atom.xml b/test/fixtures/tesla_mock/https___social.heldscal.la_api_statuses_user_timeline_23211.atom.xml similarity index 100% rename from test/fixtures/httpoison_mock/https___social.heldscal.la_api_statuses_user_timeline_23211.atom.xml rename to test/fixtures/tesla_mock/https___social.heldscal.la_api_statuses_user_timeline_23211.atom.xml diff --git a/test/fixtures/httpoison_mock/https___social.heldscal.la_api_statuses_user_timeline_29191.atom.xml b/test/fixtures/tesla_mock/https___social.heldscal.la_api_statuses_user_timeline_29191.atom.xml similarity index 100% rename from test/fixtures/httpoison_mock/https___social.heldscal.la_api_statuses_user_timeline_29191.atom.xml rename to test/fixtures/tesla_mock/https___social.heldscal.la_api_statuses_user_timeline_29191.atom.xml diff --git a/test/fixtures/httpoison_mock/https___social.heldscal.la_user_23211.xml b/test/fixtures/tesla_mock/https___social.heldscal.la_user_23211.xml similarity index 100% rename from test/fixtures/httpoison_mock/https___social.heldscal.la_user_23211.xml rename to test/fixtures/tesla_mock/https___social.heldscal.la_user_23211.xml diff --git a/test/fixtures/httpoison_mock/https___social.heldscal.la_user_29191.xml b/test/fixtures/tesla_mock/https___social.heldscal.la_user_29191.xml similarity index 100% rename from test/fixtures/httpoison_mock/https___social.heldscal.la_user_29191.xml rename to test/fixtures/tesla_mock/https___social.heldscal.la_user_29191.xml diff --git a/test/fixtures/httpoison_mock/https__info.pleroma.site_activity.json b/test/fixtures/tesla_mock/https__info.pleroma.site_activity.json similarity index 100% rename from test/fixtures/httpoison_mock/https__info.pleroma.site_activity.json rename to test/fixtures/tesla_mock/https__info.pleroma.site_activity.json diff --git a/test/fixtures/httpoison_mock/https__info.pleroma.site_activity2.json b/test/fixtures/tesla_mock/https__info.pleroma.site_activity2.json similarity index 100% rename from test/fixtures/httpoison_mock/https__info.pleroma.site_activity2.json rename to test/fixtures/tesla_mock/https__info.pleroma.site_activity2.json diff --git a/test/fixtures/httpoison_mock/https__info.pleroma.site_activity3.json b/test/fixtures/tesla_mock/https__info.pleroma.site_activity3.json similarity index 100% rename from test/fixtures/httpoison_mock/https__info.pleroma.site_activity3.json rename to test/fixtures/tesla_mock/https__info.pleroma.site_activity3.json diff --git a/test/fixtures/httpoison_mock/https__info.pleroma.site_activity4.json b/test/fixtures/tesla_mock/https__info.pleroma.site_activity4.json similarity index 100% rename from test/fixtures/httpoison_mock/https__info.pleroma.site_activity4.json rename to test/fixtures/tesla_mock/https__info.pleroma.site_activity4.json diff --git a/test/fixtures/httpoison_mock/kaniini@gerzilla.de.json b/test/fixtures/tesla_mock/kaniini@gerzilla.de.json similarity index 100% rename from test/fixtures/httpoison_mock/kaniini@gerzilla.de.json rename to test/fixtures/tesla_mock/kaniini@gerzilla.de.json diff --git a/test/fixtures/httpoison_mock/kaniini@hubzilla.example.org.json b/test/fixtures/tesla_mock/kaniini@hubzilla.example.org.json similarity index 100% rename from test/fixtures/httpoison_mock/kaniini@hubzilla.example.org.json rename to test/fixtures/tesla_mock/kaniini@hubzilla.example.org.json diff --git a/test/fixtures/httpoison_mock/lain_squeet.me_webfinger.xml b/test/fixtures/tesla_mock/lain_squeet.me_webfinger.xml similarity index 100% rename from test/fixtures/httpoison_mock/lain_squeet.me_webfinger.xml rename to test/fixtures/tesla_mock/lain_squeet.me_webfinger.xml diff --git a/test/fixtures/httpoison_mock/lucifermysticus.json b/test/fixtures/tesla_mock/lucifermysticus.json similarity index 100% rename from test/fixtures/httpoison_mock/lucifermysticus.json rename to test/fixtures/tesla_mock/lucifermysticus.json diff --git a/test/fixtures/httpoison_mock/macgirvin.com_host_meta b/test/fixtures/tesla_mock/macgirvin.com_host_meta similarity index 100% rename from test/fixtures/httpoison_mock/macgirvin.com_host_meta rename to test/fixtures/tesla_mock/macgirvin.com_host_meta diff --git a/test/fixtures/httpoison_mock/mamot.fr_host_meta b/test/fixtures/tesla_mock/mamot.fr_host_meta similarity index 100% rename from test/fixtures/httpoison_mock/mamot.fr_host_meta rename to test/fixtures/tesla_mock/mamot.fr_host_meta diff --git a/test/fixtures/httpoison_mock/mastodon.social_host_meta b/test/fixtures/tesla_mock/mastodon.social_host_meta similarity index 100% rename from test/fixtures/httpoison_mock/mastodon.social_host_meta rename to test/fixtures/tesla_mock/mastodon.social_host_meta diff --git a/test/fixtures/httpoison_mock/mastodon.xyz_host_meta b/test/fixtures/tesla_mock/mastodon.xyz_host_meta similarity index 100% rename from test/fixtures/httpoison_mock/mastodon.xyz_host_meta rename to test/fixtures/tesla_mock/mastodon.xyz_host_meta diff --git a/test/fixtures/httpoison_mock/mayumayu.json b/test/fixtures/tesla_mock/mayumayu.json similarity index 100% rename from test/fixtures/httpoison_mock/mayumayu.json rename to test/fixtures/tesla_mock/mayumayu.json diff --git a/test/fixtures/httpoison_mock/mayumayupost.json b/test/fixtures/tesla_mock/mayumayupost.json similarity index 100% rename from test/fixtures/httpoison_mock/mayumayupost.json rename to test/fixtures/tesla_mock/mayumayupost.json diff --git a/test/fixtures/httpoison_mock/mike@osada.macgirvin.com.json b/test/fixtures/tesla_mock/mike@osada.macgirvin.com.json similarity index 100% rename from test/fixtures/httpoison_mock/mike@osada.macgirvin.com.json rename to test/fixtures/tesla_mock/mike@osada.macgirvin.com.json diff --git a/test/fixtures/httpoison_mock/nonexistant@social.heldscal.la.xml b/test/fixtures/tesla_mock/nonexistant@social.heldscal.la.xml similarity index 100% rename from test/fixtures/httpoison_mock/nonexistant@social.heldscal.la.xml rename to test/fixtures/tesla_mock/nonexistant@social.heldscal.la.xml diff --git a/test/fixtures/httpoison_mock/pawoo.net_host_meta b/test/fixtures/tesla_mock/pawoo.net_host_meta similarity index 100% rename from test/fixtures/httpoison_mock/pawoo.net_host_meta rename to test/fixtures/tesla_mock/pawoo.net_host_meta diff --git a/test/fixtures/httpoison_mock/peertube.moe-vid.json b/test/fixtures/tesla_mock/peertube.moe-vid.json similarity index 100% rename from test/fixtures/httpoison_mock/peertube.moe-vid.json rename to test/fixtures/tesla_mock/peertube.moe-vid.json diff --git a/test/fixtures/httpoison_mock/pleroma.soykaf.com_host_meta b/test/fixtures/tesla_mock/pleroma.soykaf.com_host_meta similarity index 100% rename from test/fixtures/httpoison_mock/pleroma.soykaf.com_host_meta rename to test/fixtures/tesla_mock/pleroma.soykaf.com_host_meta diff --git a/test/fixtures/httpoison_mock/puckipedia.com.json b/test/fixtures/tesla_mock/puckipedia.com.json similarity index 100% rename from test/fixtures/httpoison_mock/puckipedia.com.json rename to test/fixtures/tesla_mock/puckipedia.com.json diff --git a/test/fixtures/httpoison_mock/rinpatch.json b/test/fixtures/tesla_mock/rinpatch.json similarity index 100% rename from test/fixtures/httpoison_mock/rinpatch.json rename to test/fixtures/tesla_mock/rinpatch.json diff --git a/test/fixtures/httpoison_mock/rye.json b/test/fixtures/tesla_mock/rye.json similarity index 100% rename from test/fixtures/httpoison_mock/rye.json rename to test/fixtures/tesla_mock/rye.json diff --git a/test/fixtures/httpoison_mock/sakamoto.atom b/test/fixtures/tesla_mock/sakamoto.atom similarity index 100% rename from test/fixtures/httpoison_mock/sakamoto.atom rename to test/fixtures/tesla_mock/sakamoto.atom diff --git a/test/fixtures/httpoison_mock/sakamoto_eal_feed.atom b/test/fixtures/tesla_mock/sakamoto_eal_feed.atom similarity index 100% rename from test/fixtures/httpoison_mock/sakamoto_eal_feed.atom rename to test/fixtures/tesla_mock/sakamoto_eal_feed.atom diff --git a/test/fixtures/httpoison_mock/shitposter.club_host_meta b/test/fixtures/tesla_mock/shitposter.club_host_meta similarity index 100% rename from test/fixtures/httpoison_mock/shitposter.club_host_meta rename to test/fixtures/tesla_mock/shitposter.club_host_meta diff --git a/test/fixtures/httpoison_mock/shp@pleroma.soykaf.com.feed b/test/fixtures/tesla_mock/shp@pleroma.soykaf.com.feed similarity index 100% rename from test/fixtures/httpoison_mock/shp@pleroma.soykaf.com.feed rename to test/fixtures/tesla_mock/shp@pleroma.soykaf.com.feed diff --git a/test/fixtures/httpoison_mock/shp@pleroma.soykaf.com.webfigner b/test/fixtures/tesla_mock/shp@pleroma.soykaf.com.webfigner similarity index 100% rename from test/fixtures/httpoison_mock/shp@pleroma.soykaf.com.webfigner rename to test/fixtures/tesla_mock/shp@pleroma.soykaf.com.webfigner diff --git a/test/fixtures/httpoison_mock/shp@social.heldscal.la.xml b/test/fixtures/tesla_mock/shp@social.heldscal.la.xml similarity index 100% rename from test/fixtures/httpoison_mock/shp@social.heldscal.la.xml rename to test/fixtures/tesla_mock/shp@social.heldscal.la.xml diff --git a/test/fixtures/httpoison_mock/skruyb@mamot.fr.atom b/test/fixtures/tesla_mock/skruyb@mamot.fr.atom similarity index 100% rename from test/fixtures/httpoison_mock/skruyb@mamot.fr.atom rename to test/fixtures/tesla_mock/skruyb@mamot.fr.atom diff --git a/test/fixtures/httpoison_mock/social.heldscal.la_host_meta b/test/fixtures/tesla_mock/social.heldscal.la_host_meta similarity index 100% rename from test/fixtures/httpoison_mock/social.heldscal.la_host_meta rename to test/fixtures/tesla_mock/social.heldscal.la_host_meta diff --git a/test/fixtures/httpoison_mock/social.sakamoto.gq_host_meta b/test/fixtures/tesla_mock/social.sakamoto.gq_host_meta similarity index 100% rename from test/fixtures/httpoison_mock/social.sakamoto.gq_host_meta rename to test/fixtures/tesla_mock/social.sakamoto.gq_host_meta diff --git a/test/fixtures/httpoison_mock/social.stopwatchingus-heidelberg.de_host_meta b/test/fixtures/tesla_mock/social.stopwatchingus-heidelberg.de_host_meta similarity index 100% rename from test/fixtures/httpoison_mock/social.stopwatchingus-heidelberg.de_host_meta rename to test/fixtures/tesla_mock/social.stopwatchingus-heidelberg.de_host_meta diff --git a/test/fixtures/httpoison_mock/social.wxcafe.net_host_meta b/test/fixtures/tesla_mock/social.wxcafe.net_host_meta similarity index 100% rename from test/fixtures/httpoison_mock/social.wxcafe.net_host_meta rename to test/fixtures/tesla_mock/social.wxcafe.net_host_meta diff --git a/test/fixtures/httpoison_mock/spc_5381.atom b/test/fixtures/tesla_mock/spc_5381.atom similarity index 100% rename from test/fixtures/httpoison_mock/spc_5381.atom rename to test/fixtures/tesla_mock/spc_5381.atom diff --git a/test/fixtures/httpoison_mock/spc_5381_xrd.xml b/test/fixtures/tesla_mock/spc_5381_xrd.xml similarity index 100% rename from test/fixtures/httpoison_mock/spc_5381_xrd.xml rename to test/fixtures/tesla_mock/spc_5381_xrd.xml diff --git a/test/fixtures/httpoison_mock/squeet.me_host_meta b/test/fixtures/tesla_mock/squeet.me_host_meta similarity index 100% rename from test/fixtures/httpoison_mock/squeet.me_host_meta rename to test/fixtures/tesla_mock/squeet.me_host_meta diff --git a/test/fixtures/httpoison_mock/status.alpicola.com_host_meta b/test/fixtures/tesla_mock/status.alpicola.com_host_meta similarity index 100% rename from test/fixtures/httpoison_mock/status.alpicola.com_host_meta rename to test/fixtures/tesla_mock/status.alpicola.com_host_meta diff --git a/test/fixtures/httpoison_mock/status.emelie.json b/test/fixtures/tesla_mock/status.emelie.json similarity index 100% rename from test/fixtures/httpoison_mock/status.emelie.json rename to test/fixtures/tesla_mock/status.emelie.json diff --git a/test/fixtures/httpoison_mock/webfinger_emelie.json b/test/fixtures/tesla_mock/webfinger_emelie.json similarity index 100% rename from test/fixtures/httpoison_mock/webfinger_emelie.json rename to test/fixtures/tesla_mock/webfinger_emelie.json diff --git a/test/fixtures/httpoison_mock/winterdienst_webfinger.json b/test/fixtures/tesla_mock/winterdienst_webfinger.json similarity index 100% rename from test/fixtures/httpoison_mock/winterdienst_webfinger.json rename to test/fixtures/tesla_mock/winterdienst_webfinger.json diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex index 30169edb0..e6f357412 100644 --- a/test/support/http_request_mock.ex +++ b/test/support/http_request_mock.ex @@ -31,8 +31,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: - File.read!("test/fixtures/httpoison_mock/https___osada.macgirvin.com_channel_mike.json") + body: File.read!("test/fixtures/tesla_mock/https___osada.macgirvin.com_channel_mike.json") }} end @@ -40,7 +39,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/status.emelie.json") + body: File.read!("test/fixtures/tesla_mock/status.emelie.json") }} end @@ -48,7 +47,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/emelie.json") + body: File.read!("test/fixtures/tesla_mock/emelie.json") }} end @@ -56,7 +55,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/rinpatch.json") + body: File.read!("test/fixtures/tesla_mock/rinpatch.json") }} end @@ -69,7 +68,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/webfinger_emelie.json") + body: File.read!("test/fixtures/tesla_mock/webfinger_emelie.json") }} end @@ -77,7 +76,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/emelie.atom") + body: File.read!("test/fixtures/tesla_mock/emelie.atom") }} end @@ -90,7 +89,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/mike@osada.macgirvin.com.json") + body: File.read!("test/fixtures/tesla_mock/mike@osada.macgirvin.com.json") }} end @@ -103,7 +102,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/https___social.heldscal.la_user_29191.xml") + body: File.read!("test/fixtures/tesla_mock/https___social.heldscal.la_user_29191.xml") }} end @@ -111,7 +110,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/https___pawoo.net_users_pekorino.atom") + body: File.read!("test/fixtures/tesla_mock/https___pawoo.net_users_pekorino.atom") }} end @@ -124,7 +123,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/https___pawoo.net_users_pekorino.xml") + body: File.read!("test/fixtures/tesla_mock/https___pawoo.net_users_pekorino.xml") }} end @@ -137,7 +136,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/atarifrosch_feed.xml") + body: File.read!("test/fixtures/tesla_mock/atarifrosch_feed.xml") }} end @@ -150,7 +149,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/atarifrosch_webfinger.xml") + body: File.read!("test/fixtures/tesla_mock/atarifrosch_webfinger.xml") }} end @@ -158,7 +157,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/https___mamot.fr_users_Skruyb.atom") + body: File.read!("test/fixtures/tesla_mock/https___mamot.fr_users_Skruyb.atom") }} end @@ -171,7 +170,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/skruyb@mamot.fr.atom") + body: File.read!("test/fixtures/tesla_mock/skruyb@mamot.fr.atom") }} end @@ -184,7 +183,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/nonexistant@social.heldscal.la.xml") + body: File.read!("test/fixtures/tesla_mock/nonexistant@social.heldscal.la.xml") }} end @@ -197,7 +196,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/lain_squeet.me_webfinger.xml") + body: File.read!("test/fixtures/tesla_mock/lain_squeet.me_webfinger.xml") }} end @@ -210,7 +209,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/lucifermysticus.json") + body: File.read!("test/fixtures/tesla_mock/lucifermysticus.json") }} end @@ -218,7 +217,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/https___prismo.news__mxb.json") + body: File.read!("test/fixtures/tesla_mock/https___prismo.news__mxb.json") }} end @@ -231,7 +230,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/kaniini@hubzilla.example.org.json") + body: File.read!("test/fixtures/tesla_mock/kaniini@hubzilla.example.org.json") }} end @@ -239,7 +238,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/rye.json") + body: File.read!("test/fixtures/tesla_mock/rye.json") }} end @@ -247,7 +246,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/rye.json") + body: File.read!("test/fixtures/tesla_mock/rye.json") }} end @@ -257,7 +256,7 @@ defmodule HttpRequestMock do status: 200, body: File.read!( - "test/fixtures/httpoison_mock/http___mastodon.example.org_users_admin_status_1234.json" + "test/fixtures/tesla_mock/http___mastodon.example.org_users_admin_status_1234.json" ) }} end @@ -266,7 +265,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/puckipedia.com.json") + body: File.read!("test/fixtures/tesla_mock/puckipedia.com.json") }} end @@ -274,7 +273,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/7even.json") + body: File.read!("test/fixtures/tesla_mock/7even.json") }} end @@ -282,7 +281,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/peertube.moe-vid.json") + body: File.read!("test/fixtures/tesla_mock/peertube.moe-vid.json") }} end @@ -290,7 +289,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/baptiste.gelex.xyz-user.json") + body: File.read!("test/fixtures/tesla_mock/baptiste.gelex.xyz-user.json") }} end @@ -298,7 +297,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/baptiste.gelex.xyz-article.json") + body: File.read!("test/fixtures/tesla_mock/baptiste.gelex.xyz-article.json") }} end @@ -306,7 +305,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/admin@mastdon.example.org.json") + body: File.read!("test/fixtures/tesla_mock/admin@mastdon.example.org.json") }} end @@ -331,7 +330,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/7369654.html") + body: File.read!("test/fixtures/tesla_mock/7369654.html") }} end @@ -339,7 +338,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/mayumayu.json") + body: File.read!("test/fixtures/tesla_mock/mayumayu.json") }} end @@ -352,7 +351,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/mayumayupost.json") + body: File.read!("test/fixtures/tesla_mock/mayumayupost.json") }} end @@ -362,7 +361,7 @@ defmodule HttpRequestMock do status: 200, body: File.read!( - "test/fixtures/httpoison_mock/https___pleroma.soykaf.com_users_lain_feed.atom.xml" + "test/fixtures/tesla_mock/https___pleroma.soykaf.com_users_lain_feed.atom.xml" ) }} end @@ -375,7 +374,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/https___pleroma.soykaf.com_users_lain.xml") + body: File.read!("test/fixtures/tesla_mock/https___pleroma.soykaf.com_users_lain.xml") }} end @@ -385,7 +384,7 @@ defmodule HttpRequestMock do status: 200, body: File.read!( - "test/fixtures/httpoison_mock/https___shitposter.club_api_statuses_user_timeline_1.atom.xml" + "test/fixtures/tesla_mock/https___shitposter.club_api_statuses_user_timeline_1.atom.xml" ) }} end @@ -399,7 +398,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/https___shitposter.club_user_1.xml") + body: File.read!("test/fixtures/tesla_mock/https___shitposter.club_user_1.xml") }} end @@ -407,8 +406,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: - File.read!("test/fixtures/httpoison_mock/https___shitposter.club_notice_2827873.html") + body: File.read!("test/fixtures/tesla_mock/https___shitposter.club_notice_2827873.html") }} end @@ -418,7 +416,7 @@ defmodule HttpRequestMock do status: 200, body: File.read!( - "test/fixtures/httpoison_mock/https___shitposter.club_api_statuses_show_2827873.atom.xml" + "test/fixtures/tesla_mock/https___shitposter.club_api_statuses_show_2827873.atom.xml" ) }} end @@ -431,7 +429,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/spc_5381.atom") + body: File.read!("test/fixtures/tesla_mock/spc_5381.atom") }} end @@ -444,7 +442,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/spc_5381_xrd.xml") + body: File.read!("test/fixtures/tesla_mock/spc_5381_xrd.xml") }} end @@ -452,7 +450,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/shitposter.club_host_meta") + body: File.read!("test/fixtures/tesla_mock/shitposter.club_host_meta") }} end @@ -460,7 +458,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/7369654.atom") + body: File.read!("test/fixtures/tesla_mock/7369654.atom") }} end @@ -468,7 +466,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/7369654.html") + body: File.read!("test/fixtures/tesla_mock/7369654.html") }} end @@ -476,7 +474,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/sakamoto_eal_feed.atom") + body: File.read!("test/fixtures/tesla_mock/sakamoto_eal_feed.atom") }} end @@ -484,7 +482,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/social.sakamoto.gq_host_meta") + body: File.read!("test/fixtures/tesla_mock/social.sakamoto.gq_host_meta") }} end @@ -497,7 +495,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/eal_sakamoto.xml") + body: File.read!("test/fixtures/tesla_mock/eal_sakamoto.xml") }} end @@ -507,14 +505,14 @@ defmodule HttpRequestMock do _, Accept: "application/atom+xml" ) do - {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/httpoison_mock/sakamoto.atom")}} + {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/sakamoto.atom")}} end def get("http://mastodon.social/.well-known/host-meta", _, _, _) do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/mastodon.social_host_meta") + body: File.read!("test/fixtures/tesla_mock/mastodon.social_host_meta") }} end @@ -528,9 +526,7 @@ defmodule HttpRequestMock do %Tesla.Env{ status: 200, body: - File.read!( - "test/fixtures/httpoison_mock/https___mastodon.social_users_lambadalambda.xml" - ) + File.read!("test/fixtures/tesla_mock/https___mastodon.social_users_lambadalambda.xml") }} end @@ -538,7 +534,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/gs.example.org_host_meta") + body: File.read!("test/fixtures/tesla_mock/gs.example.org_host_meta") }} end @@ -552,9 +548,7 @@ defmodule HttpRequestMock do %Tesla.Env{ status: 200, body: - File.read!( - "test/fixtures/httpoison_mock/http___gs.example.org_4040_index.php_user_1.xml" - ) + File.read!("test/fixtures/tesla_mock/http___gs.example.org_4040_index.php_user_1.xml") }} end @@ -573,7 +567,7 @@ defmodule HttpRequestMock do status: 200, body: File.read!( - "test/fixtures/httpoison_mock/http__gs.example.org_index.php_api_statuses_user_timeline_1.atom.xml" + "test/fixtures/tesla_mock/http__gs.example.org_index.php_api_statuses_user_timeline_1.atom.xml" ) }} end @@ -584,14 +578,14 @@ defmodule HttpRequestMock do status: 200, body: File.read!( - "test/fixtures/httpoison_mock/https___social.heldscal.la_api_statuses_user_timeline_29191.atom.xml" + "test/fixtures/tesla_mock/https___social.heldscal.la_api_statuses_user_timeline_29191.atom.xml" ) }} end def get("http://squeet.me/.well-known/host-meta", _, _, _) do {:ok, - %Tesla.Env{status: 200, body: File.read!("test/fixtures/httpoison_mock/squeet.me_host_meta")}} + %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/squeet.me_host_meta")}} end def get( @@ -603,7 +597,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/lain_squeet.me_webfinger.xml") + body: File.read!("test/fixtures/tesla_mock/lain_squeet.me_webfinger.xml") }} end @@ -616,7 +610,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/shp@social.heldscal.la.xml") + body: File.read!("test/fixtures/tesla_mock/shp@social.heldscal.la.xml") }} end @@ -624,7 +618,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/framatube.org_host_meta") + body: File.read!("test/fixtures/tesla_mock/framatube.org_host_meta") }} end @@ -638,7 +632,7 @@ defmodule HttpRequestMock do %Tesla.Env{ status: 200, headers: [{"content-type", "application/json"}], - body: File.read!("test/fixtures/httpoison_mock/framasoft@framatube.org.json") + body: File.read!("test/fixtures/tesla_mock/framasoft@framatube.org.json") }} end @@ -646,7 +640,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/gnusocial.de_host_meta") + body: File.read!("test/fixtures/tesla_mock/gnusocial.de_host_meta") }} end @@ -659,7 +653,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/winterdienst_webfinger.json") + body: File.read!("test/fixtures/tesla_mock/winterdienst_webfinger.json") }} end @@ -667,7 +661,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/status.alpicola.com_host_meta") + body: File.read!("test/fixtures/tesla_mock/status.alpicola.com_host_meta") }} end @@ -675,7 +669,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/macgirvin.com_host_meta") + body: File.read!("test/fixtures/tesla_mock/macgirvin.com_host_meta") }} end @@ -683,7 +677,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/gerzilla.de_host_meta") + body: File.read!("test/fixtures/tesla_mock/gerzilla.de_host_meta") }} end @@ -697,7 +691,7 @@ defmodule HttpRequestMock do %Tesla.Env{ status: 200, headers: [{"content-type", "application/json"}], - body: File.read!("test/fixtures/httpoison_mock/kaniini@gerzilla.de.json") + body: File.read!("test/fixtures/tesla_mock/kaniini@gerzilla.de.json") }} end @@ -707,7 +701,7 @@ defmodule HttpRequestMock do status: 200, body: File.read!( - "test/fixtures/httpoison_mock/https___social.heldscal.la_api_statuses_user_timeline_23211.atom.xml" + "test/fixtures/tesla_mock/https___social.heldscal.la_api_statuses_user_timeline_23211.atom.xml" ) }} end @@ -721,7 +715,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/https___social.heldscal.la_user_23211.xml") + body: File.read!("test/fixtures/tesla_mock/https___social.heldscal.la_user_23211.xml") }} end @@ -729,7 +723,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/social.heldscal.la_host_meta") + body: File.read!("test/fixtures/tesla_mock/social.heldscal.la_host_meta") }} end @@ -737,7 +731,7 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{ status: 200, - body: File.read!("test/fixtures/httpoison_mock/social.heldscal.la_host_meta") + body: File.read!("test/fixtures/tesla_mock/social.heldscal.la_host_meta") }} end From 5184b0f41aabb79a47e8d3b8c76b88ca04a53b24 Mon Sep 17 00:00:00 2001 From: Eugenij Date: Wed, 3 Jul 2019 10:19:51 +0000 Subject: [PATCH 16/43] Use fallback values for search queries This is to make sure the entire request doesn't return a 500 error if user or status search times out. --- CHANGELOG.md | 1 + .../web/mastodon_api/search_controller.ex | 18 ++++++++++++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a0dad453..aefc09177 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Fixed - Not being able to pin unlisted posts +- Mastodon API: Handling of search timeouts (`/api/v1/search` and `/api/v2/search`) ### Changed - Configuration: Filter.AnonymizeFilename added ability to retain file extension with custom text diff --git a/lib/pleroma/web/mastodon_api/search_controller.ex b/lib/pleroma/web/mastodon_api/search_controller.ex index 0d1e2355d..efa9cc788 100644 --- a/lib/pleroma/web/mastodon_api/search_controller.ex +++ b/lib/pleroma/web/mastodon_api/search_controller.ex @@ -17,8 +17,8 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do plug(Pleroma.Plugs.RateLimiter, :search when action in [:search, :search2, :account_search]) def search2(%{assigns: %{user: user}} = conn, %{"q" => query} = params) do - accounts = User.search(query, search_options(params, user)) - statuses = Activity.search(user, query) + accounts = with_fallback(fn -> User.search(query, search_options(params, user)) end, []) + statuses = with_fallback(fn -> Activity.search(user, query) end, []) tags_path = Web.base_url() <> "/tag/" tags = @@ -40,8 +40,8 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do end def search(%{assigns: %{user: user}} = conn, %{"q" => query} = params) do - accounts = User.search(query, search_options(params, user)) - statuses = Activity.search(user, query) + accounts = with_fallback(fn -> User.search(query, search_options(params, user)) end, []) + statuses = with_fallback(fn -> Activity.search(user, query) end, []) tags = query @@ -76,4 +76,14 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do for_user: user ] end + + defp with_fallback(f, fallback) do + try do + f.() + rescue + error -> + Logger.error("#{__MODULE__} search error: #{inspect(error)}") + fallback + end + end end From efefee82415d872209cf5884375891e577f54781 Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Wed, 3 Jul 2019 14:56:02 +0300 Subject: [PATCH 17/43] Split alters rather than work with indexes --- .../20170522160642_case_insensivtivity.exs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/priv/repo/migrations/20170522160642_case_insensivtivity.exs b/priv/repo/migrations/20170522160642_case_insensivtivity.exs index 470a545e5..9a67727e9 100644 --- a/priv/repo/migrations/20170522160642_case_insensivtivity.exs +++ b/priv/repo/migrations/20170522160642_case_insensivtivity.exs @@ -1,22 +1,28 @@ defmodule Pleroma.Repo.Migrations.CaseInsensivtivity do use Ecto.Migration + # Two-steps alters are intentional. + # When alter of 2 columns is done in a single operation, + # inconsistent failures happen because of index on `email` column. + def up do execute("create extension if not exists citext") - drop_if_exists(index(:users, [:email])) - alter table(:users) do modify(:email, :citext) - modify(:nickname, :citext) end - create_if_not_exists(index(:users, [:email])) + alter table(:users) do + modify(:nickname, :citext) + end end def down do alter table(:users) do modify(:email, :string) + end + + alter table(:users) do modify(:nickname, :string) end From 15eb1235ed54b65d79a6e955be7b4302993bd620 Mon Sep 17 00:00:00 2001 From: Eugenij Date: Wed, 3 Jul 2019 18:35:57 +0000 Subject: [PATCH 18/43] Expose `skip_thread_containment` in nodeinfo --- CHANGELOG.md | 3 +++ lib/pleroma/web/nodeinfo/nodeinfo_controller.ex | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aefc09177..3dbbd8225 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Changed - Configuration: Filter.AnonymizeFilename added ability to retain file extension with custom text +### Changed +- NodeInfo: Return `skipThreadContainment` in `metadata` for the `skip_thread_containment` option + ## [1.0.0] - 2019-06-29 ### Security - Mastodon API: Fix display names not being sanitized diff --git a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex index 32be430b7..869dda5c5 100644 --- a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex +++ b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex @@ -162,7 +162,8 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do accountActivationRequired: Config.get([:instance, :account_activation_required], false), invitesEnabled: Config.get([:instance, :invites_enabled], false), features: features, - restrictedNicknames: Config.get([Pleroma.User, :restricted_nicknames]) + restrictedNicknames: Config.get([Pleroma.User, :restricted_nicknames]), + skipThreadContainment: Config.get([:instance, :skip_thread_containment], false) } } end From 977c2d044810d71fa89f9976ef342b6f708a027a Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 5 Jul 2019 06:19:27 +0200 Subject: [PATCH 19/43] tasks/pleroma/instance.ex: Change :upload_dir to :uploads_dir Closes: https://git.pleroma.social/pleroma/pleroma/issues/1058 --- lib/mix/tasks/pleroma/instance.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mix/tasks/pleroma/instance.ex b/lib/mix/tasks/pleroma/instance.ex index a27c4b897..2ae16adc0 100644 --- a/lib/mix/tasks/pleroma/instance.ex +++ b/lib/mix/tasks/pleroma/instance.ex @@ -149,7 +149,7 @@ defmodule Mix.Tasks.Pleroma.Instance do uploads_dir = get_option( options, - :upload_dir, + :uploads_dir, "What directory should media uploads go in (when using the local uploader)?", Pleroma.Config.get([Pleroma.Uploaders.Local, :uploads]) ) From 6dadf5d6f41dda8d0f760da25f4394eecf467a80 Mon Sep 17 00:00:00 2001 From: Maksim Date: Fri, 5 Jul 2019 04:22:08 +0000 Subject: [PATCH 20/43] [#1043] fix search accounts. --- lib/pleroma/user.ex | 11 +- lib/pleroma/user/search.ex | 21 ++++ test/support/helpers.ex | 6 + test/user_search_test.exs | 221 +++++++++++++++++++++++++++++++++++++ test/user_test.exs | 183 ------------------------------ 5 files changed, 252 insertions(+), 190 deletions(-) create mode 100644 test/user_search_test.exs diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index f7191762f..09f86aaa2 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -836,15 +836,12 @@ defmodule Pleroma.User do def mutes?(nil, _), do: false def mutes?(user, %{ap_id: ap_id}), do: Enum.member?(user.info.mutes, ap_id) - def blocks?(user, %{ap_id: ap_id}) do - blocks = user.info.blocks - domain_blocks = user.info.domain_blocks + def blocks?(%User{info: info} = _user, %{ap_id: ap_id}) do + blocks = info.blocks + domain_blocks = info.domain_blocks %{host: host} = URI.parse(ap_id) - Enum.member?(blocks, ap_id) || - Enum.any?(domain_blocks, fn domain -> - host == domain - end) + Enum.member?(blocks, ap_id) || Enum.any?(domain_blocks, &(&1 == host)) end def subscribed_to?(user, %{ap_id: ap_id}) do diff --git a/lib/pleroma/user/search.ex b/lib/pleroma/user/search.ex index ed06c2ab9..7680c2afd 100644 --- a/lib/pleroma/user/search.ex +++ b/lib/pleroma/user/search.ex @@ -43,6 +43,8 @@ defmodule Pleroma.User.Search do defp search_query(query_string, for_user, following) do for_user |> base_query(following) + |> filter_blocked_user(for_user) + |> filter_blocked_domains(for_user) |> search_subqueries(query_string) |> union_subqueries |> distinct_query() @@ -55,6 +57,25 @@ defmodule Pleroma.User.Search do defp base_query(_user, false), do: User defp base_query(user, true), do: User.get_followers_query(user) + defp filter_blocked_user(query, %User{info: %{blocks: blocks}}) + when length(blocks) > 0 do + from(q in query, where: not (q.ap_id in ^blocks)) + end + + defp filter_blocked_user(query, _), do: query + + defp filter_blocked_domains(query, %User{info: %{domain_blocks: domain_blocks}}) + when length(domain_blocks) > 0 do + domains = Enum.join(domain_blocks, ",") + + from( + q in query, + where: fragment("substring(ap_id from '.*://([^/]*)') NOT IN (?)", ^domains) + ) + end + + defp filter_blocked_domains(query, _), do: query + defp paginate(query, limit, offset) do from(q in query, limit: ^limit, offset: ^offset) end diff --git a/test/support/helpers.ex b/test/support/helpers.ex index 6e389ce52..1a92be065 100644 --- a/test/support/helpers.ex +++ b/test/support/helpers.ex @@ -9,6 +9,12 @@ defmodule Pleroma.Tests.Helpers do defmacro __using__(_opts) do quote do + def collect_ids(collection) do + collection + |> Enum.map(& &1.id) + |> Enum.sort() + end + def refresh_record(%{id: id, __struct__: model} = _), do: refresh_record(model, %{id: id}) diff --git a/test/user_search_test.exs b/test/user_search_test.exs new file mode 100644 index 000000000..8f8472aae --- /dev/null +++ b/test/user_search_test.exs @@ -0,0 +1,221 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.UserSearchTest do + alias Pleroma.Repo + alias Pleroma.User + use Pleroma.DataCase + + import Pleroma.Factory + + setup_all do + Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) + :ok + end + + describe "User.search" do + test "accepts limit parameter" do + Enum.each(0..4, &insert(:user, %{nickname: "john#{&1}"})) + assert length(User.search("john", limit: 3)) == 3 + assert length(User.search("john")) == 5 + end + + test "accepts offset parameter" do + Enum.each(0..4, &insert(:user, %{nickname: "john#{&1}"})) + assert length(User.search("john", limit: 3)) == 3 + assert length(User.search("john", limit: 3, offset: 3)) == 2 + end + + test "finds a user by full or partial nickname" do + user = insert(:user, %{nickname: "john"}) + + Enum.each(["john", "jo", "j"], fn query -> + assert user == + User.search(query) + |> List.first() + |> Map.put(:search_rank, nil) + |> Map.put(:search_type, nil) + end) + end + + test "finds a user by full or partial name" do + user = insert(:user, %{name: "John Doe"}) + + Enum.each(["John Doe", "JOHN", "doe", "j d", "j", "d"], fn query -> + assert user == + User.search(query) + |> List.first() + |> Map.put(:search_rank, nil) + |> Map.put(:search_type, nil) + end) + end + + test "finds users, preferring nickname matches over name matches" do + u1 = insert(:user, %{name: "lain", nickname: "nick1"}) + u2 = insert(:user, %{nickname: "lain", name: "nick1"}) + + assert [u2.id, u1.id] == Enum.map(User.search("lain"), & &1.id) + end + + test "finds users, considering density of matched tokens" do + u1 = insert(:user, %{name: "Bar Bar plus Word Word"}) + u2 = insert(:user, %{name: "Word Word Bar Bar Bar"}) + + assert [u2.id, u1.id] == Enum.map(User.search("bar word"), & &1.id) + end + + test "finds users, ranking by similarity" do + u1 = insert(:user, %{name: "lain"}) + _u2 = insert(:user, %{name: "ean"}) + u3 = insert(:user, %{name: "ebn", nickname: "lain@mastodon.social"}) + u4 = insert(:user, %{nickname: "lain@pleroma.soykaf.com"}) + + assert [u4.id, u3.id, u1.id] == Enum.map(User.search("lain@ple", for_user: u1), & &1.id) + end + + test "finds users, handling misspelled requests" do + u1 = insert(:user, %{name: "lain"}) + + assert [u1.id] == Enum.map(User.search("laiin"), & &1.id) + end + + test "finds users, boosting ranks of friends and followers" do + u1 = insert(:user) + u2 = insert(:user, %{name: "Doe"}) + follower = insert(:user, %{name: "Doe"}) + friend = insert(:user, %{name: "Doe"}) + + {:ok, follower} = User.follow(follower, u1) + {:ok, u1} = User.follow(u1, friend) + + assert [friend.id, follower.id, u2.id] -- + Enum.map(User.search("doe", resolve: false, for_user: u1), & &1.id) == [] + end + + test "finds followers of user by partial name" do + u1 = insert(:user) + u2 = insert(:user, %{name: "Jimi"}) + follower_jimi = insert(:user, %{name: "Jimi Hendrix"}) + follower_lizz = insert(:user, %{name: "Lizz Wright"}) + friend = insert(:user, %{name: "Jimi"}) + + {:ok, follower_jimi} = User.follow(follower_jimi, u1) + {:ok, _follower_lizz} = User.follow(follower_lizz, u2) + {:ok, u1} = User.follow(u1, friend) + + assert Enum.map(User.search("jimi", following: true, for_user: u1), & &1.id) == [ + follower_jimi.id + ] + + assert User.search("lizz", following: true, for_user: u1) == [] + end + + test "find local and remote users for authenticated users" do + u1 = insert(:user, %{name: "lain"}) + u2 = insert(:user, %{name: "ebn", nickname: "lain@mastodon.social", local: false}) + u3 = insert(:user, %{nickname: "lain@pleroma.soykaf.com", local: false}) + + results = + "lain" + |> User.search(for_user: u1) + |> Enum.map(& &1.id) + |> Enum.sort() + + assert [u1.id, u2.id, u3.id] == results + end + + test "find only local users for unauthenticated users" do + %{id: id} = insert(:user, %{name: "lain"}) + insert(:user, %{name: "ebn", nickname: "lain@mastodon.social", local: false}) + insert(:user, %{nickname: "lain@pleroma.soykaf.com", local: false}) + + assert [%{id: ^id}] = User.search("lain") + end + + test "find only local users for authenticated users when `limit_to_local_content` is `:all`" do + Pleroma.Config.put([:instance, :limit_to_local_content], :all) + + %{id: id} = insert(:user, %{name: "lain"}) + insert(:user, %{name: "ebn", nickname: "lain@mastodon.social", local: false}) + insert(:user, %{nickname: "lain@pleroma.soykaf.com", local: false}) + + assert [%{id: ^id}] = User.search("lain") + + Pleroma.Config.put([:instance, :limit_to_local_content], :unauthenticated) + end + + test "find all users for unauthenticated users when `limit_to_local_content` is `false`" do + Pleroma.Config.put([:instance, :limit_to_local_content], false) + + u1 = insert(:user, %{name: "lain"}) + u2 = insert(:user, %{name: "ebn", nickname: "lain@mastodon.social", local: false}) + u3 = insert(:user, %{nickname: "lain@pleroma.soykaf.com", local: false}) + + results = + "lain" + |> User.search() + |> Enum.map(& &1.id) + |> Enum.sort() + + assert [u1.id, u2.id, u3.id] == results + + Pleroma.Config.put([:instance, :limit_to_local_content], :unauthenticated) + end + + test "finds a user whose name is nil" do + _user = insert(:user, %{name: "notamatch", nickname: "testuser@pleroma.amplifie.red"}) + user_two = insert(:user, %{name: nil, nickname: "lain@pleroma.soykaf.com"}) + + assert user_two == + User.search("lain@pleroma.soykaf.com") + |> List.first() + |> Map.put(:search_rank, nil) + |> Map.put(:search_type, nil) + end + + test "does not yield false-positive matches" do + insert(:user, %{name: "John Doe"}) + + Enum.each(["mary", "a", ""], fn query -> + assert [] == User.search(query) + end) + end + + test "works with URIs" do + user = insert(:user) + + results = + User.search("http://mastodon.example.org/users/admin", resolve: true, for_user: user) + + result = results |> List.first() + + user = User.get_cached_by_ap_id("http://mastodon.example.org/users/admin") + + assert length(results) == 1 + assert user == result |> Map.put(:search_rank, nil) |> Map.put(:search_type, nil) + end + + test "excludes a blocked users from search result" do + user = insert(:user, %{nickname: "Bill"}) + + [blocked_user | users] = Enum.map(0..3, &insert(:user, %{nickname: "john#{&1}"})) + + blocked_user2 = + insert( + :user, + %{nickname: "john awful", ap_id: "https://awful-and-rude-instance.com/user/bully"} + ) + + User.block_domain(user, "awful-and-rude-instance.com") + User.block(user, blocked_user) + + account_ids = User.search("john", for_user: refresh_record(user)) |> collect_ids + + assert account_ids == collect_ids(users) + refute Enum.member?(account_ids, blocked_user.id) + refute Enum.member?(account_ids, blocked_user2.id) + assert length(account_ids) == 3 + end + end +end diff --git a/test/user_test.exs b/test/user_test.exs index 198a97fae..fb497843c 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -1012,189 +1012,6 @@ defmodule Pleroma.UserTest do end end - describe "User.search" do - test "accepts limit parameter" do - Enum.each(0..4, &insert(:user, %{nickname: "john#{&1}"})) - assert length(User.search("john", limit: 3)) == 3 - assert length(User.search("john")) == 5 - end - - test "accepts offset parameter" do - Enum.each(0..4, &insert(:user, %{nickname: "john#{&1}"})) - assert length(User.search("john", limit: 3)) == 3 - assert length(User.search("john", limit: 3, offset: 3)) == 2 - end - - test "finds a user by full or partial nickname" do - user = insert(:user, %{nickname: "john"}) - - Enum.each(["john", "jo", "j"], fn query -> - assert user == - User.search(query) - |> List.first() - |> Map.put(:search_rank, nil) - |> Map.put(:search_type, nil) - end) - end - - test "finds a user by full or partial name" do - user = insert(:user, %{name: "John Doe"}) - - Enum.each(["John Doe", "JOHN", "doe", "j d", "j", "d"], fn query -> - assert user == - User.search(query) - |> List.first() - |> Map.put(:search_rank, nil) - |> Map.put(:search_type, nil) - end) - end - - test "finds users, preferring nickname matches over name matches" do - u1 = insert(:user, %{name: "lain", nickname: "nick1"}) - u2 = insert(:user, %{nickname: "lain", name: "nick1"}) - - assert [u2.id, u1.id] == Enum.map(User.search("lain"), & &1.id) - end - - test "finds users, considering density of matched tokens" do - u1 = insert(:user, %{name: "Bar Bar plus Word Word"}) - u2 = insert(:user, %{name: "Word Word Bar Bar Bar"}) - - assert [u2.id, u1.id] == Enum.map(User.search("bar word"), & &1.id) - end - - test "finds users, ranking by similarity" do - u1 = insert(:user, %{name: "lain"}) - _u2 = insert(:user, %{name: "ean"}) - u3 = insert(:user, %{name: "ebn", nickname: "lain@mastodon.social"}) - u4 = insert(:user, %{nickname: "lain@pleroma.soykaf.com"}) - - assert [u4.id, u3.id, u1.id] == Enum.map(User.search("lain@ple", for_user: u1), & &1.id) - end - - test "finds users, handling misspelled requests" do - u1 = insert(:user, %{name: "lain"}) - - assert [u1.id] == Enum.map(User.search("laiin"), & &1.id) - end - - test "finds users, boosting ranks of friends and followers" do - u1 = insert(:user) - u2 = insert(:user, %{name: "Doe"}) - follower = insert(:user, %{name: "Doe"}) - friend = insert(:user, %{name: "Doe"}) - - {:ok, follower} = User.follow(follower, u1) - {:ok, u1} = User.follow(u1, friend) - - assert [friend.id, follower.id, u2.id] -- - Enum.map(User.search("doe", resolve: false, for_user: u1), & &1.id) == [] - end - - test "finds followers of user by partial name" do - u1 = insert(:user) - u2 = insert(:user, %{name: "Jimi"}) - follower_jimi = insert(:user, %{name: "Jimi Hendrix"}) - follower_lizz = insert(:user, %{name: "Lizz Wright"}) - friend = insert(:user, %{name: "Jimi"}) - - {:ok, follower_jimi} = User.follow(follower_jimi, u1) - {:ok, _follower_lizz} = User.follow(follower_lizz, u2) - {:ok, u1} = User.follow(u1, friend) - - assert Enum.map(User.search("jimi", following: true, for_user: u1), & &1.id) == [ - follower_jimi.id - ] - - assert User.search("lizz", following: true, for_user: u1) == [] - end - - test "find local and remote users for authenticated users" do - u1 = insert(:user, %{name: "lain"}) - u2 = insert(:user, %{name: "ebn", nickname: "lain@mastodon.social", local: false}) - u3 = insert(:user, %{nickname: "lain@pleroma.soykaf.com", local: false}) - - results = - "lain" - |> User.search(for_user: u1) - |> Enum.map(& &1.id) - |> Enum.sort() - - assert [u1.id, u2.id, u3.id] == results - end - - test "find only local users for unauthenticated users" do - %{id: id} = insert(:user, %{name: "lain"}) - insert(:user, %{name: "ebn", nickname: "lain@mastodon.social", local: false}) - insert(:user, %{nickname: "lain@pleroma.soykaf.com", local: false}) - - assert [%{id: ^id}] = User.search("lain") - end - - test "find only local users for authenticated users when `limit_to_local_content` is `:all`" do - Pleroma.Config.put([:instance, :limit_to_local_content], :all) - - %{id: id} = insert(:user, %{name: "lain"}) - insert(:user, %{name: "ebn", nickname: "lain@mastodon.social", local: false}) - insert(:user, %{nickname: "lain@pleroma.soykaf.com", local: false}) - - assert [%{id: ^id}] = User.search("lain") - - Pleroma.Config.put([:instance, :limit_to_local_content], :unauthenticated) - end - - test "find all users for unauthenticated users when `limit_to_local_content` is `false`" do - Pleroma.Config.put([:instance, :limit_to_local_content], false) - - u1 = insert(:user, %{name: "lain"}) - u2 = insert(:user, %{name: "ebn", nickname: "lain@mastodon.social", local: false}) - u3 = insert(:user, %{nickname: "lain@pleroma.soykaf.com", local: false}) - - results = - "lain" - |> User.search() - |> Enum.map(& &1.id) - |> Enum.sort() - - assert [u1.id, u2.id, u3.id] == results - - Pleroma.Config.put([:instance, :limit_to_local_content], :unauthenticated) - end - - test "finds a user whose name is nil" do - _user = insert(:user, %{name: "notamatch", nickname: "testuser@pleroma.amplifie.red"}) - user_two = insert(:user, %{name: nil, nickname: "lain@pleroma.soykaf.com"}) - - assert user_two == - User.search("lain@pleroma.soykaf.com") - |> List.first() - |> Map.put(:search_rank, nil) - |> Map.put(:search_type, nil) - end - - test "does not yield false-positive matches" do - insert(:user, %{name: "John Doe"}) - - Enum.each(["mary", "a", ""], fn query -> - assert [] == User.search(query) - end) - end - - test "works with URIs" do - user = insert(:user) - - results = - User.search("http://mastodon.example.org/users/admin", resolve: true, for_user: user) - - result = results |> List.first() - - user = User.get_cached_by_ap_id("http://mastodon.example.org/users/admin") - - assert length(results) == 1 - assert user == result |> Map.put(:search_rank, nil) |> Map.put(:search_type, nil) - end - end - test "auth_active?/1 works correctly" do Pleroma.Config.put([:instance, :account_activation_required], true) From 6c50fbcd14b56665979199150659635575fd1b25 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Fri, 5 Jul 2019 19:33:53 +0300 Subject: [PATCH 21/43] Admin API: Allow querying user by ID --- CHANGELOG.md | 1 + docs/api/admin_api.md | 11 +++++++++++ lib/pleroma/web/admin_api/admin_api_controller.ex | 2 +- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3dbbd8225..86991efe9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Support for the [`tagged` filter](https://github.com/tootsuite/mastodon/pull/9755) in [`GET /api/v1/accounts/:id/statuses`](https://docs.joinmastodon.org/api/rest/accounts/#get-api-v1-accounts-id-statuses) - Admin API: Return users' tags when querying reports - Admin API: Return avatar and display name when querying users +- Admin API: Allow querying user by ID ### Fixed - Not being able to pin unlisted posts diff --git a/docs/api/admin_api.md b/docs/api/admin_api.md index 74bde3ece..02baa09ed 100644 --- a/docs/api/admin_api.md +++ b/docs/api/admin_api.md @@ -187,6 +187,17 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret - On failure: `Not found` - On success: JSON of the user +## `/api/pleroma/admin/users/:id` + +### Retrive the details of a user + +- Method: `GET` +- Params: + - `id` +- Response: + - On failure: `Not found` + - On success: JSON of the user + ## `/api/pleroma/admin/relay` ### Follow a Relay diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 498beb56a..0a2482a8c 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -74,7 +74,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do end def user_show(conn, %{"nickname" => nickname}) do - with %User{} = user <- User.get_cached_by_nickname(nickname) do + with %User{} = user <- User.get_cached_by_nickname_or_id(nickname) do conn |> json(AccountView.render("show.json", %{user: user})) else From e2f4135a8c5bc1f54be4287c64081c90dfea8125 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sat, 6 Jul 2019 07:18:26 +0000 Subject: [PATCH 22/43] Apply suggestion to CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 852e7e2c4..d16847f7c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [Unreleased] ### Added - MRF: Support for priming the mediaproxy cache (`Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy`) -- Federation: Support for restricting max. reply-to depth on fetching +Configuration: `federation_incoming_replies_max_depth` option - Mastodon API: Support for the [`tagged` filter](https://github.com/tootsuite/mastodon/pull/9755) in [`GET /api/v1/accounts/:id/statuses`](https://docs.joinmastodon.org/api/rest/accounts/#get-api-v1-accounts-id-statuses) - Admin API: Return users' tags when querying reports - Admin API: Return avatar and display name when querying users From 9f235028569968871ef9ea933459c6e9369e737a Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Sat, 6 Jul 2019 15:16:56 +0300 Subject: [PATCH 23/43] Fix docs --- docs/api/admin_api.md | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/docs/api/admin_api.md b/docs/api/admin_api.md index 02baa09ed..bce5e399b 100644 --- a/docs/api/admin_api.md +++ b/docs/api/admin_api.md @@ -176,24 +176,13 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret - `nickname` - `status` BOOLEAN field, false value means deactivation. -## `/api/pleroma/admin/users/:nickname` +## `/api/pleroma/admin/users/:nickname_or_id` ### Retrive the details of a user - Method: `GET` - Params: - - `nickname` -- Response: - - On failure: `Not found` - - On success: JSON of the user - -## `/api/pleroma/admin/users/:id` - -### Retrive the details of a user - -- Method: `GET` -- Params: - - `id` + - `nickname` or `id` - Response: - On failure: `Not found` - On success: JSON of the user From a7885748c7f37db232a097aa24132ed59229360c Mon Sep 17 00:00:00 2001 From: KokaKiwi Date: Sat, 22 Jun 2019 19:45:21 +0200 Subject: [PATCH 24/43] MastoAPI streaming: Replace access_token with Sec-WebSocket-Protocol --- lib/pleroma/web/mastodon_api/websocket_handler.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/websocket_handler.ex b/lib/pleroma/web/mastodon_api/websocket_handler.ex index 3299e1721..db6ae23b0 100644 --- a/lib/pleroma/web/mastodon_api/websocket_handler.ex +++ b/lib/pleroma/web/mastodon_api/websocket_handler.ex @@ -29,7 +29,7 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do def init(%{qs: qs} = req, state) do with params <- :cow_qs.parse_qs(qs), - access_token <- List.keyfind(params, "access_token", 0), + access_token <- :cowboy_req.header("sec-websocket-protocol", req, 0), {_, stream} <- List.keyfind(params, "stream", 0), {:ok, user} <- allow_request(stream, access_token), topic when is_binary(topic) <- expand_topic(stream, params) do @@ -89,7 +89,7 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do end # Authenticated streams. - defp allow_request(stream, {"access_token", access_token}) when stream in @streams do + defp allow_request(stream, access_token) when stream in @streams do with %Token{user_id: user_id} <- Repo.get_by(Token, token: access_token), user = %User{} <- User.get_cached_by_id(user_id) do {:ok, user} From e174614eb9d9ea465611aac694912bbdbaf2a03c Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Sat, 6 Jul 2019 20:26:08 +0200 Subject: [PATCH 25/43] MastoAPI Streaming: Keep compatibility with access_token --- CHANGELOG.md | 1 + .../web/mastodon_api/websocket_handler.ex | 19 ++++++++++++++----- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3dbbd8225..408085335 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Support for the [`tagged` filter](https://github.com/tootsuite/mastodon/pull/9755) in [`GET /api/v1/accounts/:id/statuses`](https://docs.joinmastodon.org/api/rest/accounts/#get-api-v1-accounts-id-statuses) - Admin API: Return users' tags when querying reports - Admin API: Return avatar and display name when querying users +- Mastodon API, streaming: Add support for passing the token in the `Sec-WebSocket-Protocol` header ### Fixed - Not being able to pin unlisted posts diff --git a/lib/pleroma/web/mastodon_api/websocket_handler.ex b/lib/pleroma/web/mastodon_api/websocket_handler.ex index db6ae23b0..dbd3542ea 100644 --- a/lib/pleroma/web/mastodon_api/websocket_handler.ex +++ b/lib/pleroma/web/mastodon_api/websocket_handler.ex @@ -29,9 +29,10 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do def init(%{qs: qs} = req, state) do with params <- :cow_qs.parse_qs(qs), - access_token <- :cowboy_req.header("sec-websocket-protocol", req, 0), + sec_websocket <- :cowboy_req.header("sec-websocket-protocol", req, nil), + access_token <- List.keyfind(params, "access_token", 0), {_, stream} <- List.keyfind(params, "stream", 0), - {:ok, user} <- allow_request(stream, access_token), + {:ok, user} <- allow_request(stream, [access_token, sec_websocket]), topic when is_binary(topic) <- expand_topic(stream, params) do {:cowboy_websocket, req, %{user: user, topic: topic}, %{idle_timeout: @timeout}} else @@ -84,13 +85,21 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do end # Public streams without authentication. - defp allow_request(stream, nil) when stream in @anonymous_streams do + defp allow_request(stream, [nil, nil]) when stream in @anonymous_streams do {:ok, nil} end # Authenticated streams. - defp allow_request(stream, access_token) when stream in @streams do - with %Token{user_id: user_id} <- Repo.get_by(Token, token: access_token), + defp allow_request(stream, [access_token, sec_websocket]) when stream in @streams do + token = + with {"access_token", token} <- access_token do + token + else + _ -> sec_websocket + end + + with true <- is_bitstring(token), + %Token{user_id: user_id} <- Repo.get_by(Token, token: token), user = %User{} <- User.get_cached_by_id(user_id) do {:ok, user} else From b5ba41a7255e23285810d865f0fef7701ab4ca6c Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Sun, 7 Jul 2019 08:58:24 +0200 Subject: [PATCH 26/43] mastodon_websocket_test.exs: Test for Sec-WebSocket-Protocol header --- test/integration/mastodon_websocket_test.exs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/integration/mastodon_websocket_test.exs b/test/integration/mastodon_websocket_test.exs index a604713d8..3975cdcd6 100644 --- a/test/integration/mastodon_websocket_test.exs +++ b/test/integration/mastodon_websocket_test.exs @@ -107,5 +107,12 @@ defmodule Pleroma.Integration.MastodonWebsocketTest do assert {:ok, _} = start_socket("?stream=user:notification&access_token=#{token.token}") assert {:error, {403, "Forbidden"}} = start_socket("?stream=user:notification") end + + test "accepts valid token on Sec-WebSocket-Protocol header", %{token: token} do + assert {:ok, _} = start_socket("?stream=user", [{"Sec-WebSocket-Protocol", token.token}]) + + assert {:error, {403, "Forbidden"}} = + start_socket("?stream=user", [{"Sec-WebSocket-Protocol", "I am a friend"}]) + end end end From f5ad4309747e85192e6034fd362103b0b71869d0 Mon Sep 17 00:00:00 2001 From: Sachin Joshi Date: Sun, 7 Jul 2019 14:13:40 +0545 Subject: [PATCH 27/43] make sure the url used by proxy is same as origin url encoding or decoding it breaks some of the signed url --- lib/pleroma/web/media_proxy/media_proxy.ex | 23 +--------------------- test/media_proxy_test.exs | 7 +++++-- 2 files changed, 6 insertions(+), 24 deletions(-) diff --git a/lib/pleroma/web/media_proxy/media_proxy.ex b/lib/pleroma/web/media_proxy/media_proxy.ex index cee6d8481..dd8888a02 100644 --- a/lib/pleroma/web/media_proxy/media_proxy.ex +++ b/lib/pleroma/web/media_proxy/media_proxy.ex @@ -33,20 +33,7 @@ defmodule Pleroma.Web.MediaProxy do def encode_url(url) do secret = Pleroma.Config.get([Pleroma.Web.Endpoint, :secret_key_base]) - - # Must preserve `%2F` for compatibility with S3 - # https://git.pleroma.social/pleroma/pleroma/issues/580 - replacement = get_replacement(url, ":2F:") - - # The URL is url-decoded and encoded again to ensure it is correctly encoded and not twice. - base64 = - url - |> String.replace("%2F", replacement) - |> URI.decode() - |> URI.encode() - |> String.replace(replacement, "%2F") - |> Base.url_encode64(@base64_opts) - + base64 = Base.url_encode64(url, @base64_opts) sig = :crypto.hmac(:sha, secret, base64) sig64 = sig |> Base.url_encode64(@base64_opts) @@ -80,12 +67,4 @@ defmodule Pleroma.Web.MediaProxy do |> Enum.filter(fn value -> value end) |> Path.join() end - - defp get_replacement(url, replacement) do - if String.contains?(url, replacement) do - get_replacement(url, replacement <> replacement) - else - replacement - end - end end diff --git a/test/media_proxy_test.exs b/test/media_proxy_test.exs index b23aeb88b..cd1cbd202 100644 --- a/test/media_proxy_test.exs +++ b/test/media_proxy_test.exs @@ -70,9 +70,12 @@ defmodule Pleroma.MediaProxyTest do assert decode_result(encoded) == url end - test "ensures urls are url-encoded" do + # Some of the signed url expect the special character in the url to be same + # for the proxy to work. + # Issue https://git.pleroma.social/pleroma/pleroma/issues/1055 + test "ensures urls are maintained (character are not encoded or decoded)" do assert decode_result(url("https://pleroma.social/Hello world.jpg")) == - "https://pleroma.social/Hello%20world.jpg" + "https://pleroma.social/Hello world.jpg" assert decode_result(url("https://pleroma.social/Hello%20world.jpg")) == "https://pleroma.social/Hello%20world.jpg" From 72b88c82bc038c8ecf6eba2012582f495f30ef43 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Mon, 8 Jul 2019 11:47:40 +0300 Subject: [PATCH 28/43] Mastodon API: Fix embedded relationships not being rendered inside of statuses --- CHANGELOG.md | 1 + .../web/mastodon_api/views/status_view.ex | 4 +-- test/web/mastodon_api/status_view_test.exs | 35 +++++++++++++++++++ 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3dbbd8225..84a16ff3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Fixed - Not being able to pin unlisted posts - Mastodon API: Handling of search timeouts (`/api/v1/search` and `/api/v2/search`) +- Mastodon API: Embedded relationships not being properly rendered in the Account entity of Status entity ### Changed - Configuration: Filter.AnonymizeFilename added ability to retain file extension with custom text diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 6836d331a..ec582b919 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -104,7 +104,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do id: to_string(activity.id), uri: activity_object.data["id"], url: activity_object.data["id"], - account: AccountView.render("account.json", %{user: user}), + account: AccountView.render("account.json", %{user: user, for: opts[:for]}), in_reply_to_id: nil, in_reply_to_account_id: nil, reblog: reblogged, @@ -221,7 +221,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do id: to_string(activity.id), uri: object.data["id"], url: url, - account: AccountView.render("account.json", %{user: user}), + account: AccountView.render("account.json", %{user: user, for: opts[:for]}), in_reply_to_id: reply_to && to_string(reply_to.id), in_reply_to_account_id: reply_to_user && to_string(reply_to_user.id), reblog: nil, diff --git a/test/web/mastodon_api/status_view_test.exs b/test/web/mastodon_api/status_view_test.exs index ec75150ab..73791a95b 100644 --- a/test/web/mastodon_api/status_view_test.exs +++ b/test/web/mastodon_api/status_view_test.exs @@ -444,4 +444,39 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do assert Enum.at(result[:options], 2)[:votes_count] == 1 end end + + test "embeds a relationship in the account" do + user = insert(:user) + other_user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{ + "status" => "drink more water" + }) + + result = StatusView.render("status.json", %{activity: activity, for: other_user}) + + assert result[:account][:pleroma][:relationship] == + AccountView.render("relationship.json", %{user: other_user, target: user}) + end + + test "embeds a relationship in the account in reposts" do + user = insert(:user) + other_user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{ + "status" => "˙˙ɐʎns" + }) + + {:ok, activity, _object} = CommonAPI.repeat(activity.id, other_user) + + result = StatusView.render("status.json", %{activity: activity, for: user}) + + assert result[:account][:pleroma][:relationship] == + AccountView.render("relationship.json", %{user: user, target: other_user}) + + assert result[:reblog][:account][:pleroma][:relationship] == + AccountView.render("relationship.json", %{user: user, target: user}) + end end From 7f609ee8f4608b25428f070e54db2346a69fb239 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Mon, 8 Jul 2019 12:16:32 +0300 Subject: [PATCH 29/43] OGP/TwitterCard: Add fallbacks in case the attachment key is nonexistent --- lib/pleroma/web/metadata/opengraph.ex | 2 ++ lib/pleroma/web/metadata/twitter_card.ex | 1 + 2 files changed, 3 insertions(+) diff --git a/lib/pleroma/web/metadata/opengraph.ex b/lib/pleroma/web/metadata/opengraph.ex index 357b80a2d..4033ec38f 100644 --- a/lib/pleroma/web/metadata/opengraph.ex +++ b/lib/pleroma/web/metadata/opengraph.ex @@ -121,4 +121,6 @@ defmodule Pleroma.Web.Metadata.Providers.OpenGraph do acc ++ rendered_tags end) end + + defp build_attachments(_), do: [] end diff --git a/lib/pleroma/web/metadata/twitter_card.ex b/lib/pleroma/web/metadata/twitter_card.ex index 040b872e7..9baf5ac97 100644 --- a/lib/pleroma/web/metadata/twitter_card.ex +++ b/lib/pleroma/web/metadata/twitter_card.ex @@ -116,6 +116,7 @@ defmodule Pleroma.Web.Metadata.Providers.TwitterCard do acc ++ rendered_tags end) end + defp build_attachments(_id, _object), do: [] defp player_url(id) do Pleroma.Web.Router.Helpers.o_status_url(Pleroma.Web.Endpoint, :notice_player, id) From 9e58d3c62471d5a9c0230a9ada19efc2722ff46a Mon Sep 17 00:00:00 2001 From: rinpatch Date: Mon, 8 Jul 2019 12:59:49 +0300 Subject: [PATCH 30/43] FallbackRedirector: Do not crash on Metadata rendering errors --- CHANGELOG.md | 3 ++- lib/pleroma/web/router.ex | 16 +++++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3dbbd8225..25fcf9dd4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Admin API: Return avatar and display name when querying users ### Fixed -- Not being able to pin unlisted posts +- Not being able to pin unlisted postss +- Metadata rendering crashes no longer result in 500 errors - Mastodon API: Handling of search timeouts (`/api/v1/search` and `/api/v2/search`) ### Changed diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 055289dc5..ff9ed1640 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -724,6 +724,7 @@ end defmodule Fallback.RedirectController do use Pleroma.Web, :controller + require Logger alias Pleroma.User alias Pleroma.Web.Metadata @@ -750,7 +751,20 @@ defmodule Fallback.RedirectController do def redirector_with_meta(conn, params) do {:ok, index_content} = File.read(index_file_path()) - tags = Metadata.build_tags(params) + + tags = + try do + Metadata.build_tags(params) + rescue + e -> + Logger.error( + "Metadata rendering for #{conn.request_path} failed.\n" <> + Exception.format(:error, e, __STACKTRACE__) + ) + + "" + end + response = String.replace(index_content, "", tags) conn From 682f1897b7d562d50f05cdb8da46d2ca55dfa22f Mon Sep 17 00:00:00 2001 From: rinpatch Date: Mon, 8 Jul 2019 13:00:45 +0300 Subject: [PATCH 31/43] Enable OpenGraph and TwitterCard by default Closes #1034 --- config/config.exs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/config/config.exs b/config/config.exs index e337f00aa..3aa03831b 100644 --- a/config/config.exs +++ b/config/config.exs @@ -358,7 +358,11 @@ config :pleroma, :gopher, port: 9999 config :pleroma, Pleroma.Web.Metadata, - providers: [Pleroma.Web.Metadata.Providers.RelMe], + providers: [ + Pleroma.Web.Metadata.Providers.OpenGraph, + Pleroma.Web.Metadata.Providers.TwitterCard, + Pleroma.Web.Metadata.Providers.RelMe + ], unfurl_nsfw: false config :pleroma, :suggestions, From f5b91bc1576cfcfe702ade5c94727c752e61f0ac Mon Sep 17 00:00:00 2001 From: rinpatch Date: Mon, 8 Jul 2019 13:04:14 +0300 Subject: [PATCH 32/43] Improve wording in CHANGELOG.md --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 25fcf9dd4..99f6fe474 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,8 +11,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Admin API: Return avatar and display name when querying users ### Fixed -- Not being able to pin unlisted postss -- Metadata rendering crashes no longer result in 500 errors +- Not being able to pin unlisted posts +- Metadata rendering errors resulting in the entire page being inaccessible - Mastodon API: Handling of search timeouts (`/api/v1/search` and `/api/v2/search`) ### Changed From 44b2e1fdad7e7379fa03bc4ba0342e576fb7bf75 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Mon, 8 Jul 2019 14:05:57 +0300 Subject: [PATCH 33/43] Formatting --- lib/pleroma/web/metadata/twitter_card.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/web/metadata/twitter_card.ex b/lib/pleroma/web/metadata/twitter_card.ex index 9baf5ac97..8dd01e0d5 100644 --- a/lib/pleroma/web/metadata/twitter_card.ex +++ b/lib/pleroma/web/metadata/twitter_card.ex @@ -116,6 +116,7 @@ defmodule Pleroma.Web.Metadata.Providers.TwitterCard do acc ++ rendered_tags end) end + defp build_attachments(_id, _object), do: [] defp player_url(id) do From f3cc2acb0f4ae9112ea30a259acc384f5138c0fc Mon Sep 17 00:00:00 2001 From: rinpatch Date: Mon, 8 Jul 2019 14:13:30 +0300 Subject: [PATCH 34/43] Add a changelog entry for changing the defaults --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 99f6fe474..e665a1986 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Handling of search timeouts (`/api/v1/search` and `/api/v2/search`) ### Changed +- Configuration: OpenGraph and TwitterCard providers enabled by default - Configuration: Filter.AnonymizeFilename added ability to retain file extension with custom text ### Changed From abe2e8881f946aafc2012edd43373c22837387af Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 9 Jul 2019 15:30:51 +0900 Subject: [PATCH 35/43] Testing: Don't federate in testing. --- config/test.exs | 3 ++- lib/pleroma/web/activity_pub/utils.ex | 17 ++++++++++------- test/conversation_test.exs | 10 ++++++++++ .../activity_pub_controller_test.exs | 7 +++++++ test/web/federator_test.exs | 7 +++++++ test/web/ostatus/ostatus_controller_test.exs | 7 +++++++ test/web/plugs/federating_plug_test.exs | 13 +++++++++++-- .../web_finger/web_finger_controller_test.exs | 6 ++++++ test/web/websub/websub_controller_test.exs | 10 ++++++++++ 9 files changed, 70 insertions(+), 10 deletions(-) diff --git a/config/test.exs b/config/test.exs index 9d441a7f5..012021f2a 100644 --- a/config/test.exs +++ b/config/test.exs @@ -28,7 +28,8 @@ config :pleroma, Pleroma.Emails.Mailer, adapter: Swoosh.Adapters.Test config :pleroma, :instance, email: "admin@example.com", notify_email: "noreply@example.com", - skip_thread_containment: false + skip_thread_containment: false, + federating: false # Configure your database config :pleroma, Pleroma.Repo, diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 514266cee..4288ea4c8 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -170,14 +170,17 @@ defmodule Pleroma.Web.ActivityPub.Utils do Enqueues an activity for federation if it's local """ def maybe_federate(%Activity{local: true} = activity) do - priority = - case activity.data["type"] do - "Delete" -> 10 - "Create" -> 1 - _ -> 5 - end + if Pleroma.Config.get!([:instance, :federating]) do + priority = + case activity.data["type"] do + "Delete" -> 10 + "Create" -> 1 + _ -> 5 + end + + Pleroma.Web.Federator.publish(activity, priority) + end - Pleroma.Web.Federator.publish(activity, priority) :ok end diff --git a/test/conversation_test.exs b/test/conversation_test.exs index 5903d10ff..aa193e0d4 100644 --- a/test/conversation_test.exs +++ b/test/conversation_test.exs @@ -11,6 +11,16 @@ defmodule Pleroma.ConversationTest do import Pleroma.Factory + setup_all do + config_path = [:instance, :federating] + initial_setting = Pleroma.Config.get(config_path) + + Pleroma.Config.put(config_path, true) + on_exit(fn -> Pleroma.Config.put(config_path, initial_setting) end) + + :ok + end + test "it goes through old direct conversations" do user = insert(:user) other_user = insert(:user) diff --git a/test/web/activity_pub/activity_pub_controller_test.exs b/test/web/activity_pub/activity_pub_controller_test.exs index 8b3233729..5a8a67155 100644 --- a/test/web/activity_pub/activity_pub_controller_test.exs +++ b/test/web/activity_pub/activity_pub_controller_test.exs @@ -15,6 +15,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do setup_all do Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) + + config_path = [:instance, :federating] + initial_setting = Pleroma.Config.get(config_path) + + Pleroma.Config.put(config_path, true) + on_exit(fn -> Pleroma.Config.put(config_path, initial_setting) end) + :ok end diff --git a/test/web/federator_test.exs b/test/web/federator_test.exs index 0f43bc8f2..69dd4d747 100644 --- a/test/web/federator_test.exs +++ b/test/web/federator_test.exs @@ -12,6 +12,13 @@ defmodule Pleroma.Web.FederatorTest do setup_all do Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) + + config_path = [:instance, :federating] + initial_setting = Pleroma.Config.get(config_path) + + Pleroma.Config.put(config_path, true) + on_exit(fn -> Pleroma.Config.put(config_path, initial_setting) end) + :ok end diff --git a/test/web/ostatus/ostatus_controller_test.exs b/test/web/ostatus/ostatus_controller_test.exs index 7441e5fce..eae44dba5 100644 --- a/test/web/ostatus/ostatus_controller_test.exs +++ b/test/web/ostatus/ostatus_controller_test.exs @@ -12,6 +12,13 @@ defmodule Pleroma.Web.OStatus.OStatusControllerTest do setup_all do Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) + + config_path = [:instance, :federating] + initial_setting = Pleroma.Config.get(config_path) + + Pleroma.Config.put(config_path, true) + on_exit(fn -> Pleroma.Config.put(config_path, initial_setting) end) + :ok end diff --git a/test/web/plugs/federating_plug_test.exs b/test/web/plugs/federating_plug_test.exs index 530562325..c01e01124 100644 --- a/test/web/plugs/federating_plug_test.exs +++ b/test/web/plugs/federating_plug_test.exs @@ -5,6 +5,15 @@ defmodule Pleroma.Web.FederatingPlugTest do use Pleroma.Web.ConnCase + setup_all do + config_path = [:instance, :federating] + initial_setting = Pleroma.Config.get(config_path) + + on_exit(fn -> Pleroma.Config.put(config_path, initial_setting) end) + + :ok + end + test "returns and halt the conn when federating is disabled" do Pleroma.Config.put([:instance, :federating], false) @@ -14,11 +23,11 @@ defmodule Pleroma.Web.FederatingPlugTest do assert conn.status == 404 assert conn.halted - - Pleroma.Config.put([:instance, :federating], true) end test "does nothing when federating is enabled" do + Pleroma.Config.put([:instance, :federating], true) + conn = build_conn() |> Pleroma.Web.FederatingPlug.call(%{}) diff --git a/test/web/web_finger/web_finger_controller_test.exs b/test/web/web_finger/web_finger_controller_test.exs index 43fccfc7a..a14ed3126 100644 --- a/test/web/web_finger/web_finger_controller_test.exs +++ b/test/web/web_finger/web_finger_controller_test.exs @@ -10,6 +10,12 @@ defmodule Pleroma.Web.WebFinger.WebFingerControllerTest do setup do mock(fn env -> apply(HttpRequestMock, :request, [env]) end) + + config_path = [:instance, :federating] + initial_setting = Pleroma.Config.get(config_path) + + Pleroma.Config.put(config_path, true) + on_exit(fn -> Pleroma.Config.put(config_path, initial_setting) end) :ok end diff --git a/test/web/websub/websub_controller_test.exs b/test/web/websub/websub_controller_test.exs index f79745d58..aa7262beb 100644 --- a/test/web/websub/websub_controller_test.exs +++ b/test/web/websub/websub_controller_test.exs @@ -9,6 +9,16 @@ defmodule Pleroma.Web.Websub.WebsubControllerTest do alias Pleroma.Web.Websub alias Pleroma.Web.Websub.WebsubClientSubscription + setup_all do + config_path = [:instance, :federating] + initial_setting = Pleroma.Config.get(config_path) + + Pleroma.Config.put(config_path, true) + on_exit(fn -> Pleroma.Config.put(config_path, initial_setting) end) + + :ok + end + test "websub subscription request", %{conn: conn} do user = insert(:user) From 23d4781e73c4a34fcc8d442cf1d3e2863a07ad84 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 9 Jul 2019 08:52:49 +0000 Subject: [PATCH 36/43] change for local user search --- lib/pleroma/user/search.ex | 6 +++++- test/user_search_test.exs | 31 +++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/user/search.ex b/lib/pleroma/user/search.ex index 7680c2afd..64eb6d2bc 100644 --- a/lib/pleroma/user/search.ex +++ b/lib/pleroma/user/search.ex @@ -150,7 +150,7 @@ defmodule Pleroma.User.Search do @spec fts_search_subquery(User.t() | Ecto.Query.t(), String.t()) :: Ecto.Query.t() defp fts_search_subquery(query, term) do processed_query = - term + String.trim_trailing(term, "@" <> local_domain()) |> String.replace(~r/\W+/, " ") |> String.trim() |> String.split() @@ -192,6 +192,8 @@ defmodule Pleroma.User.Search do @spec trigram_search_subquery(User.t() | Ecto.Query.t(), String.t()) :: Ecto.Query.t() defp trigram_search_subquery(query, term) do + term = String.trim_trailing(term, "@" <> local_domain()) + from( u in query, select_merge: %{ @@ -209,4 +211,6 @@ defmodule Pleroma.User.Search do ) |> User.restrict_deactivated() end + + defp local_domain, do: Pleroma.Config.get([Pleroma.Web.Endpoint, :url, :host]) end diff --git a/test/user_search_test.exs b/test/user_search_test.exs index 8f8472aae..1f0162486 100644 --- a/test/user_search_test.exs +++ b/test/user_search_test.exs @@ -217,5 +217,36 @@ defmodule Pleroma.UserSearchTest do refute Enum.member?(account_ids, blocked_user2.id) assert length(account_ids) == 3 end + + test "local user has the same search_rank as for users with the same nickname, but another domain" do + user = insert(:user) + insert(:user, nickname: "lain@mastodon.social") + insert(:user, nickname: "lain") + insert(:user, nickname: "lain@pleroma.social") + + assert User.search("lain@localhost", resolve: true, for_user: user) + |> Enum.each(fn u -> u.search_rank == 0.5 end) + end + + test "localhost is the part of the domain" do + user = insert(:user) + insert(:user, nickname: "another@somedomain") + insert(:user, nickname: "lain") + insert(:user, nickname: "lain@examplelocalhost") + + result = User.search("lain@examplelocalhost", resolve: true, for_user: user) + assert Enum.each(result, fn u -> u.search_rank == 0.5 end) + assert length(result) == 2 + end + + test "local user search with users" do + user = insert(:user) + local_user = insert(:user, nickname: "lain") + insert(:user, nickname: "another@localhost.com") + insert(:user, nickname: "localhost@localhost.com") + + [result] = User.search("lain@localhost", resolve: true, for_user: user) + assert Map.put(result, :search_rank, nil) |> Map.put(:search_type, nil) == local_user + end end end From dd5a41e2a4312a3dc7a1083d3d0ac5b356afafa8 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Tue, 9 Jul 2019 10:39:36 +0000 Subject: [PATCH 37/43] Apply suggestion to docs/config.md --- docs/config.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/config.md b/docs/config.md index 6cbbb6ce9..822c34c51 100644 --- a/docs/config.md +++ b/docs/config.md @@ -87,7 +87,7 @@ config :pleroma, Pleroma.Emails.Mailer, * `invites_enabled`: Enable user invitations for admins (depends on `registrations_open: false`). * `account_activation_required`: Require users to confirm their emails before signing in. * `federating`: Enable federation with other instances -* `federation_incoming_replies_max_depth`: Max. depth of reply-to activities fetching on incoming federation (to prevent memory leakage on extremely nested incoming threads). If set to `nil`, threads of any depth will be fetched. +* `federation_incoming_replies_max_depth`: Max. depth of reply-to activities fetching on incoming federation, to prevent out-of-memory situations while fetching very long threads. If set to `nil`, threads of any depth will be fetched. Lower this value if you experience out-of-memory crashes. * `federation_reachability_timeout_days`: Timeout (in days) of each external federation target being unreachable prior to pausing federating to it. * `allow_relay`: Enable Pleroma’s Relay, which makes it possible to follow a whole instance * `rewrite_policy`: Message Rewrite Policy, either one or a list. Here are the ones available by default: From 4213a4e2aa144ba0a3dff69d5b991a2deba0aa85 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 9 Jul 2019 09:41:41 -0500 Subject: [PATCH 38/43] Test that all ASCII encoded characters are preserved --- test/media_proxy_test.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/media_proxy_test.exs b/test/media_proxy_test.exs index cd1cbd202..1b6b3c1fd 100644 --- a/test/media_proxy_test.exs +++ b/test/media_proxy_test.exs @@ -145,9 +145,9 @@ defmodule Pleroma.MediaProxyTest do end # https://git.pleroma.social/pleroma/pleroma/issues/580 - test "encoding S3 links (must preserve `%2F`)" do + test "preserve ascii encoding" do url = - "https://s3.amazonaws.com/example/test.png?X-Amz-Credential=your-access-key-id%2F20130721%2Fus-east-1%2Fs3%2Faws4_request" + "https://pleroma.com/%20/%21/%22/%23/%24/%25/%26/%27/%28/%29/%2A/%2B/%2C/%2D/%2E/%2F/%30/%31/%32/%33/%34/%35/%36/%37/%38/%39/%3A/%3B/%3C/%3D/%3E/%3F/%40/%41/%42/%43/%44/%45/%46/%47/%48/%49/%4A/%4B/%4C/%4D/%4E/%4F/%50/%51/%52/%53/%54/%55/%56/%57/%58/%59/%5A/%5B/%5C/%5D/%5E/%5F/%60/%61/%62/%63/%64/%65/%66/%67/%68/%69/%6A/%6B/%6C/%6D/%6E/%6F/%70/%71/%72/%73/%74/%75/%76/%77/%78/%79/%7A/%7B/%7C/%7D/%7E/%7F/%80/%81/%82/%83/%84/%85/%86/%87/%88/%89/%8A/%8B/%8C/%8D/%8E/%8F/%90/%91/%92/%93/%94/%95/%96/%97/%98/%99/%9A/%9B/%9C/%9D/%9E/%9F/%C2%A0/%A1/%A2/%A3/%A4/%A5/%A6/%A7/%A8/%A9/%AA/%AB/%AC/%C2%AD/%AE/%AF/%B0/%B1/%B2/%B3/%B4/%B5/%B6/%B7/%B8/%B9/%BA/%BB/%BC/%BD/%BE/%BF/%C0/%C1/%C2/%C3/%C4/%C5/%C6/%C7/%C8/%C9/%CA/%CB/%CC/%CD/%CE/%CF/%D0/%D1/%D2/%D3/%D4/%D5/%D6/%D7/%D8/%D9/%DA/%DB/%DC/%DD/%DE/%DF/%E0/%E1/%E2/%E3/%E4/%E5/%E6/%E7/%E8/%E9/%EA/%EB/%EC/%ED/%EE/%EF/%F0/%F1/%F2/%F3/%F4/%F5/%F6/%F7/%F8/%F9/%FA/%FB/%FC/%FD/%FE/%FF" encoded = url(url) assert decode_result(encoded) == url From 98f13eac9e38c6ec44a7146cfc58114b0148f462 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 9 Jul 2019 10:11:42 -0500 Subject: [PATCH 39/43] Capitalize --- test/media_proxy_test.exs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/test/media_proxy_test.exs b/test/media_proxy_test.exs index 1b6b3c1fd..144d261db 100644 --- a/test/media_proxy_test.exs +++ b/test/media_proxy_test.exs @@ -145,7 +145,7 @@ defmodule Pleroma.MediaProxyTest do end # https://git.pleroma.social/pleroma/pleroma/issues/580 - test "preserve ascii encoding" do + test "preserve ASCII encoding" do url = "https://pleroma.com/%20/%21/%22/%23/%24/%25/%26/%27/%28/%29/%2A/%2B/%2C/%2D/%2E/%2F/%30/%31/%32/%33/%34/%35/%36/%37/%38/%39/%3A/%3B/%3C/%3D/%3E/%3F/%40/%41/%42/%43/%44/%45/%46/%47/%48/%49/%4A/%4B/%4C/%4D/%4E/%4F/%50/%51/%52/%53/%54/%55/%56/%57/%58/%59/%5A/%5B/%5C/%5D/%5E/%5F/%60/%61/%62/%63/%64/%65/%66/%67/%68/%69/%6A/%6B/%6C/%6D/%6E/%6F/%70/%71/%72/%73/%74/%75/%76/%77/%78/%79/%7A/%7B/%7C/%7D/%7E/%7F/%80/%81/%82/%83/%84/%85/%86/%87/%88/%89/%8A/%8B/%8C/%8D/%8E/%8F/%90/%91/%92/%93/%94/%95/%96/%97/%98/%99/%9A/%9B/%9C/%9D/%9E/%9F/%C2%A0/%A1/%A2/%A3/%A4/%A5/%A6/%A7/%A8/%A9/%AA/%AB/%AC/%C2%AD/%AE/%AF/%B0/%B1/%B2/%B3/%B4/%B5/%B6/%B7/%B8/%B9/%BA/%BB/%BC/%BD/%BE/%BF/%C0/%C1/%C2/%C3/%C4/%C5/%C6/%C7/%C8/%C9/%CA/%CB/%CC/%CD/%CE/%CF/%D0/%D1/%D2/%D3/%D4/%D5/%D6/%D7/%D8/%D9/%DA/%DB/%DC/%DD/%DE/%DF/%E0/%E1/%E2/%E3/%E4/%E5/%E6/%E7/%E8/%E9/%EA/%EB/%EC/%ED/%EE/%EF/%F0/%F1/%F2/%F3/%F4/%F5/%F6/%F7/%F8/%F9/%FA/%FB/%FC/%FD/%FE/%FF" @@ -153,6 +153,17 @@ defmodule Pleroma.MediaProxyTest do assert decode_result(encoded) == url end + # This includes unsafe/reserved characters which are not interpreted as part of the URL + # and would otherwise have to be ASCII encoded. It is our role to ensure the proxied URL + # is unmodified, so we are testing these characters anyway. + test "preserve non-unicode characters per RFC3986" do + url = + "https://pleroma.com/ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz1234567890-._~:/?#[]@!$&'()*+,;=|^`{}" + + encoded = url(url) + assert decode_result(encoded) == url + end + test "does not change whitelisted urls" do upload_config = Pleroma.Config.get([Pleroma.Upload]) media_url = "https://media.pleroma.social" From ce3ffad13a5ceeab383f43bf576ff8bbbd0af42f Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 9 Jul 2019 10:23:22 -0500 Subject: [PATCH 40/43] Remove duplicated test. New one is more comprehensive. --- test/media_proxy_test.exs | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/test/media_proxy_test.exs b/test/media_proxy_test.exs index 144d261db..13922fe4a 100644 --- a/test/media_proxy_test.exs +++ b/test/media_proxy_test.exs @@ -70,17 +70,6 @@ defmodule Pleroma.MediaProxyTest do assert decode_result(encoded) == url end - # Some of the signed url expect the special character in the url to be same - # for the proxy to work. - # Issue https://git.pleroma.social/pleroma/pleroma/issues/1055 - test "ensures urls are maintained (character are not encoded or decoded)" do - assert decode_result(url("https://pleroma.social/Hello world.jpg")) == - "https://pleroma.social/Hello world.jpg" - - assert decode_result(url("https://pleroma.social/Hello%20world.jpg")) == - "https://pleroma.social/Hello%20world.jpg" - end - test "validates signature" do secret_key_base = Pleroma.Config.get([Pleroma.Web.Endpoint, :secret_key_base]) @@ -144,7 +133,10 @@ defmodule Pleroma.MediaProxyTest do assert String.starts_with?(encoded, Pleroma.Config.get([:media_proxy, :base_url])) end - # https://git.pleroma.social/pleroma/pleroma/issues/580 + # Some sites expect ASCII encoded characters in the URL to be preserved even if + # unnecessary. + # Issues: https://git.pleroma.social/pleroma/pleroma/issues/580 + # https://git.pleroma.social/pleroma/pleroma/issues/1055 test "preserve ASCII encoding" do url = "https://pleroma.com/%20/%21/%22/%23/%24/%25/%26/%27/%28/%29/%2A/%2B/%2C/%2D/%2E/%2F/%30/%31/%32/%33/%34/%35/%36/%37/%38/%39/%3A/%3B/%3C/%3D/%3E/%3F/%40/%41/%42/%43/%44/%45/%46/%47/%48/%49/%4A/%4B/%4C/%4D/%4E/%4F/%50/%51/%52/%53/%54/%55/%56/%57/%58/%59/%5A/%5B/%5C/%5D/%5E/%5F/%60/%61/%62/%63/%64/%65/%66/%67/%68/%69/%6A/%6B/%6C/%6D/%6E/%6F/%70/%71/%72/%73/%74/%75/%76/%77/%78/%79/%7A/%7B/%7C/%7D/%7E/%7F/%80/%81/%82/%83/%84/%85/%86/%87/%88/%89/%8A/%8B/%8C/%8D/%8E/%8F/%90/%91/%92/%93/%94/%95/%96/%97/%98/%99/%9A/%9B/%9C/%9D/%9E/%9F/%C2%A0/%A1/%A2/%A3/%A4/%A5/%A6/%A7/%A8/%A9/%AA/%AB/%AC/%C2%AD/%AE/%AF/%B0/%B1/%B2/%B3/%B4/%B5/%B6/%B7/%B8/%B9/%BA/%BB/%BC/%BD/%BE/%BF/%C0/%C1/%C2/%C3/%C4/%C5/%C6/%C7/%C8/%C9/%CA/%CB/%CC/%CD/%CE/%CF/%D0/%D1/%D2/%D3/%D4/%D5/%D6/%D7/%D8/%D9/%DA/%DB/%DC/%DD/%DE/%DF/%E0/%E1/%E2/%E3/%E4/%E5/%E6/%E7/%E8/%E9/%EA/%EB/%EC/%ED/%EE/%EF/%F0/%F1/%F2/%F3/%F4/%F5/%F6/%F7/%F8/%F9/%FA/%FB/%FC/%FD/%FE/%FF" From e143747445a0cd4f9b34c1b96ab7e87632e21a74 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 9 Jul 2019 10:55:36 -0500 Subject: [PATCH 41/43] Add test for URLs with Unicode characters too --- test/media_proxy_test.exs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/media_proxy_test.exs b/test/media_proxy_test.exs index 13922fe4a..1d6d170b7 100644 --- a/test/media_proxy_test.exs +++ b/test/media_proxy_test.exs @@ -156,6 +156,13 @@ defmodule Pleroma.MediaProxyTest do assert decode_result(encoded) == url end + test "preserve unicode characters" do + url = "https://ko.wikipedia.org/wiki/위키백과:대문" + + encoded = url(url) + assert decode_result(encoded) == url + end + test "does not change whitelisted urls" do upload_config = Pleroma.Config.get([Pleroma.Upload]) media_url = "https://media.pleroma.social" From 4e6e5d80428a40f0403560b3c8381ea48cf4373e Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 9 Jul 2019 16:54:13 +0000 Subject: [PATCH 42/43] reverse proxy tests --- config/test.exs | 2 + lib/mix/tasks/pleroma/ecto/ecto.ex | 1 + lib/pleroma/reverse_proxy/client.ex | 24 ++ .../{ => reverse_proxy}/reverse_proxy.ex | 10 +- mix.exs | 3 +- mix.lock | 3 +- test/http/request_builder_test.exs | 91 ++++++ test/reverse_proxy_test.exs | 297 ++++++++++++++++++ test/tasks/ecto/ecto_test.exs | 11 + test/tasks/pleroma_test.exs | 46 +++ test/tasks/robots_txt_test.exs | 43 +++ test/test_helper.exs | 2 +- 12 files changed, 524 insertions(+), 9 deletions(-) create mode 100644 lib/pleroma/reverse_proxy/client.ex rename lib/pleroma/{ => reverse_proxy}/reverse_proxy.ex (97%) create mode 100644 test/http/request_builder_test.exs create mode 100644 test/reverse_proxy_test.exs create mode 100644 test/tasks/ecto/ecto_test.exs create mode 100644 test/tasks/pleroma_test.exs create mode 100644 test/tasks/robots_txt_test.exs diff --git a/config/test.exs b/config/test.exs index 012021f2a..63443dde0 100644 --- a/config/test.exs +++ b/config/test.exs @@ -75,6 +75,8 @@ rum_enabled = System.get_env("RUM_ENABLED") == "true" config :pleroma, :database, rum_enabled: rum_enabled IO.puts("RUM enabled: #{rum_enabled}") +config :pleroma, Pleroma.ReverseProxy.Client, Pleroma.ReverseProxy.ClientMock + try do import_config "test.secret.exs" rescue diff --git a/lib/mix/tasks/pleroma/ecto/ecto.ex b/lib/mix/tasks/pleroma/ecto/ecto.ex index 324f57fdd..b66f63376 100644 --- a/lib/mix/tasks/pleroma/ecto/ecto.ex +++ b/lib/mix/tasks/pleroma/ecto/ecto.ex @@ -1,6 +1,7 @@ # Pleroma: A lightweight social networking server # Copyright © 2017-2018 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-onl + defmodule Mix.Tasks.Pleroma.Ecto do @doc """ Ensures the given repository's migrations path exists on the file system. diff --git a/lib/pleroma/reverse_proxy/client.ex b/lib/pleroma/reverse_proxy/client.ex new file mode 100644 index 000000000..57c2d2cfd --- /dev/null +++ b/lib/pleroma/reverse_proxy/client.ex @@ -0,0 +1,24 @@ +defmodule Pleroma.ReverseProxy.Client do + @callback request(atom(), String.t(), [tuple()], String.t(), list()) :: + {:ok, pos_integer(), [tuple()], reference() | map()} + | {:ok, pos_integer(), [tuple()]} + | {:ok, reference()} + | {:error, term()} + + @callback stream_body(reference() | pid() | map()) :: + {:ok, binary()} | :done | {:error, String.t()} + + @callback close(reference() | pid() | map()) :: :ok + + def request(method, url, headers, "", opts \\ []) do + client().request(method, url, headers, "", opts) + end + + def stream_body(ref), do: client().stream_body(ref) + + def close(ref), do: client().close(ref) + + defp client do + Pleroma.Config.get([Pleroma.ReverseProxy.Client], :hackney) + end +end diff --git a/lib/pleroma/reverse_proxy.ex b/lib/pleroma/reverse_proxy/reverse_proxy.ex similarity index 97% rename from lib/pleroma/reverse_proxy.ex rename to lib/pleroma/reverse_proxy/reverse_proxy.ex index de0f6e1bc..bf31e9cba 100644 --- a/lib/pleroma/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy/reverse_proxy.ex @@ -146,7 +146,7 @@ defmodule Pleroma.ReverseProxy do Logger.debug("#{__MODULE__} #{method} #{url} #{inspect(headers)}") method = method |> String.downcase() |> String.to_existing_atom() - case hackney().request(method, url, headers, "", hackney_opts) do + case client().request(method, url, headers, "", hackney_opts) do {:ok, code, headers, client} when code in @valid_resp_codes -> {:ok, code, downcase_headers(headers), client} @@ -173,7 +173,7 @@ defmodule Pleroma.ReverseProxy do halt(conn) {:error, :closed, conn} -> - :hackney.close(client) + client().close(client) halt(conn) {:error, error, conn} -> @@ -181,7 +181,7 @@ defmodule Pleroma.ReverseProxy do "#{__MODULE__} request to #{url} failed while reading/chunking: #{inspect(error)}" ) - :hackney.close(client) + client().close(client) halt(conn) end end @@ -196,7 +196,7 @@ defmodule Pleroma.ReverseProxy do duration, Keyword.get(opts, :max_read_duration, @max_read_duration) ), - {:ok, data} <- hackney().stream_body(client), + {:ok, data} <- client().stream_body(client), {:ok, duration} <- increase_read_duration(duration), sent_so_far = sent_so_far + byte_size(data), :ok <- body_size_constraint(sent_so_far, Keyword.get(opts, :max_body_size)), @@ -378,5 +378,5 @@ defmodule Pleroma.ReverseProxy do {:ok, :no_duration_limit, :no_duration_limit} end - defp hackney, do: Pleroma.Config.get(:hackney, :hackney) + defp client, do: Pleroma.ReverseProxy.Client end diff --git a/mix.exs b/mix.exs index 22d3d50df..48a43fccb 100644 --- a/mix.exs +++ b/mix.exs @@ -151,7 +151,8 @@ defmodule Pleroma.Mixfile do {:esshd, "~> 0.1.0", runtime: Application.get_env(:esshd, :enabled, false)}, {:ex_rated, "~> 1.3"}, {:plug_static_index_html, "~> 1.0.0"}, - {:excoveralls, "~> 0.11.1", only: :test} + {:excoveralls, "~> 0.11.1", only: :test}, + {:mox, "~> 0.5", only: :test} ] ++ oauth_deps() end diff --git a/mix.lock b/mix.lock index cae8d7d84..e711be635 100644 --- a/mix.lock +++ b/mix.lock @@ -52,6 +52,7 @@ "mochiweb": {:hex, :mochiweb, "2.15.0", "e1daac474df07651e5d17cc1e642c4069c7850dc4508d3db7263a0651330aacc", [:rebar3], [], "hexpm"}, "mock": {:hex, :mock, "0.3.3", "42a433794b1291a9cf1525c6d26b38e039e0d3a360732b5e467bfc77ef26c914", [:mix], [{:meck, "~> 0.8.13", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm"}, "mogrify": {:hex, :mogrify, "0.6.1", "de1b527514f2d95a7bbe9642eb556061afb337e220cf97adbf3a4e6438ed70af", [:mix], [], "hexpm"}, + "mox": {:hex, :mox, "0.5.1", "f86bb36026aac1e6f924a4b6d024b05e9adbed5c63e8daa069bd66fb3292165b", [:mix], [], "hexpm"}, "nimble_parsec": {:hex, :nimble_parsec, "0.5.0", "90e2eca3d0266e5c53f8fbe0079694740b9c91b6747f2b7e3c5d21966bba8300", [:mix], [], "hexpm"}, "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm"}, "pbkdf2_elixir": {:hex, :pbkdf2_elixir, "0.12.3", "6706a148809a29c306062862c803406e88f048277f6e85b68faf73291e820b84", [:mix], [], "hexpm"}, @@ -65,14 +66,12 @@ "plug_crypto": {:hex, :plug_crypto, "1.0.0", "18e49317d3fa343f24620ed22795ec29d4a5e602d52d1513ccea0b07d8ea7d4d", [:mix], [], "hexpm"}, "plug_static_index_html": {:hex, :plug_static_index_html, "1.0.0", "840123d4d3975585133485ea86af73cb2600afd7f2a976f9f5fd8b3808e636a0", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm"}, - "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm"}, "postgrex": {:hex, :postgrex, "0.14.3", "5754dee2fdf6e9e508cbf49ab138df964278700b764177e8f3871e658b345a1e", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, "prometheus": {:hex, :prometheus, "4.2.2", "a830e77b79dc6d28183f4db050a7cac926a6c58f1872f9ef94a35cd989aceef8", [:mix, :rebar3], [], "hexpm"}, "prometheus_ecto": {:hex, :prometheus_ecto, "1.4.1", "6c768ea9654de871e5b32fab2eac348467b3021604ebebbcbd8bcbe806a65ed5", [:mix], [{:ecto, "~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm"}, "prometheus_ex": {:hex, :prometheus_ex, "3.0.5", "fa58cfd983487fc5ead331e9a3e0aa622c67232b3ec71710ced122c4c453a02f", [:mix], [{:prometheus, "~> 4.0", [hex: :prometheus, repo: "hexpm", optional: false]}], "hexpm"}, "prometheus_phoenix": {:hex, :prometheus_phoenix, "1.2.1", "964a74dfbc055f781d3a75631e06ce3816a2913976d1df7830283aa3118a797a", [:mix], [{:phoenix, "~> 1.3", [hex: :phoenix, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.3 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm"}, "prometheus_plugs": {:hex, :prometheus_plugs, "1.1.5", "25933d48f8af3a5941dd7b621c889749894d8a1082a6ff7c67cc99dec26377c5", [:mix], [{:accept, "~> 0.1", [hex: :accept, repo: "hexpm", optional: false]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}, {:prometheus_process_collector, "~> 1.1", [hex: :prometheus_process_collector, repo: "hexpm", optional: true]}], "hexpm"}, - "prometheus_process_collector": {:hex, :prometheus_process_collector, "1.4.0", "6dbd39e3165b9ef1c94a7a820e9ffe08479f949dcdd431ed4aaea7b250eebfde", [:rebar3], [{:prometheus, "~> 4.0", [hex: :prometheus, repo: "hexpm", optional: false]}], "hexpm"}, "quack": {:hex, :quack, "0.1.1", "cca7b4da1a233757fdb44b3334fce80c94785b3ad5a602053b7a002b5a8967bf", [:mix], [{:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: false]}, {:tesla, "~> 1.2.0", [hex: :tesla, repo: "hexpm", optional: false]}], "hexpm"}, "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm"}, "recon": {:git, "https://github.com/ferd/recon.git", "75d70c7c08926d2f24f1ee6de14ee50fe8a52763", [tag: "2.4.0"]}, diff --git a/test/http/request_builder_test.exs b/test/http/request_builder_test.exs new file mode 100644 index 000000000..a368999ff --- /dev/null +++ b/test/http/request_builder_test.exs @@ -0,0 +1,91 @@ +defmodule Pleroma.HTTP.RequestBuilderTest do + use ExUnit.Case, async: true + alias Pleroma.HTTP.RequestBuilder + + describe "headers/2" do + test "don't send pleroma user agent" do + assert RequestBuilder.headers(%{}, []) == %{headers: []} + end + + test "send pleroma user agent" do + send = Pleroma.Config.get([:http, :send_user_agent]) + Pleroma.Config.put([:http, :send_user_agent], true) + + on_exit(fn -> + Pleroma.Config.put([:http, :send_user_agent], send) + end) + + assert RequestBuilder.headers(%{}, []) == %{ + headers: [{"User-Agent", Pleroma.Application.user_agent()}] + } + end + end + + describe "add_optional_params/3" do + test "don't add if keyword is empty" do + assert RequestBuilder.add_optional_params(%{}, %{}, []) == %{} + end + + test "add query parameter" do + assert RequestBuilder.add_optional_params( + %{}, + %{query: :query, body: :body, another: :val}, + [ + {:query, "param1=val1¶m2=val2"}, + {:body, "some body"} + ] + ) == %{query: "param1=val1¶m2=val2", body: "some body"} + end + end + + describe "add_param/4" do + test "add file parameter" do + %{ + body: %Tesla.Multipart{ + boundary: _, + content_type_params: [], + parts: [ + %Tesla.Multipart.Part{ + body: %File.Stream{ + line_or_bytes: 2048, + modes: [:raw, :read_ahead, :read, :binary], + path: "some-path/filename.png", + raw: true + }, + dispositions: [name: "filename.png", filename: "filename.png"], + headers: [] + } + ] + } + } = RequestBuilder.add_param(%{}, :file, "filename.png", "some-path/filename.png") + end + + test "add key to body" do + %{ + body: %Tesla.Multipart{ + boundary: _, + content_type_params: [], + parts: [ + %Tesla.Multipart.Part{ + body: "\"someval\"", + dispositions: [name: "somekey"], + headers: ["Content-Type": "application/json"] + } + ] + } + } = RequestBuilder.add_param(%{}, :body, "somekey", "someval") + end + + test "add form parameter" do + assert RequestBuilder.add_param(%{}, :form, "somename", "someval") == %{ + body: %{"somename" => "someval"} + } + end + + test "add for location" do + assert RequestBuilder.add_param(%{}, :some_location, "somekey", "someval") == %{ + some_location: [{"somekey", "someval"}] + } + end + end +end diff --git a/test/reverse_proxy_test.exs b/test/reverse_proxy_test.exs new file mode 100644 index 000000000..75a61445a --- /dev/null +++ b/test/reverse_proxy_test.exs @@ -0,0 +1,297 @@ +defmodule Pleroma.ReverseProxyTest do + use Pleroma.Web.ConnCase, async: true + import ExUnit.CaptureLog + import ExUnit.CaptureLog + import Mox + alias Pleroma.ReverseProxy + alias Pleroma.ReverseProxy.ClientMock + + setup_all do + {:ok, _} = Registry.start_link(keys: :unique, name: Pleroma.ReverseProxy.ClientMock) + :ok + end + + setup :verify_on_exit! + + defp user_agent_mock(user_agent, invokes) do + json = Jason.encode!(%{"user-agent": user_agent}) + + ClientMock + |> expect(:request, fn :get, url, _, _, _ -> + Registry.register(Pleroma.ReverseProxy.ClientMock, url, 0) + + {:ok, 200, + [ + {"content-type", "application/json"}, + {"content-length", byte_size(json) |> to_string()} + ], %{url: url}} + end) + |> expect(:stream_body, invokes, fn %{url: url} -> + case Registry.lookup(Pleroma.ReverseProxy.ClientMock, url) do + [{_, 0}] -> + Registry.update_value(Pleroma.ReverseProxy.ClientMock, url, &(&1 + 1)) + {:ok, json} + + [{_, 1}] -> + Registry.unregister(Pleroma.ReverseProxy.ClientMock, url) + :done + end + end) + end + + describe "user-agent" do + test "don't keep", %{conn: conn} do + user_agent_mock("hackney/1.15.1", 2) + conn = ReverseProxy.call(conn, "/user-agent") + assert json_response(conn, 200) == %{"user-agent" => "hackney/1.15.1"} + end + + test "keep", %{conn: conn} do + user_agent_mock(Pleroma.Application.user_agent(), 2) + conn = ReverseProxy.call(conn, "/user-agent-keep", keep_user_agent: true) + assert json_response(conn, 200) == %{"user-agent" => Pleroma.Application.user_agent()} + end + end + + test "closed connection", %{conn: conn} do + ClientMock + |> expect(:request, fn :get, "/closed", _, _, _ -> {:ok, 200, [], %{}} end) + |> expect(:stream_body, fn _ -> {:error, :closed} end) + |> expect(:close, fn _ -> :ok end) + + conn = ReverseProxy.call(conn, "/closed") + assert conn.halted + end + + describe "max_body " do + test "length returns error if content-length more than option", %{conn: conn} do + user_agent_mock("hackney/1.15.1", 0) + + assert capture_log(fn -> + ReverseProxy.call(conn, "/user-agent", max_body_length: 4) + end) =~ + "[error] Elixir.Pleroma.ReverseProxy: request to \"/user-agent\" failed: :body_too_large" + end + + defp stream_mock(invokes, with_close? \\ false) do + ClientMock + |> expect(:request, fn :get, "/stream-bytes/" <> length, _, _, _ -> + Registry.register(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length, 0) + + {:ok, 200, [{"content-type", "application/octet-stream"}], + %{url: "/stream-bytes/" <> length}} + end) + |> expect(:stream_body, invokes, fn %{url: "/stream-bytes/" <> length} -> + max = String.to_integer(length) + + case Registry.lookup(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length) do + [{_, current}] when current < max -> + Registry.update_value( + Pleroma.ReverseProxy.ClientMock, + "/stream-bytes/" <> length, + &(&1 + 10) + ) + + {:ok, "0123456789"} + + [{_, ^max}] -> + Registry.unregister(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length) + :done + end + end) + + if with_close? do + expect(ClientMock, :close, fn _ -> :ok end) + end + end + + test "max_body_size returns error if streaming body more than that option", %{conn: conn} do + stream_mock(3, true) + + assert capture_log(fn -> + ReverseProxy.call(conn, "/stream-bytes/50", max_body_size: 30) + end) =~ + "[warn] Elixir.Pleroma.ReverseProxy request to /stream-bytes/50 failed while reading/chunking: :body_too_large" + end + end + + describe "HEAD requests" do + test "common", %{conn: conn} do + ClientMock + |> expect(:request, fn :head, "/head", _, _, _ -> + {:ok, 200, [{"content-type", "text/html; charset=utf-8"}]} + end) + + conn = ReverseProxy.call(Map.put(conn, :method, "HEAD"), "/head") + assert html_response(conn, 200) == "" + end + end + + defp error_mock(status) when is_integer(status) do + ClientMock + |> expect(:request, fn :get, "/status/" <> _, _, _, _ -> + {:error, status} + end) + end + + describe "returns error on" do + test "500", %{conn: conn} do + error_mock(500) + + capture_log(fn -> ReverseProxy.call(conn, "/status/500") end) =~ + "[error] Elixir.Pleroma.ReverseProxy: request to /status/500 failed with HTTP status 500" + end + + test "400", %{conn: conn} do + error_mock(400) + + capture_log(fn -> ReverseProxy.call(conn, "/status/400") end) =~ + "[error] Elixir.Pleroma.ReverseProxy: request to /status/400 failed with HTTP status 400" + end + + test "204", %{conn: conn} do + ClientMock + |> expect(:request, fn :get, "/status/204", _, _, _ -> {:ok, 204, [], %{}} end) + + capture_log(fn -> + conn = ReverseProxy.call(conn, "/status/204") + assert conn.resp_body == "Request failed: No Content" + assert conn.halted + end) =~ + "[error] Elixir.Pleroma.ReverseProxy: request to \"/status/204\" failed with HTTP status 204" + end + end + + test "streaming", %{conn: conn} do + stream_mock(21) + conn = ReverseProxy.call(conn, "/stream-bytes/200") + assert conn.state == :chunked + assert byte_size(conn.resp_body) == 200 + assert Plug.Conn.get_resp_header(conn, "content-type") == ["application/octet-stream"] + end + + defp headers_mock(_) do + ClientMock + |> expect(:request, fn :get, "/headers", headers, _, _ -> + Registry.register(Pleroma.ReverseProxy.ClientMock, "/headers", 0) + {:ok, 200, [{"content-type", "application/json"}], %{url: "/headers", headers: headers}} + end) + |> expect(:stream_body, 2, fn %{url: url, headers: headers} -> + case Registry.lookup(Pleroma.ReverseProxy.ClientMock, url) do + [{_, 0}] -> + Registry.update_value(Pleroma.ReverseProxy.ClientMock, url, &(&1 + 1)) + headers = for {k, v} <- headers, into: %{}, do: {String.capitalize(k), v} + {:ok, Jason.encode!(%{headers: headers})} + + [{_, 1}] -> + Registry.unregister(Pleroma.ReverseProxy.ClientMock, url) + :done + end + end) + + :ok + end + + describe "keep request headers" do + setup [:headers_mock] + + test "header passes", %{conn: conn} do + conn = + Plug.Conn.put_req_header( + conn, + "accept", + "text/html" + ) + |> ReverseProxy.call("/headers") + + %{"headers" => headers} = json_response(conn, 200) + assert headers["Accept"] == "text/html" + end + + test "header is filtered", %{conn: conn} do + conn = + Plug.Conn.put_req_header( + conn, + "accept-language", + "en-US" + ) + |> ReverseProxy.call("/headers") + + %{"headers" => headers} = json_response(conn, 200) + refute headers["Accept-Language"] + end + end + + test "returns 400 on non GET, HEAD requests", %{conn: conn} do + conn = ReverseProxy.call(Map.put(conn, :method, "POST"), "/ip") + assert conn.status == 400 + end + + describe "cache resp headers" do + test "returns headers", %{conn: conn} do + ClientMock + |> expect(:request, fn :get, "/cache/" <> ttl, _, _, _ -> + {:ok, 200, [{"cache-control", "public, max-age=" <> ttl}], %{}} + end) + |> expect(:stream_body, fn _ -> :done end) + + conn = ReverseProxy.call(conn, "/cache/10") + assert {"cache-control", "public, max-age=10"} in conn.resp_headers + end + + test "add cache-control", %{conn: conn} do + ClientMock + |> expect(:request, fn :get, "/cache", _, _, _ -> + {:ok, 200, [{"ETag", "some ETag"}], %{}} + end) + |> expect(:stream_body, fn _ -> :done end) + + conn = ReverseProxy.call(conn, "/cache") + assert {"cache-control", "public"} in conn.resp_headers + end + end + + defp disposition_headers_mock(headers) do + ClientMock + |> expect(:request, fn :get, "/disposition", _, _, _ -> + Registry.register(Pleroma.ReverseProxy.ClientMock, "/disposition", 0) + + {:ok, 200, headers, %{url: "/disposition"}} + end) + |> expect(:stream_body, 2, fn %{url: "/disposition"} -> + case Registry.lookup(Pleroma.ReverseProxy.ClientMock, "/disposition") do + [{_, 0}] -> + Registry.update_value(Pleroma.ReverseProxy.ClientMock, "/disposition", &(&1 + 1)) + {:ok, ""} + + [{_, 1}] -> + Registry.unregister(Pleroma.ReverseProxy.ClientMock, "/disposition") + :done + end + end) + end + + describe "response content disposition header" do + test "not atachment", %{conn: conn} do + disposition_headers_mock([ + {"content-type", "image/gif"}, + {"content-length", 0} + ]) + + conn = ReverseProxy.call(conn, "/disposition") + + assert {"content-type", "image/gif"} in conn.resp_headers + end + + test "with content-disposition header", %{conn: conn} do + disposition_headers_mock([ + {"content-disposition", "attachment; filename=\"filename.jpg\""}, + {"content-length", 0} + ]) + + conn = ReverseProxy.call(conn, "/disposition") + + assert {"content-disposition", "attachment; filename=\"filename.jpg\""} in conn.resp_headers + end + end +end diff --git a/test/tasks/ecto/ecto_test.exs b/test/tasks/ecto/ecto_test.exs new file mode 100644 index 000000000..b48662c88 --- /dev/null +++ b/test/tasks/ecto/ecto_test.exs @@ -0,0 +1,11 @@ +defmodule Mix.Tasks.Pleroma.EctoTest do + use ExUnit.Case, async: true + + test "raise on bad path" do + assert_raise RuntimeError, ~r/Could not find migrations directory/, fn -> + Mix.Tasks.Pleroma.Ecto.ensure_migrations_path(Pleroma.Repo, + migrations_path: "some-path" + ) + end + end +end diff --git a/test/tasks/pleroma_test.exs b/test/tasks/pleroma_test.exs new file mode 100644 index 000000000..e236ccbbb --- /dev/null +++ b/test/tasks/pleroma_test.exs @@ -0,0 +1,46 @@ +defmodule Mix.PleromaTest do + use ExUnit.Case, async: true + import Mix.Pleroma + + setup_all do + Mix.shell(Mix.Shell.Process) + + on_exit(fn -> + Mix.shell(Mix.Shell.IO) + end) + + :ok + end + + describe "shell_prompt/1" do + test "input" do + send(self(), {:mix_shell_input, :prompt, "Yes"}) + + answer = shell_prompt("Do you want this?") + assert_received {:mix_shell, :prompt, [message]} + assert message =~ "Do you want this?" + assert answer == "Yes" + end + + test "with defval" do + send(self(), {:mix_shell_input, :prompt, "\n"}) + + answer = shell_prompt("Do you want this?", "defval") + + assert_received {:mix_shell, :prompt, [message]} + assert message =~ "Do you want this? [defval]" + assert answer == "defval" + end + end + + describe "get_option/3" do + test "get from options" do + assert get_option([domain: "some-domain.com"], :domain, "Promt") == "some-domain.com" + end + + test "get from prompt" do + send(self(), {:mix_shell_input, :prompt, "another-domain.com"}) + assert get_option([], :domain, "Prompt") == "another-domain.com" + end + end +end diff --git a/test/tasks/robots_txt_test.exs b/test/tasks/robots_txt_test.exs new file mode 100644 index 000000000..539193f73 --- /dev/null +++ b/test/tasks/robots_txt_test.exs @@ -0,0 +1,43 @@ +defmodule Mix.Tasks.Pleroma.RobotsTxtTest do + use ExUnit.Case, async: true + alias Mix.Tasks.Pleroma.RobotsTxt + + test "creates new dir" do + path = "test/fixtures/new_dir/" + file_path = path <> "robots.txt" + + static_dir = Pleroma.Config.get([:instance, :static_dir]) + Pleroma.Config.put([:instance, :static_dir], path) + + on_exit(fn -> + Pleroma.Config.put([:instance, :static_dir], static_dir) + {:ok, ["test/fixtures/new_dir/", "test/fixtures/new_dir/robots.txt"]} = File.rm_rf(path) + end) + + RobotsTxt.run(["disallow_all"]) + + assert File.exists?(file_path) + {:ok, file} = File.read(file_path) + + assert file == "User-Agent: *\nDisallow: /\n" + end + + test "to existance folder" do + path = "test/fixtures/" + file_path = path <> "robots.txt" + static_dir = Pleroma.Config.get([:instance, :static_dir]) + Pleroma.Config.put([:instance, :static_dir], path) + + on_exit(fn -> + Pleroma.Config.put([:instance, :static_dir], static_dir) + :ok = File.rm(file_path) + end) + + RobotsTxt.run(["disallow_all"]) + + assert File.exists?(file_path) + {:ok, file} = File.read(file_path) + + assert file == "User-Agent: *\nDisallow: /\n" + end +end diff --git a/test/test_helper.exs b/test/test_helper.exs index f604ba63d..3e33f0335 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -3,6 +3,6 @@ # SPDX-License-Identifier: AGPL-3.0-only ExUnit.start() - Ecto.Adapters.SQL.Sandbox.mode(Pleroma.Repo, :manual) +Mox.defmock(Pleroma.ReverseProxy.ClientMock, for: Pleroma.ReverseProxy.Client) {:ok, _} = Application.ensure_all_started(:ex_machina) From d6b0fce6e944e8a3dd05091ef2388c610362f824 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 9 Jul 2019 17:36:35 +0000 Subject: [PATCH 43/43] Fix/1019 correct count remote users --- CHANGELOG.md | 1 + config/config.exs | 9 +- docs/config.md | 6 + lib/pleroma/application.ex | 6 +- lib/pleroma/user.ex | 66 +++++++++- lib/pleroma/user/query.ex | 19 ++- lib/pleroma/user/synchronization.ex | 60 +++++++++ lib/pleroma/user/synchronization_worker.ex | 32 +++++ .../users_mock/masto_closed_followers.json | 7 ++ .../users_mock/masto_closed_following.json | 7 ++ .../users_mock/pleroma_followers.json | 20 +++ .../users_mock/pleroma_following.json | 20 +++ test/support/http_request_mock.ex | 48 +++++++ test/user/synchronization_test.exs | 104 ++++++++++++++++ test/user/synchronization_worker_test.exs | 49 ++++++++ test/user_test.exs | 117 ++++++++++++++++++ 16 files changed, 564 insertions(+), 7 deletions(-) create mode 100644 lib/pleroma/user/synchronization.ex create mode 100644 lib/pleroma/user/synchronization_worker.ex create mode 100644 test/fixtures/users_mock/masto_closed_followers.json create mode 100644 test/fixtures/users_mock/masto_closed_following.json create mode 100644 test/fixtures/users_mock/pleroma_followers.json create mode 100644 test/fixtures/users_mock/pleroma_following.json create mode 100644 test/user/synchronization_test.exs create mode 100644 test/user/synchronization_worker_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index 75fa50e00..2d59639bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ Configuration: `federation_incoming_replies_max_depth` option - Admin API: Return users' tags when querying reports - Admin API: Return avatar and display name when querying users - Mastodon API, streaming: Add support for passing the token in the `Sec-WebSocket-Protocol` header +- Added synchronization of following/followers counters for external users ### Fixed - Not being able to pin unlisted posts diff --git a/config/config.exs b/config/config.exs index 675fbb551..09681f122 100644 --- a/config/config.exs +++ b/config/config.exs @@ -249,7 +249,14 @@ config :pleroma, :instance, remote_post_retention_days: 90, skip_thread_containment: true, limit_to_local_content: :unauthenticated, - dynamic_configuration: false + dynamic_configuration: false, + external_user_synchronization: [ + enabled: false, + # every 2 hours + interval: 60 * 60 * 2, + max_retries: 3, + limit: 500 + ] config :pleroma, :markup, # XXX - unfortunately, inline images must be enabled by default right now, because diff --git a/docs/config.md b/docs/config.md index 822c34c51..931155fe9 100644 --- a/docs/config.md +++ b/docs/config.md @@ -125,6 +125,12 @@ config :pleroma, Pleroma.Emails.Mailer, * `skip_thread_containment`: Skip filter out broken threads. The default is `false`. * `limit_to_local_content`: Limit unauthenticated users to search for local statutes and users only. Possible values: `:unauthenticated`, `:all` and `false`. The default is `:unauthenticated`. * `dynamic_configuration`: Allow transferring configuration to DB with the subsequent customization from Admin api. +* `external_user_synchronization`: Following/followers counters synchronization settings. + * `enabled`: Enables synchronization + * `interval`: Interval between synchronization. + * `max_retries`: Max rettries for host. After exceeding the limit, the check will not be carried out for users from this host. + * `limit`: Users batch size for processing in one time. + ## :logger diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index ba4cf8486..86c348a0d 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -151,7 +151,11 @@ defmodule Pleroma.Application do start: {Pleroma.Web.Endpoint, :start_link, []}, type: :supervisor }, - %{id: Pleroma.Gopher.Server, start: {Pleroma.Gopher.Server, :start_link, []}} + %{id: Pleroma.Gopher.Server, start: {Pleroma.Gopher.Server, :start_link, []}}, + %{ + id: Pleroma.User.SynchronizationWorker, + start: {Pleroma.User.SynchronizationWorker, :start_link, []} + } ] # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 09f86aaa2..d03810d1a 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -107,15 +107,25 @@ defmodule Pleroma.User do def ap_followers(%User{follower_address: fa}) when is_binary(fa), do: fa def ap_followers(%User{} = user), do: "#{ap_id(user)}/followers" - def user_info(%User{} = user) do + def user_info(%User{} = user, args \\ %{}) do + following_count = + if args[:following_count], do: args[:following_count], else: following_count(user) + + follower_count = + if args[:follower_count], do: args[:follower_count], else: user.info.follower_count + %{ - following_count: following_count(user), note_count: user.info.note_count, - follower_count: user.info.follower_count, locked: user.info.locked, confirmation_pending: user.info.confirmation_pending, default_scope: user.info.default_scope } + |> Map.put(:following_count, following_count) + |> Map.put(:follower_count, follower_count) + end + + def set_info_cache(user, args) do + Cachex.put(:user_cache, "user_info:#{user.id}", user_info(user, args)) end def restrict_deactivated(query) do @@ -1000,6 +1010,56 @@ defmodule Pleroma.User do ) end + @spec sync_follow_counter() :: :ok + def sync_follow_counter, + do: PleromaJobQueue.enqueue(:background, __MODULE__, [:sync_follow_counters]) + + @spec perform(:sync_follow_counters) :: :ok + def perform(:sync_follow_counters) do + {:ok, _pid} = Agent.start_link(fn -> %{} end, name: :domain_errors) + config = Pleroma.Config.get([:instance, :external_user_synchronization]) + + :ok = sync_follow_counters(config) + Agent.stop(:domain_errors) + end + + @spec sync_follow_counters(keyword()) :: :ok + def sync_follow_counters(opts \\ []) do + users = external_users(opts) + + if length(users) > 0 do + errors = Agent.get(:domain_errors, fn state -> state end) + {last, updated_errors} = User.Synchronization.call(users, errors, opts) + Agent.update(:domain_errors, fn _state -> updated_errors end) + sync_follow_counters(max_id: last.id, limit: opts[:limit]) + else + :ok + end + end + + @spec external_users(keyword()) :: [User.t()] + def external_users(opts \\ []) do + query = + User.Query.build(%{ + external: true, + active: true, + order_by: :id, + select: [:id, :ap_id, :info] + }) + + query = + if opts[:max_id], + do: where(query, [u], u.id > ^opts[:max_id]), + else: query + + query = + if opts[:limit], + do: limit(query, ^opts[:limit]), + else: query + + Repo.all(query) + end + def blocks_import(%User{} = blocker, blocked_identifiers) when is_list(blocked_identifiers), do: PleromaJobQueue.enqueue(:background, __MODULE__, [ diff --git a/lib/pleroma/user/query.ex b/lib/pleroma/user/query.ex index ace9c05f2..f9bcc9e19 100644 --- a/lib/pleroma/user/query.ex +++ b/lib/pleroma/user/query.ex @@ -7,7 +7,7 @@ defmodule Pleroma.User.Query do User query builder module. Builds query from new query or another user query. ## Example: - query = Pleroma.User.Query(%{nickname: "nickname"}) + query = Pleroma.User.Query.build(%{nickname: "nickname"}) another_query = Pleroma.User.Query.build(query, %{email: "email@example.com"}) Pleroma.Repo.all(query) Pleroma.Repo.all(another_query) @@ -47,7 +47,10 @@ defmodule Pleroma.User.Query do friends: User.t(), recipients_from_activity: [String.t()], nickname: [String.t()], - ap_id: [String.t()] + ap_id: [String.t()], + order_by: term(), + select: term(), + limit: pos_integer() } | %{} @@ -141,6 +144,18 @@ defmodule Pleroma.User.Query do where(query, [u], u.ap_id in ^to or fragment("? && ?", u.following, ^to)) end + defp compose_query({:order_by, key}, query) do + order_by(query, [u], field(u, ^key)) + end + + defp compose_query({:select, keys}, query) do + select(query, [u], ^keys) + end + + defp compose_query({:limit, limit}, query) do + limit(query, ^limit) + end + defp compose_query(_unsupported_param, query), do: query defp prepare_tag_criteria(tag, query) do diff --git a/lib/pleroma/user/synchronization.ex b/lib/pleroma/user/synchronization.ex new file mode 100644 index 000000000..93660e08c --- /dev/null +++ b/lib/pleroma/user/synchronization.ex @@ -0,0 +1,60 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.User.Synchronization do + alias Pleroma.HTTP + alias Pleroma.User + + @spec call([User.t()], map(), keyword()) :: {User.t(), map()} + def call(users, errors, opts \\ []) do + do_call(users, errors, opts) + end + + defp do_call([user | []], errors, opts) do + updated = fetch_counters(user, errors, opts) + {user, updated} + end + + defp do_call([user | others], errors, opts) do + updated = fetch_counters(user, errors, opts) + do_call(others, updated, opts) + end + + defp fetch_counters(user, errors, opts) do + %{host: host} = URI.parse(user.ap_id) + + info = %{} + {following, errors} = fetch_counter(user.ap_id <> "/following", host, errors, opts) + info = if following, do: Map.put(info, :following_count, following), else: info + + {followers, errors} = fetch_counter(user.ap_id <> "/followers", host, errors, opts) + info = if followers, do: Map.put(info, :follower_count, followers), else: info + + User.set_info_cache(user, info) + errors + end + + defp available_domain?(domain, errors, opts) do + max_retries = Keyword.get(opts, :max_retries, 3) + not (Map.has_key?(errors, domain) && errors[domain] >= max_retries) + end + + defp fetch_counter(url, host, errors, opts) do + with true <- available_domain?(host, errors, opts), + {:ok, %{body: body, status: code}} when code in 200..299 <- + HTTP.get( + url, + [{:Accept, "application/activity+json"}] + ), + {:ok, data} <- Jason.decode(body) do + {data["totalItems"], errors} + else + false -> + {nil, errors} + + _ -> + {nil, Map.update(errors, host, 1, &(&1 + 1))} + end + end +end diff --git a/lib/pleroma/user/synchronization_worker.ex b/lib/pleroma/user/synchronization_worker.ex new file mode 100644 index 000000000..ba9cc3556 --- /dev/null +++ b/lib/pleroma/user/synchronization_worker.ex @@ -0,0 +1,32 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-onl + +defmodule Pleroma.User.SynchronizationWorker do + use GenServer + + def start_link do + config = Pleroma.Config.get([:instance, :external_user_synchronization]) + + if config[:enabled] do + GenServer.start_link(__MODULE__, interval: config[:interval]) + else + :ignore + end + end + + def init(opts) do + schedule_next(opts) + {:ok, opts} + end + + def handle_info(:sync_follow_counters, opts) do + Pleroma.User.sync_follow_counter() + schedule_next(opts) + {:noreply, opts} + end + + defp schedule_next(opts) do + Process.send_after(self(), :sync_follow_counters, opts[:interval]) + end +end diff --git a/test/fixtures/users_mock/masto_closed_followers.json b/test/fixtures/users_mock/masto_closed_followers.json new file mode 100644 index 000000000..da296892d --- /dev/null +++ b/test/fixtures/users_mock/masto_closed_followers.json @@ -0,0 +1,7 @@ +{ + "@context": "https://www.w3.org/ns/activitystreams", + "id": "http://localhost:4001/users/masto_closed/followers", + "type": "OrderedCollection", + "totalItems": 437, + "first": "http://localhost:4001/users/masto_closed/followers?page=1" +} diff --git a/test/fixtures/users_mock/masto_closed_following.json b/test/fixtures/users_mock/masto_closed_following.json new file mode 100644 index 000000000..146d49f9c --- /dev/null +++ b/test/fixtures/users_mock/masto_closed_following.json @@ -0,0 +1,7 @@ +{ + "@context": "https://www.w3.org/ns/activitystreams", + "id": "http://localhost:4001/users/masto_closed/following", + "type": "OrderedCollection", + "totalItems": 152, + "first": "http://localhost:4001/users/masto_closed/following?page=1" +} diff --git a/test/fixtures/users_mock/pleroma_followers.json b/test/fixtures/users_mock/pleroma_followers.json new file mode 100644 index 000000000..db71d084b --- /dev/null +++ b/test/fixtures/users_mock/pleroma_followers.json @@ -0,0 +1,20 @@ +{ + "type": "OrderedCollection", + "totalItems": 527, + "id": "http://localhost:4001/users/fuser2/followers", + "first": { + "type": "OrderedCollectionPage", + "totalItems": 527, + "partOf": "http://localhost:4001/users/fuser2/followers", + "orderedItems": [], + "next": "http://localhost:4001/users/fuser2/followers?page=2", + "id": "http://localhost:4001/users/fuser2/followers?page=1" + }, + "@context": [ + "https://www.w3.org/ns/activitystreams", + "http://localhost:4001/schemas/litepub-0.1.jsonld", + { + "@language": "und" + } + ] +} diff --git a/test/fixtures/users_mock/pleroma_following.json b/test/fixtures/users_mock/pleroma_following.json new file mode 100644 index 000000000..33d087703 --- /dev/null +++ b/test/fixtures/users_mock/pleroma_following.json @@ -0,0 +1,20 @@ +{ + "type": "OrderedCollection", + "totalItems": 267, + "id": "http://localhost:4001/users/fuser2/following", + "first": { + "type": "OrderedCollectionPage", + "totalItems": 267, + "partOf": "http://localhost:4001/users/fuser2/following", + "orderedItems": [], + "next": "http://localhost:4001/users/fuser2/following?page=2", + "id": "http://localhost:4001/users/fuser2/following?page=1" + }, + "@context": [ + "https://www.w3.org/ns/activitystreams", + "http://localhost:4001/schemas/litepub-0.1.jsonld", + { + "@language": "und" + } + ] +} diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex index e6f357412..c593a5e4a 100644 --- a/test/support/http_request_mock.ex +++ b/test/support/http_request_mock.ex @@ -759,6 +759,54 @@ defmodule HttpRequestMock do {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/ogp.html")}} end + def get("http://localhost:4001/users/masto_closed/followers", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/users_mock/masto_closed_followers.json") + }} + end + + def get("http://localhost:4001/users/masto_closed/following", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/users_mock/masto_closed_following.json") + }} + end + + def get("http://localhost:4001/users/fuser2/followers", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/users_mock/pleroma_followers.json") + }} + end + + def get("http://localhost:4001/users/fuser2/following", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/users_mock/pleroma_following.json") + }} + end + + def get("http://domain-with-errors:4001/users/fuser1/followers", _, _, _) do + {:ok, + %Tesla.Env{ + status: 504, + body: "" + }} + end + + def get("http://domain-with-errors:4001/users/fuser1/following", _, _, _) do + {:ok, + %Tesla.Env{ + status: 504, + body: "" + }} + end + def get("http://example.com/ogp-missing-data", _, _, _) do {:ok, %Tesla.Env{ diff --git a/test/user/synchronization_test.exs b/test/user/synchronization_test.exs new file mode 100644 index 000000000..67b669431 --- /dev/null +++ b/test/user/synchronization_test.exs @@ -0,0 +1,104 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.User.SynchronizationTest do + use Pleroma.DataCase + import Pleroma.Factory + alias Pleroma.User + alias Pleroma.User.Synchronization + + setup do + Tesla.Mock.mock(fn env -> apply(HttpRequestMock, :request, [env]) end) + :ok + end + + test "update following/followers counters" do + user1 = + insert(:user, + local: false, + ap_id: "http://localhost:4001/users/masto_closed" + ) + + user2 = insert(:user, local: false, ap_id: "http://localhost:4001/users/fuser2") + + users = User.external_users() + assert length(users) == 2 + {user, %{}} = Synchronization.call(users, %{}) + assert user == List.last(users) + + %{follower_count: followers, following_count: following} = User.get_cached_user_info(user1) + assert followers == 437 + assert following == 152 + + %{follower_count: followers, following_count: following} = User.get_cached_user_info(user2) + + assert followers == 527 + assert following == 267 + end + + test "don't check host if errors exist" do + user1 = insert(:user, local: false, ap_id: "http://domain-with-errors:4001/users/fuser1") + + user2 = insert(:user, local: false, ap_id: "http://domain-with-errors:4001/users/fuser2") + + users = User.external_users() + assert length(users) == 2 + + {user, %{"domain-with-errors" => 2}} = + Synchronization.call(users, %{"domain-with-errors" => 2}, max_retries: 2) + + assert user == List.last(users) + + %{follower_count: followers, following_count: following} = User.get_cached_user_info(user1) + assert followers == 0 + assert following == 0 + + %{follower_count: followers, following_count: following} = User.get_cached_user_info(user2) + + assert followers == 0 + assert following == 0 + end + + test "don't check host if errors appeared" do + user1 = insert(:user, local: false, ap_id: "http://domain-with-errors:4001/users/fuser1") + + user2 = insert(:user, local: false, ap_id: "http://domain-with-errors:4001/users/fuser2") + + users = User.external_users() + assert length(users) == 2 + + {user, %{"domain-with-errors" => 2}} = Synchronization.call(users, %{}, max_retries: 2) + + assert user == List.last(users) + + %{follower_count: followers, following_count: following} = User.get_cached_user_info(user1) + assert followers == 0 + assert following == 0 + + %{follower_count: followers, following_count: following} = User.get_cached_user_info(user2) + + assert followers == 0 + assert following == 0 + end + + test "other users after error appeared" do + user1 = insert(:user, local: false, ap_id: "http://domain-with-errors:4001/users/fuser1") + user2 = insert(:user, local: false, ap_id: "http://localhost:4001/users/fuser2") + + users = User.external_users() + assert length(users) == 2 + + {user, %{"domain-with-errors" => 2}} = Synchronization.call(users, %{}, max_retries: 2) + assert user == List.last(users) + + %{follower_count: followers, following_count: following} = User.get_cached_user_info(user1) + assert followers == 0 + assert following == 0 + + %{follower_count: followers, following_count: following} = User.get_cached_user_info(user2) + + assert followers == 527 + assert following == 267 + end +end diff --git a/test/user/synchronization_worker_test.exs b/test/user/synchronization_worker_test.exs new file mode 100644 index 000000000..835c5327f --- /dev/null +++ b/test/user/synchronization_worker_test.exs @@ -0,0 +1,49 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.User.SynchronizationWorkerTest do + use Pleroma.DataCase + import Pleroma.Factory + + setup do + Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) + + config = Pleroma.Config.get([:instance, :external_user_synchronization]) + + for_update = [enabled: true, interval: 1000] + + Pleroma.Config.put([:instance, :external_user_synchronization], for_update) + + on_exit(fn -> + Pleroma.Config.put([:instance, :external_user_synchronization], config) + end) + + :ok + end + + test "sync follow counters" do + user1 = + insert(:user, + local: false, + ap_id: "http://localhost:4001/users/masto_closed" + ) + + user2 = insert(:user, local: false, ap_id: "http://localhost:4001/users/fuser2") + + {:ok, _} = Pleroma.User.SynchronizationWorker.start_link() + :timer.sleep(1500) + + %{follower_count: followers, following_count: following} = + Pleroma.User.get_cached_user_info(user1) + + assert followers == 437 + assert following == 152 + + %{follower_count: followers, following_count: following} = + Pleroma.User.get_cached_user_info(user2) + + assert followers == 527 + assert following == 267 + end +end diff --git a/test/user_test.exs b/test/user_test.exs index fb497843c..0f27d73f7 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -1183,4 +1183,121 @@ defmodule Pleroma.UserTest do assert user_two.ap_id in ap_ids end end + + describe "sync followers count" do + setup do + user1 = insert(:user, local: false, ap_id: "http://localhost:4001/users/masto_closed") + user2 = insert(:user, local: false, ap_id: "http://localhost:4001/users/fuser2") + insert(:user, local: true) + insert(:user, local: false, info: %{deactivated: true}) + {:ok, user1: user1, user2: user2} + end + + test "external_users/1 external active users with limit", %{user1: user1, user2: user2} do + [fdb_user1] = User.external_users(limit: 1) + + assert fdb_user1.ap_id + assert fdb_user1.ap_id == user1.ap_id + assert fdb_user1.id == user1.id + + [fdb_user2] = User.external_users(max_id: fdb_user1.id, limit: 1) + + assert fdb_user2.ap_id + assert fdb_user2.ap_id == user2.ap_id + assert fdb_user2.id == user2.id + + assert User.external_users(max_id: fdb_user2.id, limit: 1) == [] + end + + test "sync_follow_counters/1", %{user1: user1, user2: user2} do + {:ok, _pid} = Agent.start_link(fn -> %{} end, name: :domain_errors) + + :ok = User.sync_follow_counters() + + %{follower_count: followers, following_count: following} = User.get_cached_user_info(user1) + assert followers == 437 + assert following == 152 + + %{follower_count: followers, following_count: following} = User.get_cached_user_info(user2) + + assert followers == 527 + assert following == 267 + + Agent.stop(:domain_errors) + end + + test "sync_follow_counters/1 in separate batches", %{user1: user1, user2: user2} do + {:ok, _pid} = Agent.start_link(fn -> %{} end, name: :domain_errors) + + :ok = User.sync_follow_counters(limit: 1) + + %{follower_count: followers, following_count: following} = User.get_cached_user_info(user1) + assert followers == 437 + assert following == 152 + + %{follower_count: followers, following_count: following} = User.get_cached_user_info(user2) + + assert followers == 527 + assert following == 267 + + Agent.stop(:domain_errors) + end + + test "perform/1 with :sync_follow_counters", %{user1: user1, user2: user2} do + :ok = User.perform(:sync_follow_counters) + %{follower_count: followers, following_count: following} = User.get_cached_user_info(user1) + assert followers == 437 + assert following == 152 + + %{follower_count: followers, following_count: following} = User.get_cached_user_info(user2) + + assert followers == 527 + assert following == 267 + end + end + + describe "set_info_cache/2" do + setup do + user = insert(:user) + {:ok, user: user} + end + + test "update from args", %{user: user} do + User.set_info_cache(user, %{following_count: 15, follower_count: 18}) + + %{follower_count: followers, following_count: following} = User.get_cached_user_info(user) + assert followers == 18 + assert following == 15 + end + + test "without args", %{user: user} do + User.set_info_cache(user, %{}) + + %{follower_count: followers, following_count: following} = User.get_cached_user_info(user) + assert followers == 0 + assert following == 0 + end + end + + describe "user_info/2" do + setup do + user = insert(:user) + {:ok, user: user} + end + + test "update from args", %{user: user} do + %{follower_count: followers, following_count: following} = + User.user_info(user, %{following_count: 15, follower_count: 18}) + + assert followers == 18 + assert following == 15 + end + + test "without args", %{user: user} do + %{follower_count: followers, following_count: following} = User.user_info(user) + + assert followers == 0 + assert following == 0 + end + end end