From 452ca5250d7f7eeb51804d885122a60c8c4b84bf Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Fri, 28 Jun 2019 15:15:32 +0300 Subject: [PATCH 01/14] Merge admin and mastodon user views for reports --- .../web/admin_api/views/account_view.ex | 8 ++ .../web/admin_api/views/report_view.ex | 10 ++- .../admin_api/admin_api_controller_test.exs | 86 ++++++++++++++----- test/web/admin_api/views/report_view_test.exs | 24 +++++- 4 files changed, 100 insertions(+), 28 deletions(-) diff --git a/lib/pleroma/web/admin_api/views/account_view.ex b/lib/pleroma/web/admin_api/views/account_view.ex index 28bb667d8..7e1b9c431 100644 --- a/lib/pleroma/web/admin_api/views/account_view.ex +++ b/lib/pleroma/web/admin_api/views/account_view.ex @@ -5,8 +5,11 @@ defmodule Pleroma.Web.AdminAPI.AccountView do use Pleroma.Web, :view + alias Pleroma.HTML + alias Pleroma.User alias Pleroma.User.Info alias Pleroma.Web.AdminAPI.AccountView + alias Pleroma.Web.MediaProxy def render("index.json", %{users: users, count: count, page_size: page_size}) do %{ @@ -17,9 +20,14 @@ def render("index.json", %{users: users, count: count, page_size: page_size}) do end def render("show.json", %{user: user}) do + avatar = User.avatar_url(user) |> MediaProxy.url() + display_name = HTML.strip_tags(user.name || user.nickname) + %{ "id" => user.id, + "avatar" => avatar, "nickname" => user.nickname, + "display_name" => display_name, "deactivated" => user.info.deactivated, "local" => user.local, "roles" => Info.roles(user.info), diff --git a/lib/pleroma/web/admin_api/views/report_view.ex b/lib/pleroma/web/admin_api/views/report_view.ex index e7db3a8ff..73ccdc582 100644 --- a/lib/pleroma/web/admin_api/views/report_view.ex +++ b/lib/pleroma/web/admin_api/views/report_view.ex @@ -8,7 +8,6 @@ defmodule Pleroma.Web.AdminAPI.ReportView do alias Pleroma.HTML alias Pleroma.User alias Pleroma.Web.CommonAPI.Utils - alias Pleroma.Web.MastodonAPI.AccountView alias Pleroma.Web.MastodonAPI.StatusView def render("index.json", %{reports: reports}) do @@ -38,12 +37,17 @@ def render("show.json", %{report: report}) do %{ id: report.id, - account: AccountView.render("account.json", %{user: account}), - actor: AccountView.render("account.json", %{user: user}), + account: merge_account_views(account), + actor: merge_account_views(user), content: content, created_at: created_at, statuses: StatusView.render("index.json", %{activities: statuses, as: :activity}), state: report.data["state"] } end + + defp merge_account_views(user) do + Pleroma.Web.MastodonAPI.AccountView.render("account.json", %{user: user}) + |> Map.merge(Pleroma.Web.AdminAPI.AccountView.render("show.json", %{user: user})) + end end diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index 4278ac59d..3c46eff06 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -9,6 +9,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do alias Pleroma.User alias Pleroma.UserInviteToken alias Pleroma.Web.CommonAPI + alias Pleroma.HTML + alias Pleroma.Web.MediaProxy import Pleroma.Factory describe "/api/pleroma/admin/users" do @@ -58,7 +60,9 @@ test "Show", %{conn: conn} do "local" => true, "nickname" => user.nickname, "roles" => %{"admin" => false, "moderator" => false}, - "tags" => [] + "tags" => [], + "avatar" => User.avatar_url(user) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(user.name || user.nickname) } assert expected == json_response(conn, 200) @@ -445,7 +449,9 @@ test "renders users array for the first page", %{conn: conn, admin: admin} do "nickname" => admin.nickname, "roles" => %{"admin" => true, "moderator" => false}, "local" => true, - "tags" => [] + "tags" => [], + "avatar" => User.avatar_url(admin) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(admin.name || admin.nickname) }, %{ "deactivated" => user.info.deactivated, @@ -453,7 +459,9 @@ test "renders users array for the first page", %{conn: conn, admin: admin} do "nickname" => user.nickname, "roles" => %{"admin" => false, "moderator" => false}, "local" => false, - "tags" => ["foo", "bar"] + "tags" => ["foo", "bar"], + "avatar" => User.avatar_url(user) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(user.name || user.nickname) } ] |> Enum.sort_by(& &1["nickname"]) @@ -492,7 +500,9 @@ test "regular search", %{conn: conn} do "nickname" => user.nickname, "roles" => %{"admin" => false, "moderator" => false}, "local" => true, - "tags" => [] + "tags" => [], + "avatar" => User.avatar_url(user) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(user.name || user.nickname) } ] } @@ -514,7 +524,9 @@ test "search by domain", %{conn: conn} do "nickname" => user.nickname, "roles" => %{"admin" => false, "moderator" => false}, "local" => true, - "tags" => [] + "tags" => [], + "avatar" => User.avatar_url(user) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(user.name || user.nickname) } ] } @@ -536,7 +548,9 @@ test "search by full nickname", %{conn: conn} do "nickname" => user.nickname, "roles" => %{"admin" => false, "moderator" => false}, "local" => true, - "tags" => [] + "tags" => [], + "avatar" => User.avatar_url(user) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(user.name || user.nickname) } ] } @@ -558,7 +572,9 @@ test "search by display name", %{conn: conn} do "nickname" => user.nickname, "roles" => %{"admin" => false, "moderator" => false}, "local" => true, - "tags" => [] + "tags" => [], + "avatar" => User.avatar_url(user) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(user.name || user.nickname) } ] } @@ -580,7 +596,9 @@ test "search by email", %{conn: conn} do "nickname" => user.nickname, "roles" => %{"admin" => false, "moderator" => false}, "local" => true, - "tags" => [] + "tags" => [], + "avatar" => User.avatar_url(user) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(user.name || user.nickname) } ] } @@ -602,7 +620,9 @@ test "regular search with page size", %{conn: conn} do "nickname" => user.nickname, "roles" => %{"admin" => false, "moderator" => false}, "local" => true, - "tags" => [] + "tags" => [], + "avatar" => User.avatar_url(user) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(user.name || user.nickname) } ] } @@ -619,7 +639,9 @@ test "regular search with page size", %{conn: conn} do "nickname" => user2.nickname, "roles" => %{"admin" => false, "moderator" => false}, "local" => true, - "tags" => [] + "tags" => [], + "avatar" => User.avatar_url(user2) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(user2.name || user2.nickname) } ] } @@ -646,7 +668,9 @@ test "only local users" do "nickname" => user.nickname, "roles" => %{"admin" => false, "moderator" => false}, "local" => true, - "tags" => [] + "tags" => [], + "avatar" => User.avatar_url(user) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(user.name || user.nickname) } ] } @@ -671,7 +695,9 @@ test "only local users with no query", %{admin: old_admin} do "nickname" => user.nickname, "roles" => %{"admin" => false, "moderator" => false}, "local" => true, - "tags" => [] + "tags" => [], + "avatar" => User.avatar_url(user) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(user.name || user.nickname) }, %{ "deactivated" => admin.info.deactivated, @@ -679,7 +705,9 @@ test "only local users with no query", %{admin: old_admin} do "nickname" => admin.nickname, "roles" => %{"admin" => true, "moderator" => false}, "local" => true, - "tags" => [] + "tags" => [], + "avatar" => User.avatar_url(admin) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(admin.name || admin.nickname) }, %{ "deactivated" => false, @@ -687,7 +715,9 @@ test "only local users with no query", %{admin: old_admin} do "local" => true, "nickname" => old_admin.nickname, "roles" => %{"admin" => true, "moderator" => false}, - "tags" => [] + "tags" => [], + "avatar" => User.avatar_url(old_admin) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(old_admin.name || old_admin.nickname) } ] |> Enum.sort_by(& &1["nickname"]) @@ -714,7 +744,9 @@ test "load only admins", %{conn: conn, admin: admin} do "nickname" => admin.nickname, "roles" => %{"admin" => true, "moderator" => false}, "local" => admin.local, - "tags" => [] + "tags" => [], + "avatar" => User.avatar_url(admin) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(admin.name || admin.nickname) }, %{ "deactivated" => false, @@ -722,7 +754,9 @@ test "load only admins", %{conn: conn, admin: admin} do "nickname" => second_admin.nickname, "roles" => %{"admin" => true, "moderator" => false}, "local" => second_admin.local, - "tags" => [] + "tags" => [], + "avatar" => User.avatar_url(second_admin) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(second_admin.name || second_admin.nickname) } ] |> Enum.sort_by(& &1["nickname"]) @@ -751,7 +785,9 @@ test "load only moderators", %{conn: conn} do "nickname" => moderator.nickname, "roles" => %{"admin" => false, "moderator" => true}, "local" => moderator.local, - "tags" => [] + "tags" => [], + "avatar" => User.avatar_url(moderator) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(moderator.name || moderator.nickname) } ] } @@ -773,7 +809,9 @@ test "load users with tags list", %{conn: conn} do "nickname" => user1.nickname, "roles" => %{"admin" => false, "moderator" => false}, "local" => user1.local, - "tags" => ["first"] + "tags" => ["first"], + "avatar" => User.avatar_url(user1) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(user1.name || user1.nickname) }, %{ "deactivated" => false, @@ -781,7 +819,9 @@ test "load users with tags list", %{conn: conn} do "nickname" => user2.nickname, "roles" => %{"admin" => false, "moderator" => false}, "local" => user2.local, - "tags" => ["second"] + "tags" => ["second"], + "avatar" => User.avatar_url(user2) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(user2.name || user2.nickname) } ] |> Enum.sort_by(& &1["nickname"]) @@ -815,7 +855,9 @@ test "it works with multiple filters" do "nickname" => user.nickname, "roles" => %{"admin" => false, "moderator" => false}, "local" => user.local, - "tags" => [] + "tags" => [], + "avatar" => User.avatar_url(user) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(user.name || user.nickname) } ] } @@ -838,7 +880,9 @@ test "PATCH /api/pleroma/admin/users/:nickname/toggle_activation" do "nickname" => user.nickname, "roles" => %{"admin" => false, "moderator" => false}, "local" => true, - "tags" => [] + "tags" => [], + "avatar" => User.avatar_url(user) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(user.name || user.nickname) } end diff --git a/test/web/admin_api/views/report_view_test.exs b/test/web/admin_api/views/report_view_test.exs index f35f36cac..75d8bb4b5 100644 --- a/test/web/admin_api/views/report_view_test.exs +++ b/test/web/admin_api/views/report_view_test.exs @@ -18,8 +18,16 @@ test "renders a report" do expected = %{ content: nil, - actor: AccountView.render("account.json", %{user: user}), - account: AccountView.render("account.json", %{user: other_user}), + actor: + Map.merge( + AccountView.render("account.json", %{user: user}), + Pleroma.Web.AdminAPI.AccountView.render("show.json", %{user: user}) + ), + account: + Map.merge( + AccountView.render("account.json", %{user: other_user}), + Pleroma.Web.AdminAPI.AccountView.render("show.json", %{user: other_user}) + ), statuses: [], state: "open", id: activity.id @@ -42,8 +50,16 @@ test "includes reported statuses" do expected = %{ content: nil, - actor: AccountView.render("account.json", %{user: user}), - account: AccountView.render("account.json", %{user: other_user}), + actor: + Map.merge( + AccountView.render("account.json", %{user: user}), + Pleroma.Web.AdminAPI.AccountView.render("show.json", %{user: user}) + ), + account: + Map.merge( + AccountView.render("account.json", %{user: other_user}), + Pleroma.Web.AdminAPI.AccountView.render("show.json", %{user: other_user}) + ), statuses: [StatusView.render("status.json", %{activity: activity})], state: "open", id: report_activity.id From 90927e6a9a7b29f3a12335fd8b7ba899dcc0dffd Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Fri, 28 Jun 2019 15:27:18 +0300 Subject: [PATCH 02/14] Kowalski, analysis! --- test/web/admin_api/admin_api_controller_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index 3c46eff06..4ea33a6cc 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -6,10 +6,10 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do use Pleroma.Web.ConnCase alias Pleroma.Activity + alias Pleroma.HTML alias Pleroma.User alias Pleroma.UserInviteToken alias Pleroma.Web.CommonAPI - alias Pleroma.HTML alias Pleroma.Web.MediaProxy import Pleroma.Factory From 034986e1fd389066c1ca4445af0cf8410da746e7 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Thu, 27 Jun 2019 03:06:58 +0000 Subject: [PATCH 03/14] MRF: add mediaproxy warming policy --- .../mrf/mediaproxy_warming_policy.ex | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex diff --git a/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex new file mode 100644 index 000000000..01d21a299 --- /dev/null +++ b/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex @@ -0,0 +1,56 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do + @moduledoc "Preloads any attachments in the MediaProxy cache by prefetching them" + @behaviour Pleroma.Web.ActivityPub.MRF + + alias Pleroma.HTTP + alias Pleroma.Web.MediaProxy + + require Logger + + @hackney_options [ + pool: :media, + recv_timeout: 10_000 + ] + + def perform(:prefetch, url) do + Logger.info("Prefetching #{inspect(url)}") + + url + |> MediaProxy.url() + |> HTTP.get([], adapter: @hackney_options) + end + + def perform(:preload, %{"object" => %{"attachment" => attachments}} = _message) do + Enum.each(attachments, fn + %{"url" => url} when is_list(url) -> + url + |> Enum.each(fn + %{"href" => href} -> + PleromaJobQueue.enqueue(:background, __MODULE__, [:prefetch, href]) + + x -> + Logger.debug("Unhandled attachment URL object #{inspect(x)}") + end) + + x -> + Logger.debug("Unhandled attachment #{inspect(x)}") + end) + end + + @impl true + def filter( + %{"type" => "Create", "object" => %{"attachment" => attachments} = _object} = message + ) + when is_list(attachments) and length(attachments) > 0 do + PleromaJobQueue.enqueue(:background, __MODULE__, [:preload, message]) + + {:ok, message} + end + + @impl true + def filter(message), do: {:ok, message} +end From 074ffee0b2eeae5b3911f47f4ce17b3bbf74152d Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Thu, 27 Jun 2019 21:34:27 +0000 Subject: [PATCH 04/14] docs: add documentation for MediaProxyWarmingPolicy --- CHANGELOG.md | 4 ++++ docs/config.md | 1 + 2 files changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a65988991..a6ec8674d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## [Unreleased] +### Added +- MRF: Support for priming the mediaproxy cache (`Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy`) + ## [1.0.0] - 2019-06-29 ### Security - Mastodon API: Fix display names not being sanitized diff --git a/docs/config.md b/docs/config.md index 7a53364ef..7d5be3980 100644 --- a/docs/config.md +++ b/docs/config.md @@ -98,6 +98,7 @@ config :pleroma, Pleroma.Emails.Mailer, * `Pleroma.Web.ActivityPub.MRF.RejectNonPublic`: Drops posts with non-public visibility settings (See ``:mrf_rejectnonpublic`` section) * `Pleroma.Web.ActivityPub.MRF.EnsureRePrepended`: Rewrites posts to ensure that replies to posts with subjects do not have an identical subject and instead begin with re:. * `Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy`: Rejects posts from likely spambots by rejecting posts from new users that contain links. + * `Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy`: Crawls attachments using their MediaProxy URLs so that the MediaProxy cache is primed. * `public`: Makes the client API in authentificated mode-only except for user-profiles. Useful for disabling the Local Timeline and The Whole Known Network. * `quarantined_instances`: List of ActivityPub instances where private(DMs, followers-only) activities will not be send. * `managed_config`: Whenether the config for pleroma-fe is configured in this config or in ``static/config.json`` From c22f0bfb8056889ea3180e0adbf46458eabf63a7 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Fri, 28 Jun 2019 23:41:42 +0000 Subject: [PATCH 05/14] test: add tests for mediaproxy warming policy --- .../mrf/mediaproxy_warming_policy_test.exs | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 test/web/activity_pub/mrf/mediaproxy_warming_policy_test.exs diff --git a/test/web/activity_pub/mrf/mediaproxy_warming_policy_test.exs b/test/web/activity_pub/mrf/mediaproxy_warming_policy_test.exs new file mode 100644 index 000000000..372e789be --- /dev/null +++ b/test/web/activity_pub/mrf/mediaproxy_warming_policy_test.exs @@ -0,0 +1,45 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicyTest do + use Pleroma.DataCase + + alias Pleroma.HTTP + alias Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy + + import Mock + + @message %{ + "type" => "Create", + "object" => %{ + "type" => "Note", + "content" => "content", + "attachment" => [ + %{"url" => [%{"href" => "http://example.com/image.jpg"}]} + ] + } + } + + test "it prefetches media proxy URIs" do + with_mock HTTP, get: fn _, _, _ -> {:ok, []} end do + MediaProxyWarmingPolicy.filter(@message) + assert called(HTTP.get(:_, :_, :_)) + end + end + + test "it does nothing when no attachments are present" do + object = + @message["object"] + |> Map.delete("attachment") + + message = + @message + |> Map.put("object", object) + + with_mock HTTP, get: fn _, _, _ -> {:ok, []} end do + MediaProxyWarmingPolicy.filter(message) + refute called(HTTP.get(:_, :_, :_)) + end + end +end From b2548e65dbfeb4ec20ff8fdc4230f7aedbb452f0 Mon Sep 17 00:00:00 2001 From: Yuji Nakao Date: Sat, 29 Jun 2019 04:17:49 +0000 Subject: [PATCH 06/14] Update migrating_from_source_otp_en.md: Replace `reload-daemon` with `daemon-reload`. --- docs/installation/migrating_from_source_otp_en.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation/migrating_from_source_otp_en.md b/docs/installation/migrating_from_source_otp_en.md index 0b41d0c0e..df3e2f4a0 100644 --- a/docs/installation/migrating_from_source_otp_en.md +++ b/docs/installation/migrating_from_source_otp_en.md @@ -128,7 +128,7 @@ Debian/Ubuntu: cp ~pleroma/installation/pleroma.service /etc/systemd/system/pleroma.service # Reload service files -systemctl reload-daemon +systemctl daemon-reload # Reenable pleroma to start on boot systemctl reenable pleroma From ab6de9b478bf9c2b46603c2761fd99b6506f8b08 Mon Sep 17 00:00:00 2001 From: Yuji Nakao Date: Sat, 29 Jun 2019 04:24:44 +0000 Subject: [PATCH 07/14] Move all upload contents despite upload folder itself. --- docs/installation/migrating_from_source_otp_en.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation/migrating_from_source_otp_en.md b/docs/installation/migrating_from_source_otp_en.md index 0b41d0c0e..5264790f3 100644 --- a/docs/installation/migrating_from_source_otp_en.md +++ b/docs/installation/migrating_from_source_otp_en.md @@ -49,7 +49,7 @@ mkdir -p /var/lib/pleroma/static chown -R pleroma /var/lib/pleroma # If you use the local uploader with default settings your uploads should be located in `~pleroma/uploads` -mv ~pleroma/uploads /var/lib/pleroma/uploads +mv ~pleroma/uploads/* /var/lib/pleroma/uploads # If you have created the custom public files directory with default settings it should be located in `~pleroma/instance/static` mv ~pleroma/instance/static /var/lib/pleroma/static From eddcebf8ae699edd9e503dd7a3c51112ca063a21 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sat, 29 Jun 2019 11:35:42 +0300 Subject: [PATCH 08/14] Add a warning about service files assuming installation paths --- docs/installation/migrating_from_source_otp_en.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/installation/migrating_from_source_otp_en.md b/docs/installation/migrating_from_source_otp_en.md index d07a0f622..b779be8cc 100644 --- a/docs/installation/migrating_from_source_otp_en.md +++ b/docs/installation/migrating_from_source_otp_en.md @@ -122,6 +122,8 @@ su pleroma -s $SHELL -lc "./bin/pleroma stop" ## Setting up a system service OTP releases have different service files than from-source installs so they need to be copied over again. +**Warning:** The service files assume pleroma user's home directory is `/opt/pleroma`, please make sure all paths fit your installation. + Debian/Ubuntu: ```sh # Copy the service into a proper directory From ee79c67081cef80d2345fdc9b96e925354020671 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sat, 29 Jun 2019 12:29:12 +0300 Subject: [PATCH 09/14] Build releases only on tags or develop Needed so we could push documentation updates to master without triggering a rebuild --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f8711f299..3868cb603 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -148,7 +148,7 @@ amd64: # TODO: Replace with upstream image when 1.9.0 comes out image: rinpatch/elixir:1.9.0-rc.0 only: &release-only - - master@pleroma/pleroma + - tags - develop@pleroma/pleroma artifacts: &release-artifacts name: "pleroma-$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA-$CI_JOB_NAME" From 0f6e333d8be84c6b75552411cd6838b9a0a16cad Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sat, 29 Jun 2019 13:27:15 +0300 Subject: [PATCH 10/14] Revert "Build releases only on tags or develop" This reverts commit 54d287377054fa48c5a14cf6b056cd80e838b264. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3868cb603..f8711f299 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -148,7 +148,7 @@ amd64: # TODO: Replace with upstream image when 1.9.0 comes out image: rinpatch/elixir:1.9.0-rc.0 only: &release-only - - tags + - master@pleroma/pleroma - develop@pleroma/pleroma artifacts: &release-artifacts name: "pleroma-$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA-$CI_JOB_NAME" From 4c40ada7a5f5067f26135de5fcd97b2d97f90441 Mon Sep 17 00:00:00 2001 From: deorsum Date: Sat, 29 Jun 2019 20:39:03 +1000 Subject: [PATCH 11/14] Add a caveat for docker deployment in the config docs --- docs/config.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/config.md b/docs/config.md index 7d5be3980..feef43ba9 100644 --- a/docs/config.md +++ b/docs/config.md @@ -280,7 +280,7 @@ config :pleroma, :mrf_subchain, ## Pleroma.Web.Endpoint `Phoenix` endpoint configuration, all configuration options can be viewed [here](https://hexdocs.pm/phoenix/Phoenix.Endpoint.html#module-dynamic-configuration), only common options are listed here -* `http` - a list containing http protocol configuration, all configuration options can be viewed [here](https://hexdocs.pm/plug_cowboy/Plug.Cowboy.html#module-options), only common options are listed here +* `http` - a list containing http protocol configuration, all configuration options can be viewed [here](https://hexdocs.pm/plug_cowboy/Plug.Cowboy.html#module-options), only common options are listed here. For deployment using docker, you need to set this to `[ip: {0,0,0,0}, port: 4000]` to make pleroma accessible from other containers (such as your nginx server). - `ip` - a tuple consisting of 4 integers - `port` * `url` - a list containing the configuration for generating urls, accepts From 4c60a562a7392294683caae71827d0053a3c3466 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sat, 29 Jun 2019 22:24:03 +0300 Subject: [PATCH 12/14] Fix not being able to pin unlisted posts Closes #1038 --- CHANGELOG.md | 2 ++ lib/pleroma/web/common_api/common_api.ex | 4 ++-- test/web/common_api/common_api_test.exs | 5 +++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a6ec8674d..e30ab803c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [Unreleased] ### Added - MRF: Support for priming the mediaproxy cache (`Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy`) +### Fixed +- Not being able to pin unlisted posts ## [1.0.0] - 2019-06-29 ### Security diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index f8df1e2ea..f71c67a3d 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -11,6 +11,7 @@ defmodule Pleroma.Web.CommonAPI do alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Utils + alias Pleroma.Web.ActivityPub.Visibility import Pleroma.Web.CommonAPI.Utils @@ -284,12 +285,11 @@ def pin(id_or_ap_id, %{ap_id: user_ap_id} = user) do }, object: %Object{ data: %{ - "to" => object_to, "type" => "Note" } } } = activity <- get_by_id_or_ap_id(id_or_ap_id), - true <- Enum.member?(object_to, "https://www.w3.org/ns/activitystreams#Public"), + true <- Visibility.is_public?(activity), %{valid?: true} = info_changeset <- User.Info.add_pinnned_activity(user.info, activity), changeset <- diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs index e96106f11..6f57bbe1f 100644 --- a/test/web/common_api/common_api_test.exs +++ b/test/web/common_api/common_api_test.exs @@ -188,6 +188,11 @@ test "pin status", %{user: user, activity: activity} do assert %User{info: %{pinned_activities: [^id]}} = user end + test "unlisted statuses can be pinned", %{user: user} do + {:ok, activity} = CommonAPI.post(user, %{"status" => "HI!!!", "visibility" => "unlisted"}) + assert {:ok, ^activity} = CommonAPI.pin(activity.id, user) + end + test "only self-authored can be pinned", %{activity: activity} do user = insert(:user) From acd20f166b696254c6a632101d693d03416ad68d Mon Sep 17 00:00:00 2001 From: Maksim Date: Sun, 30 Jun 2019 07:28:35 +0000 Subject: [PATCH 13/14] [#1026] Filter.AnonymizeFilename added ability to retain file extension with custom text --- CHANGELOG.md | 3 ++ docs/config.md | 2 +- lib/pleroma/config.ex | 2 +- .../upload/filter/anonymize_filename.ex | 17 ++++++-- .../upload/filter/anonymize_filename_test.exs | 40 +++++++++++++++++++ 5 files changed, 58 insertions(+), 6 deletions(-) create mode 100644 test/upload/filter/anonymize_filename_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index e30ab803c..96473b1b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Fixed - Not being able to pin unlisted posts +### Changed +- Configuration: Filter.AnonymizeFilename added ability to retain file extension with custom text + ## [1.0.0] - 2019-06-29 ### Security - Mastodon API: Fix display names not being sanitized diff --git a/docs/config.md b/docs/config.md index feef43ba9..8afccb228 100644 --- a/docs/config.md +++ b/docs/config.md @@ -36,7 +36,7 @@ No specific configuration. This filter replaces the filename (not the path) of an upload. For complete obfuscation, add `Pleroma.Upload.Filter.Dedupe` before AnonymizeFilename. -* `text`: Text to replace filenames in links. If empty, `{random}.extension` will be used. +* `text`: Text to replace filenames in links. If empty, `{random}.extension` will be used. You can get the original filename extension by using `{extension}`, for example `custom-file-name.{extension}`. ## Pleroma.Emails.Mailer * `adapter`: one of the mail adapters listed in [Swoosh readme](https://github.com/swoosh/swoosh#adapters), or `Swoosh.Adapters.Local` for in-memory mailbox. diff --git a/lib/pleroma/config.ex b/lib/pleroma/config.ex index 71a47b9fb..fcc039710 100644 --- a/lib/pleroma/config.ex +++ b/lib/pleroma/config.ex @@ -38,7 +38,7 @@ def put([key], value), do: put(key, value) def put([parent_key | keys], value) do parent = - Application.get_env(:pleroma, parent_key) + Application.get_env(:pleroma, parent_key, []) |> put_in(keys, value) Application.put_env(:pleroma, parent_key, parent) diff --git a/lib/pleroma/upload/filter/anonymize_filename.ex b/lib/pleroma/upload/filter/anonymize_filename.ex index 5ca53a79b..a8516811c 100644 --- a/lib/pleroma/upload/filter/anonymize_filename.ex +++ b/lib/pleroma/upload/filter/anonymize_filename.ex @@ -10,10 +10,19 @@ defmodule Pleroma.Upload.Filter.AnonymizeFilename do """ @behaviour Pleroma.Upload.Filter - def filter(upload) do - extension = List.last(String.split(upload.name, ".")) - name = Pleroma.Config.get([__MODULE__, :text], random(extension)) - {:ok, %Pleroma.Upload{upload | name: name}} + alias Pleroma.Config + alias Pleroma.Upload + + def filter(%Upload{name: name} = upload) do + extension = List.last(String.split(name, ".")) + name = predefined_name(extension) || random(extension) + {:ok, %Upload{upload | name: name}} + end + + @spec predefined_name(String.t()) :: String.t() | nil + defp predefined_name(extension) do + with name when not is_nil(name) <- Config.get([__MODULE__, :text]), + do: String.replace(name, "{extension}", extension) end defp random(extension) do diff --git a/test/upload/filter/anonymize_filename_test.exs b/test/upload/filter/anonymize_filename_test.exs new file mode 100644 index 000000000..02241cfa4 --- /dev/null +++ b/test/upload/filter/anonymize_filename_test.exs @@ -0,0 +1,40 @@ +defmodule Pleroma.Upload.Filter.AnonymizeFilenameTest do + use Pleroma.DataCase + + alias Pleroma.Config + alias Pleroma.Upload + + setup do + custom_filename = Config.get([Upload.Filter.AnonymizeFilename, :text]) + + on_exit(fn -> + Config.put([Upload.Filter.AnonymizeFilename, :text], custom_filename) + end) + + upload_file = %Upload{ + name: "an… image.jpg", + content_type: "image/jpg", + path: Path.absname("test/fixtures/image_tmp.jpg") + } + + %{upload_file: upload_file} + end + + test "it replaces filename on pre-defined text", %{upload_file: upload_file} do + Config.put([Upload.Filter.AnonymizeFilename, :text], "custom-file.png") + {:ok, %Upload{name: name}} = Upload.Filter.AnonymizeFilename.filter(upload_file) + assert name == "custom-file.png" + end + + test "it replaces filename on pre-defined text expression", %{upload_file: upload_file} do + Config.put([Upload.Filter.AnonymizeFilename, :text], "custom-file.{extension}") + {:ok, %Upload{name: name}} = Upload.Filter.AnonymizeFilename.filter(upload_file) + assert name == "custom-file.jpg" + end + + test "it replaces filename on random text", %{upload_file: upload_file} do + {:ok, %Upload{name: name}} = Upload.Filter.AnonymizeFilename.filter(upload_file) + assert <<_::bytes-size(14)>> <> ".jpg" = name + refute name == "an… image.jpg" + end +end From 1f76740e104e6b5f50fbb813b5d7b42499eb3466 Mon Sep 17 00:00:00 2001 From: Eugenij Date: Sun, 30 Jun 2019 09:08:46 +0000 Subject: [PATCH 14/14] Add hashtag filter to user statuses (GET /api/v1/accounts/:id/statuses) --- CHANGELOG.md | 2 ++ .../web/mastodon_api/mastodon_api_controller.ex | 4 ++++ .../mastodon_api/mastodon_api_controller_test.exs | 13 +++++++++++++ 3 files changed, 19 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 96473b1b8..663d99ffd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [Unreleased] ### Added - MRF: Support for priming the mediaproxy cache (`Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy`) +- Mastodon API: Support for the [`tagged` filter](https://github.com/tootsuite/mastodon/pull/9755) in [`GET /api/v1/accounts/:id/statuses`](https://docs.joinmastodon.org/api/rest/accounts/#get-api-v1-accounts-id-statuses) + ### Fixed - Not being able to pin unlisted posts diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 7cdba4cc0..ceb88511b 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -356,6 +356,10 @@ def public_timeline(%{assigns: %{user: user}} = conn, params) do def user_statuses(%{assigns: %{user: reading_user}} = conn, params) do with %User{} = user <- User.get_cached_by_id(params["id"]) do + params = + params + |> Map.put("tag", params["tagged"]) + activities = ActivityPub.fetch_user_activities(user, reading_user, params) conn diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 03f57dbfa..b7487c68c 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -1408,6 +1408,19 @@ test "gets a user's statuses without reblogs", %{conn: conn} do assert [%{"id" => id}] = json_response(conn, 200) assert id == to_string(post.id) end + + test "filters user's statuses by a hashtag", %{conn: conn} do + user = insert(:user) + {:ok, post} = CommonAPI.post(user, %{"status" => "#hashtag"}) + {:ok, _post} = CommonAPI.post(user, %{"status" => "hashtag"}) + + conn = + conn + |> get("/api/v1/accounts/#{user.id}/statuses", %{"tagged" => "hashtag"}) + + assert [%{"id" => id}] = json_response(conn, 200) + assert id == to_string(post.id) + end end describe "user relationships" do