From 257e059e61b89752bcde9544cb5ae645b167c96b Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 19 Aug 2020 15:31:33 +0400 Subject: [PATCH 01/45] Add account export --- lib/pleroma/export.ex | 118 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 lib/pleroma/export.ex diff --git a/lib/pleroma/export.ex b/lib/pleroma/export.ex new file mode 100644 index 000000000..82a4b7ace --- /dev/null +++ b/lib/pleroma/export.ex @@ -0,0 +1,118 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Export do + alias Pleroma.Activity + alias Pleroma.Bookmark + alias Pleroma.User + alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.ActivityPub.Transmogrifier + alias Pleroma.Web.ActivityPub.UserView + + import Ecto.Query + + def run(user) do + with {:ok, dir} <- create_dir(), + :ok <- actor(dir, user), + :ok <- statuses(dir, user), + :ok <- likes(dir, user), + :ok <- bookmarks(dir, user) do + IO.inspect({"DONE", dir}) + else + err -> IO.inspect({"export error", err}) + end + end + + def actor(dir, user) do + with {:ok, json} <- + UserView.render("user.json", %{user: user}) + |> Map.merge(%{"likes" => "likes.json", "bookmarks" => "bookmarks.json"}) + |> Jason.encode() do + File.write(dir <> "/actor.json", json) + end + end + + defp create_dir do + datetime = Calendar.NaiveDateTime.Format.iso8601_basic(NaiveDateTime.utc_now()) + dir = Path.join(System.tmp_dir!(), "archive-" <> datetime) + + with :ok <- File.mkdir(dir), do: {:ok, dir} + end + + defp write_header(file, name) do + IO.write( + file, + """ + { + "@context": "https://www.w3.org/ns/activitystreams", + "id": "#{name}.json", + "type": "OrderedCollection", + "orderedItems": [ + """ + ) + end + + defp write(query, dir, name, fun) do + path = dir <> "/#{name}.json" + + with {:ok, file} <- File.open(path, [:write, :utf8]), + :ok <- write_header(file, name) do + counter = :counters.new(1, []) + + query + |> Pleroma.RepoStreamer.chunk_stream(100) + |> Stream.each(fn items -> + Enum.each(items, fn i -> + with {:ok, str} <- fun.(i), + :ok <- IO.write(file, str <> ",\n") do + :counters.add(counter, 1, 1) + end + end) + end) + |> Stream.run() + + total = :counters.get(counter, 1) + + with :ok <- :file.pwrite(file, {:eof, -2}, "\n],\n \"totalItems\": #{total}}") do + File.close(file) + end + end + end + + def bookmarks(dir, %{id: user_id} = _user) do + Bookmark + |> where(user_id: ^user_id) + |> join(:inner, [b], activity in assoc(b, :activity)) + |> select([b, a], %{id: b.id, object: fragment("(?)->>'object'", a.data)}) + |> write(dir, "bookmarks", fn a -> {:ok, "\"#{a.object}\""} end) + end + + def likes(dir, user) do + user.ap_id + |> Activity.Queries.by_actor() + |> Activity.Queries.by_type("Like") + |> select([like], %{id: like.id, object: fragment("(?)->>'object'", like.data)}) + |> write(dir, "likes", fn a -> {:ok, "\"#{a.object}\""} end) + end + + def statuses(dir, user) do + opts = + %{} + |> Map.put(:type, ["Create", "Announce"]) + |> Map.put(:blocking_user, user) + |> Map.put(:muting_user, user) + |> Map.put(:reply_filtering_user, user) + |> Map.put(:announce_filtering_user, user) + |> Map.put(:user, user) + + [[user.ap_id], User.following(user), Pleroma.List.memberships(user)] + |> Enum.concat() + |> ActivityPub.fetch_activities_query(opts) + |> write(dir, "outbox", fn a -> + with {:ok, activity} <- Transmogrifier.prepare_outgoing(a.data) do + activity |> Map.delete("@context") |> Jason.encode() + end + end) + end +end From 9d564ffc2988f145bc9cf26477eea93b1bf01cb0 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 24 Aug 2020 20:59:57 +0400 Subject: [PATCH 02/45] Zip exported files --- lib/pleroma/export.ex | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/lib/pleroma/export.ex b/lib/pleroma/export.ex index 82a4b7ace..f0f1ef093 100644 --- a/lib/pleroma/export.ex +++ b/lib/pleroma/export.ex @@ -12,15 +12,17 @@ defmodule Pleroma.Export do import Ecto.Query + @files ['actor.json', 'outbox.json', 'likes.json', 'bookmarks.json'] + def run(user) do - with {:ok, dir} <- create_dir(), - :ok <- actor(dir, user), - :ok <- statuses(dir, user), - :ok <- likes(dir, user), - :ok <- bookmarks(dir, user) do - IO.inspect({"DONE", dir}) - else - err -> IO.inspect({"export error", err}) + with {:ok, path} <- create_dir(user), + :ok <- actor(path, user), + :ok <- statuses(path, user), + :ok <- likes(path, user), + :ok <- bookmarks(path, user), + {:ok, zip_path} <- :zip.create('#{path}.zip', @files, cwd: path), + {:ok, _} <- File.rm_rf(path) do + {:ok, zip_path} end end @@ -33,9 +35,9 @@ defmodule Pleroma.Export do end end - defp create_dir do + defp create_dir(user) do datetime = Calendar.NaiveDateTime.Format.iso8601_basic(NaiveDateTime.utc_now()) - dir = Path.join(System.tmp_dir!(), "archive-" <> datetime) + dir = Path.join(System.tmp_dir!(), "archive-#{user.id}-#{datetime}") with :ok <- File.mkdir(dir), do: {:ok, dir} end From c01a81804835fb92c145b90e3a264c5d4cf9c886 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 25 Aug 2020 18:51:09 +0400 Subject: [PATCH 03/45] Add tests --- lib/pleroma/export.ex | 8 +-- test/export_test.exs | 111 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+), 4 deletions(-) create mode 100644 test/export_test.exs diff --git a/lib/pleroma/export.ex b/lib/pleroma/export.ex index f0f1ef093..45b8ce749 100644 --- a/lib/pleroma/export.ex +++ b/lib/pleroma/export.ex @@ -26,7 +26,7 @@ defmodule Pleroma.Export do end end - def actor(dir, user) do + defp actor(dir, user) do with {:ok, json} <- UserView.render("user.json", %{user: user}) |> Map.merge(%{"likes" => "likes.json", "bookmarks" => "bookmarks.json"}) @@ -82,7 +82,7 @@ defmodule Pleroma.Export do end end - def bookmarks(dir, %{id: user_id} = _user) do + defp bookmarks(dir, %{id: user_id} = _user) do Bookmark |> where(user_id: ^user_id) |> join(:inner, [b], activity in assoc(b, :activity)) @@ -90,7 +90,7 @@ defmodule Pleroma.Export do |> write(dir, "bookmarks", fn a -> {:ok, "\"#{a.object}\""} end) end - def likes(dir, user) do + defp likes(dir, user) do user.ap_id |> Activity.Queries.by_actor() |> Activity.Queries.by_type("Like") @@ -98,7 +98,7 @@ defmodule Pleroma.Export do |> write(dir, "likes", fn a -> {:ok, "\"#{a.object}\""} end) end - def statuses(dir, user) do + defp statuses(dir, user) do opts = %{} |> Map.put(:type, ["Create", "Announce"]) diff --git a/test/export_test.exs b/test/export_test.exs new file mode 100644 index 000000000..5afd58ccc --- /dev/null +++ b/test/export_test.exs @@ -0,0 +1,111 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.ExportTest do + use Pleroma.DataCase + import Pleroma.Factory + + alias Pleroma.Web.CommonAPI + alias Pleroma.Bookmark + + test "it exports user data" do + user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"}) + + {:ok, %{object: %{data: %{"id" => id1}}} = status1} = + CommonAPI.post(user, %{status: "status1"}) + + {:ok, %{object: %{data: %{"id" => id2}}} = status2} = + CommonAPI.post(user, %{status: "status2"}) + + {:ok, %{object: %{data: %{"id" => id3}}} = status3} = + CommonAPI.post(user, %{status: "status3"}) + + CommonAPI.favorite(user, status1.id) + CommonAPI.favorite(user, status2.id) + + Bookmark.create(user.id, status2.id) + Bookmark.create(user.id, status3.id) + + assert {:ok, path} = Pleroma.Export.run(user) + assert {:ok, zipfile} = :zip.zip_open(path, [:memory]) + assert {:ok, {'actor.json', json}} = :zip.zip_get('actor.json', zipfile) + + assert %{ + "@context" => [ + "https://www.w3.org/ns/activitystreams", + "http://localhost:4001/schemas/litepub-0.1.jsonld", + %{"@language" => "und"} + ], + "bookmarks" => "bookmarks.json", + "followers" => "http://cofe.io/users/cofe/followers", + "following" => "http://cofe.io/users/cofe/following", + "id" => "http://cofe.io/users/cofe", + "inbox" => "http://cofe.io/users/cofe/inbox", + "likes" => "likes.json", + "name" => "Cofe", + "outbox" => "http://cofe.io/users/cofe/outbox", + "preferredUsername" => "cofe", + "publicKey" => %{ + "id" => "http://cofe.io/users/cofe#main-key", + "owner" => "http://cofe.io/users/cofe" + }, + "type" => "Person", + "url" => "http://cofe.io/users/cofe" + } = Jason.decode!(json) + + assert {:ok, {'outbox.json', json}} = :zip.zip_get('outbox.json', zipfile) + + assert %{ + "@context" => "https://www.w3.org/ns/activitystreams", + "id" => "outbox.json", + "orderedItems" => [ + %{ + "object" => %{ + "actor" => "http://cofe.io/users/cofe", + "content" => "status1", + "type" => "Note" + }, + "type" => "Create" + }, + %{ + "object" => %{ + "actor" => "http://cofe.io/users/cofe", + "content" => "status2" + } + }, + %{ + "actor" => "http://cofe.io/users/cofe", + "object" => %{ + "content" => "status3" + } + } + ], + "totalItems" => 3, + "type" => "OrderedCollection" + } = Jason.decode!(json) + + assert {:ok, {'likes.json', json}} = :zip.zip_get('likes.json', zipfile) + + assert %{ + "@context" => "https://www.w3.org/ns/activitystreams", + "id" => "likes.json", + "orderedItems" => [^id1, ^id2], + "totalItems" => 2, + "type" => "OrderedCollection" + } = Jason.decode!(json) + + assert {:ok, {'bookmarks.json', json}} = :zip.zip_get('bookmarks.json', zipfile) + + assert %{ + "@context" => "https://www.w3.org/ns/activitystreams", + "id" => "bookmarks.json", + "orderedItems" => [^id2, ^id3], + "totalItems" => 2, + "type" => "OrderedCollection" + } = Jason.decode!(json) + + :zip.zip_close(zipfile) + File.rm!(path) + end +end From c82f9129592553718be4bd4712a2b1848dd0a447 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 25 Aug 2020 19:16:01 +0400 Subject: [PATCH 04/45] Fix credo warning --- test/export_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/export_test.exs b/test/export_test.exs index 5afd58ccc..01ca8e7e8 100644 --- a/test/export_test.exs +++ b/test/export_test.exs @@ -6,8 +6,8 @@ defmodule Pleroma.ExportTest do use Pleroma.DataCase import Pleroma.Factory - alias Pleroma.Web.CommonAPI alias Pleroma.Bookmark + alias Pleroma.Web.CommonAPI test "it exports user data" do user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"}) From be42ab70dc9538df54ac6f30ee123666223b7287 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 31 Aug 2020 20:31:21 +0400 Subject: [PATCH 05/45] Add backup upload --- lib/pleroma/export.ex | 20 +++++++++++++++++++- test/export_test.exs | 17 ++++++++++++++++- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/export.ex b/lib/pleroma/export.ex index 45b8ce749..b84eccd78 100644 --- a/lib/pleroma/export.ex +++ b/lib/pleroma/export.ex @@ -22,7 +22,25 @@ defmodule Pleroma.Export do :ok <- bookmarks(path, user), {:ok, zip_path} <- :zip.create('#{path}.zip', @files, cwd: path), {:ok, _} <- File.rm_rf(path) do - {:ok, zip_path} + {:ok, :binary.list_to_bin(zip_path)} + end + end + + def upload(zip_path) do + uploader = Pleroma.Config.get([Pleroma.Upload, :uploader]) + file_name = zip_path |> String.split("/") |> List.last() + id = Ecto.UUID.generate() + + upload = %Pleroma.Upload{ + id: id, + name: file_name, + tempfile: zip_path, + content_type: "application/zip", + path: id <> "/" <> file_name + } + + with :ok <- uploader.put_file(upload), :ok <- File.rm(zip_path) do + {:ok, upload} end end diff --git a/test/export_test.exs b/test/export_test.exs index 01ca8e7e8..fae269974 100644 --- a/test/export_test.exs +++ b/test/export_test.exs @@ -28,7 +28,7 @@ defmodule Pleroma.ExportTest do Bookmark.create(user.id, status3.id) assert {:ok, path} = Pleroma.Export.run(user) - assert {:ok, zipfile} = :zip.zip_open(path, [:memory]) + assert {:ok, zipfile} = :zip.zip_open(String.to_charlist(path), [:memory]) assert {:ok, {'actor.json', json}} = :zip.zip_get('actor.json', zipfile) assert %{ @@ -108,4 +108,19 @@ defmodule Pleroma.ExportTest do :zip.zip_close(zipfile) File.rm!(path) end + + test "it uploads an exported backup archive" do + user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"}) + + {:ok, status1} = CommonAPI.post(user, %{status: "status1"}) + {:ok, status2} = CommonAPI.post(user, %{status: "status2"}) + {:ok, status3} = CommonAPI.post(user, %{status: "status3"}) + CommonAPI.favorite(user, status1.id) + CommonAPI.favorite(user, status2.id) + Bookmark.create(user.id, status2.id) + Bookmark.create(user.id, status3.id) + + assert {:ok, path} = Pleroma.Export.run(user) + assert {:ok, %Pleroma.Upload{}} = Pleroma.Export.upload(path) + end end From 75e07ba206b94155c5210151a49e29a11bce6e50 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 31 Aug 2020 23:07:14 +0400 Subject: [PATCH 06/45] Fix tests --- lib/pleroma/export.ex | 3 ++- test/export_test.exs | 47 +++++++++++++++++++++++++++++++++---------- 2 files changed, 38 insertions(+), 12 deletions(-) diff --git a/lib/pleroma/export.ex b/lib/pleroma/export.ex index b84eccd78..8b1bfefe2 100644 --- a/lib/pleroma/export.ex +++ b/lib/pleroma/export.ex @@ -39,7 +39,8 @@ defmodule Pleroma.Export do path: id <> "/" <> file_name } - with :ok <- uploader.put_file(upload), :ok <- File.rm(zip_path) do + with {:ok, _} <- Pleroma.Uploaders.Uploader.put_file(uploader, upload), + :ok <- File.rm(zip_path) do {:ok, upload} end end diff --git a/test/export_test.exs b/test/export_test.exs index fae269974..d7e8f558c 100644 --- a/test/export_test.exs +++ b/test/export_test.exs @@ -5,6 +5,7 @@ defmodule Pleroma.ExportTest do use Pleroma.DataCase import Pleroma.Factory + import Mock alias Pleroma.Bookmark alias Pleroma.Web.CommonAPI @@ -109,18 +110,42 @@ defmodule Pleroma.ExportTest do File.rm!(path) end - test "it uploads an exported backup archive" do - user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"}) + describe "it uploads an exported backup archive" do + setup do + clear_config(Pleroma.Uploaders.S3, + bucket: "test_bucket", + public_endpoint: "https://s3.amazonaws.com" + ) - {:ok, status1} = CommonAPI.post(user, %{status: "status1"}) - {:ok, status2} = CommonAPI.post(user, %{status: "status2"}) - {:ok, status3} = CommonAPI.post(user, %{status: "status3"}) - CommonAPI.favorite(user, status1.id) - CommonAPI.favorite(user, status2.id) - Bookmark.create(user.id, status2.id) - Bookmark.create(user.id, status3.id) + clear_config([Pleroma.Upload, :uploader]) - assert {:ok, path} = Pleroma.Export.run(user) - assert {:ok, %Pleroma.Upload{}} = Pleroma.Export.upload(path) + user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"}) + + {:ok, status1} = CommonAPI.post(user, %{status: "status1"}) + {:ok, status2} = CommonAPI.post(user, %{status: "status2"}) + {:ok, status3} = CommonAPI.post(user, %{status: "status3"}) + CommonAPI.favorite(user, status1.id) + CommonAPI.favorite(user, status2.id) + Bookmark.create(user.id, status2.id) + Bookmark.create(user.id, status3.id) + + assert {:ok, path} = Pleroma.Export.run(user) + + [path: path] + end + + test "S3", %{path: path} do + Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.S3) + + with_mock ExAws, request: fn _ -> {:ok, :ok} end do + assert {:ok, %Pleroma.Upload{}} = Pleroma.Export.upload(path) + end + end + + test "Local", %{path: path} do + Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) + + assert {:ok, %Pleroma.Upload{}} = Pleroma.Export.upload(path) + end end end From 4f3a6337454807f4145bbc1830c3d55dd883d46d Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 2 Sep 2020 20:21:33 +0400 Subject: [PATCH 07/45] Add `backups` table --- lib/pleroma/{export.ex => backup.ex} | 110 ++++++++++++++---- .../20200831192323_create_backups.exs | 17 +++ test/{export_test.exs => backup_test.exs} | 48 ++++++-- 3 files changed, 141 insertions(+), 34 deletions(-) rename lib/pleroma/{export.ex => backup.ex} (60%) create mode 100644 priv/repo/migrations/20200831192323_create_backups.exs rename test/{export_test.exs => backup_test.exs} (75%) diff --git a/lib/pleroma/export.ex b/lib/pleroma/backup.ex similarity index 60% rename from lib/pleroma/export.ex rename to lib/pleroma/backup.ex index 8b1bfefe2..4580d8f92 100644 --- a/lib/pleroma/export.ex +++ b/lib/pleroma/backup.ex @@ -2,41 +2,110 @@ # Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Export do +defmodule Pleroma.Backup do + use Ecto.Schema + + import Ecto.Changeset + import Ecto.Query + alias Pleroma.Activity alias Pleroma.Bookmark + alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.ActivityPub.UserView - import Ecto.Query + schema "backups" do + field(:content_type, :string) + field(:file_name, :string) + field(:file_size, :integer, default: 0) + field(:processed, :boolean, default: false) + + belongs_to(:user, User, type: FlakeId.Ecto.CompatType) + + timestamps() + end + + def create(user) do + with :ok <- validate_limit(user), + {:ok, backup} <- user |> new() |> Repo.insert() do + {:ok, backup} + end + end + + def new(user) do + rand_str = :crypto.strong_rand_bytes(32) |> Base.url_encode64(padding: false) + datetime = Calendar.NaiveDateTime.Format.iso8601_basic(NaiveDateTime.utc_now()) + name = "archive-#{user.nickname}-#{datetime}-#{rand_str}.zip" + + %__MODULE__{ + user_id: user.id, + content_type: "application/zip", + file_name: name + } + end + + defp validate_limit(user) do + case get_last(user.id) do + %__MODULE__{inserted_at: inserted_at} -> + days = 7 + diff = Timex.diff(NaiveDateTime.utc_now(), inserted_at, :days) + + if diff > days do + :ok + else + {:error, "Last export was less than #{days} days ago"} + end + + nil -> + :ok + end + end + + def get_last(user_id) do + __MODULE__ + |> where(user_id: ^user_id) + |> order_by(desc: :id) + |> limit(1) + |> Repo.one() + end + + def process(%__MODULE__{} = backup) do + with {:ok, zip_file} <- zip(backup), + {:ok, %{size: size}} <- File.stat(zip_file), + {:ok, _upload} <- upload(backup, zip_file) do + backup + |> cast(%{file_size: size, processed: true}, [:file_size, :processed]) + |> Repo.update() + end + end @files ['actor.json', 'outbox.json', 'likes.json', 'bookmarks.json'] + def zip(%__MODULE__{} = backup) do + backup = Repo.preload(backup, :user) + name = String.trim_trailing(backup.file_name, ".zip") + dir = Path.join(System.tmp_dir!(), name) - def run(user) do - with {:ok, path} <- create_dir(user), - :ok <- actor(path, user), - :ok <- statuses(path, user), - :ok <- likes(path, user), - :ok <- bookmarks(path, user), - {:ok, zip_path} <- :zip.create('#{path}.zip', @files, cwd: path), - {:ok, _} <- File.rm_rf(path) do + with :ok <- File.mkdir(dir), + :ok <- actor(dir, backup.user), + :ok <- statuses(dir, backup.user), + :ok <- likes(dir, backup.user), + :ok <- bookmarks(dir, backup.user), + {:ok, zip_path} <- :zip.create(String.to_charlist(dir <> ".zip"), @files, cwd: dir), + {:ok, _} <- File.rm_rf(dir) do {:ok, :binary.list_to_bin(zip_path)} end end - def upload(zip_path) do + def upload(%__MODULE__{} = backup, zip_path) do uploader = Pleroma.Config.get([Pleroma.Upload, :uploader]) - file_name = zip_path |> String.split("/") |> List.last() - id = Ecto.UUID.generate() upload = %Pleroma.Upload{ - id: id, - name: file_name, + name: backup.file_name, tempfile: zip_path, - content_type: "application/zip", - path: id <> "/" <> file_name + content_type: backup.content_type, + path: "backups/" <> backup.file_name } with {:ok, _} <- Pleroma.Uploaders.Uploader.put_file(uploader, upload), @@ -54,13 +123,6 @@ defmodule Pleroma.Export do end end - defp create_dir(user) do - datetime = Calendar.NaiveDateTime.Format.iso8601_basic(NaiveDateTime.utc_now()) - dir = Path.join(System.tmp_dir!(), "archive-#{user.id}-#{datetime}") - - with :ok <- File.mkdir(dir), do: {:ok, dir} - end - defp write_header(file, name) do IO.write( file, diff --git a/priv/repo/migrations/20200831192323_create_backups.exs b/priv/repo/migrations/20200831192323_create_backups.exs new file mode 100644 index 000000000..3ac5889e2 --- /dev/null +++ b/priv/repo/migrations/20200831192323_create_backups.exs @@ -0,0 +1,17 @@ +defmodule Pleroma.Repo.Migrations.CreateBackups do + use Ecto.Migration + + def change do + create_if_not_exists table(:backups) do + add(:user_id, references(:users, type: :uuid, on_delete: :delete_all)) + add(:file_name, :string, null: false) + add(:content_type, :string, null: false) + add(:processed, :boolean, null: false, default: false) + add(:file_size, :bigint) + + timestamps() + end + + create_if_not_exists(index(:backups, [:user_id])) + end +end diff --git a/test/export_test.exs b/test/backup_test.exs similarity index 75% rename from test/export_test.exs rename to test/backup_test.exs index d7e8f558c..27f5cb7f7 100644 --- a/test/export_test.exs +++ b/test/backup_test.exs @@ -2,15 +2,41 @@ # Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.ExportTest do +defmodule Pleroma.BackupTest do use Pleroma.DataCase import Pleroma.Factory import Mock + alias Pleroma.Backup alias Pleroma.Bookmark alias Pleroma.Web.CommonAPI - test "it exports user data" do + test "it creates a backup record" do + %{id: user_id} = user = insert(:user) + assert {:ok, backup} = Backup.create(user) + + assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup + end + + test "it return an error if the export limit is over" do + %{id: user_id} = user = insert(:user) + limit_days = 7 + + assert {:ok, backup} = Backup.create(user) + assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup + + assert Backup.create(user) == {:error, "Last export was less than #{limit_days} days ago"} + end + + test "it process a backup record" do + %{id: user_id} = user = insert(:user) + assert {:ok, %{id: backup_id} = backup} = Backup.create(user) + assert {:ok, %Backup{} = backup} = Backup.process(backup) + assert backup.file_size > 0 + assert %Backup{id: ^backup_id, processed: true, user_id: ^user_id} = backup + end + + test "it creates a zip archive with user data" do user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"}) {:ok, %{object: %{data: %{"id" => id1}}} = status1} = @@ -28,7 +54,8 @@ defmodule Pleroma.ExportTest do Bookmark.create(user.id, status2.id) Bookmark.create(user.id, status3.id) - assert {:ok, path} = Pleroma.Export.run(user) + assert {:ok, backup} = user |> Backup.new() |> Repo.insert() + assert {:ok, path} = Backup.zip(backup) assert {:ok, zipfile} = :zip.zip_open(String.to_charlist(path), [:memory]) assert {:ok, {'actor.json', json}} = :zip.zip_get('actor.json', zipfile) @@ -110,7 +137,7 @@ defmodule Pleroma.ExportTest do File.rm!(path) end - describe "it uploads an exported backup archive" do + describe "it uploads a backup archive" do setup do clear_config(Pleroma.Uploaders.S3, bucket: "test_bucket", @@ -129,23 +156,24 @@ defmodule Pleroma.ExportTest do Bookmark.create(user.id, status2.id) Bookmark.create(user.id, status3.id) - assert {:ok, path} = Pleroma.Export.run(user) + assert {:ok, backup} = user |> Backup.new() |> Repo.insert() + assert {:ok, path} = Backup.zip(backup) - [path: path] + [path: path, backup: backup] end - test "S3", %{path: path} do + test "S3", %{path: path, backup: backup} do Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.S3) with_mock ExAws, request: fn _ -> {:ok, :ok} end do - assert {:ok, %Pleroma.Upload{}} = Pleroma.Export.upload(path) + assert {:ok, %Pleroma.Upload{}} = Backup.upload(backup, path) end end - test "Local", %{path: path} do + test "Local", %{path: path, backup: backup} do Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) - assert {:ok, %Pleroma.Upload{}} = Pleroma.Export.upload(path) + assert {:ok, %Pleroma.Upload{}} = Backup.upload(backup, path) end end end From a0ad9bd734e9af0ce912c32c7480a60ff87a4368 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 2 Sep 2020 21:45:22 +0400 Subject: [PATCH 08/45] Add BackupWorker --- config/config.exs | 1 + config/description.exs | 6 ++++++ lib/pleroma/backup.ex | 11 ++++++++++- lib/pleroma/workers/backup_worker.ex | 17 +++++++++++++++++ test/backup_test.exs | 20 ++++++++++++++------ 5 files changed, 48 insertions(+), 7 deletions(-) create mode 100644 lib/pleroma/workers/backup_worker.ex diff --git a/config/config.exs b/config/config.exs index 2e6b0796a..1f10167e5 100644 --- a/config/config.exs +++ b/config/config.exs @@ -551,6 +551,7 @@ config :pleroma, Oban, queues: [ activity_expiration: 10, token_expiration: 5, + backup: 1, federator_incoming: 50, federator_outgoing: 50, ingestion_queue: 50, diff --git a/config/description.exs b/config/description.exs index 6fa78a5d1..13e44afe8 100644 --- a/config/description.exs +++ b/config/description.exs @@ -2288,6 +2288,12 @@ config :pleroma, :config_description, [ description: "Activity expiration queue", suggestions: [10] }, + %{ + key: :backup, + type: :integer, + description: "Backup queue", + suggestions: [1] + }, %{ key: :attachments_cleanup, type: :integer, diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex index 4580d8f92..9b5d2625f 100644 --- a/lib/pleroma/backup.ex +++ b/lib/pleroma/backup.ex @@ -30,7 +30,7 @@ defmodule Pleroma.Backup do def create(user) do with :ok <- validate_limit(user), {:ok, backup} <- user |> new() |> Repo.insert() do - {:ok, backup} + Pleroma.Workers.BackupWorker.enqueue("process", %{"backup_id" => backup.id}) end end @@ -71,6 +71,15 @@ defmodule Pleroma.Backup do |> Repo.one() end + def remove_outdated(%__MODULE__{id: latest_id, user_id: user_id}) do + __MODULE__ + |> where(user_id: ^user_id) + |> where([b], b.id != ^latest_id) + |> Repo.delete_all() + end + + def get(id), do: Repo.get(__MODULE__, id) + def process(%__MODULE__{} = backup) do with {:ok, zip_file} <- zip(backup), {:ok, %{size: size}} <- File.stat(zip_file), diff --git a/lib/pleroma/workers/backup_worker.ex b/lib/pleroma/workers/backup_worker.ex new file mode 100644 index 000000000..c982ffa3a --- /dev/null +++ b/lib/pleroma/workers/backup_worker.ex @@ -0,0 +1,17 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.BackupWorker do + alias Pleroma.Backup + + use Pleroma.Workers.WorkerHelper, queue: "backup" + + @impl Oban.Worker + def perform(%Job{args: %{"op" => "process", "backup_id" => backup_id}}) do + with {:ok, %Backup{} = backup} <- + backup_id |> Backup.get() |> Backup.process() do + {:ok, backup} + end + end +end diff --git a/test/backup_test.exs b/test/backup_test.exs index 27f5cb7f7..5b1f76dd9 100644 --- a/test/backup_test.exs +++ b/test/backup_test.exs @@ -3,35 +3,43 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.BackupTest do + use Oban.Testing, repo: Pleroma.Repo use Pleroma.DataCase + import Pleroma.Factory import Mock alias Pleroma.Backup alias Pleroma.Bookmark alias Pleroma.Web.CommonAPI + alias Pleroma.Workers.BackupWorker - test "it creates a backup record" do + setup do: clear_config([Pleroma.Upload, :uploader]) + + test "it creates a backup record and an Oban job" do %{id: user_id} = user = insert(:user) - assert {:ok, backup} = Backup.create(user) + assert {:ok, %Oban.Job{args: args}} = Backup.create(user) + assert_enqueued(worker: BackupWorker, args: args) + backup = Backup.get(args["backup_id"]) assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup end test "it return an error if the export limit is over" do %{id: user_id} = user = insert(:user) limit_days = 7 - - assert {:ok, backup} = Backup.create(user) + assert {:ok, %Oban.Job{args: args}} = Backup.create(user) + backup = Backup.get(args["backup_id"]) assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup assert Backup.create(user) == {:error, "Last export was less than #{limit_days} days ago"} end test "it process a backup record" do + Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) %{id: user_id} = user = insert(:user) - assert {:ok, %{id: backup_id} = backup} = Backup.create(user) - assert {:ok, %Backup{} = backup} = Backup.process(backup) + assert {:ok, %Oban.Job{args: %{"backup_id" => backup_id}} = job} = Backup.create(user) + assert {:ok, backup} = BackupWorker.perform(job) assert backup.file_size > 0 assert %Backup{id: ^backup_id, processed: true, user_id: ^user_id} = backup end From 3ad7492f9dd1c76cdbc64ad2246f8e9c8c5c4ae6 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Fri, 4 Sep 2020 18:30:39 +0400 Subject: [PATCH 09/45] Add config for Pleroma.Backup --- config/config.exs | 4 ++++ config/description.exs | 20 ++++++++++++++++++++ docs/configuration/cheatsheet.md | 5 +++++ lib/pleroma/backup.ex | 2 +- test/backup_test.exs | 2 +- 5 files changed, 31 insertions(+), 2 deletions(-) diff --git a/config/config.exs b/config/config.exs index 1f10167e5..09023e2c3 100644 --- a/config/config.exs +++ b/config/config.exs @@ -818,6 +818,10 @@ config :floki, :html_parser, Floki.HTMLParser.FastHtml config :pleroma, Pleroma.Web.Auth.Authenticator, Pleroma.Web.Auth.PleromaAuthenticator +config :pleroma, Pleroma.Backup, + purge_after_days: 30, + limit_days: 7 + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" diff --git a/config/description.exs b/config/description.exs index 13e44afe8..4942e196d 100644 --- a/config/description.exs +++ b/config/description.exs @@ -3712,5 +3712,25 @@ config :pleroma, :config_description, [ ] } ] + }, + %{ + group: :pleroma, + key: Pleroma.Backup, + type: :group, + description: "Account Backup", + children: [ + %{ + key: :purge_after_days, + type: :integer, + description: "Remove backup achives after N days", + suggestions: [30] + }, + %{ + key: :limit_days, + type: :integer, + description: "Limit user to export not more often than once per N days", + suggestions: [7] + } + ] } ] diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 42e5fe808..cc4081f14 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -1083,6 +1083,11 @@ Control favicons for instances. * `enabled`: Allow/disallow displaying and getting instances favicons +## Account Backup + +* `:purge_after_days` an integer, remove backup achives after N days. +* `:limit_days` an integer, limit user to export not more often than once per N days. + ## Frontend management Frontends in Pleroma are swappable - you can specify which one to use here. diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex index 9b5d2625f..e384b6b00 100644 --- a/lib/pleroma/backup.ex +++ b/lib/pleroma/backup.ex @@ -49,7 +49,7 @@ defmodule Pleroma.Backup do defp validate_limit(user) do case get_last(user.id) do %__MODULE__{inserted_at: inserted_at} -> - days = 7 + days = Pleroma.Config.get([Pleroma.Backup, :limit_days]) diff = Timex.diff(NaiveDateTime.utc_now(), inserted_at, :days) if diff > days do diff --git a/test/backup_test.exs b/test/backup_test.exs index 5b1f76dd9..f343b0361 100644 --- a/test/backup_test.exs +++ b/test/backup_test.exs @@ -27,7 +27,7 @@ defmodule Pleroma.BackupTest do test "it return an error if the export limit is over" do %{id: user_id} = user = insert(:user) - limit_days = 7 + limit_days = Pleroma.Config.get([Pleroma.Backup, :limit_days]) assert {:ok, %Oban.Job{args: args}} = Backup.create(user) backup = Backup.get(args["backup_id"]) assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup From 739cb1463ba07513f047b2ac8f7e22a16c89ef4e Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Fri, 4 Sep 2020 21:48:52 +0400 Subject: [PATCH 10/45] Add backups deletion --- lib/pleroma/backup.ex | 14 +++++++-- lib/pleroma/workers/backup_worker.ex | 37 ++++++++++++++++++++-- test/backup_test.exs | 47 +++++++++++++++++++++++++--- test/support/oban_helpers.ex | 3 ++ 4 files changed, 91 insertions(+), 10 deletions(-) diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex index e384b6b00..bd50fd910 100644 --- a/lib/pleroma/backup.ex +++ b/lib/pleroma/backup.ex @@ -15,6 +15,7 @@ defmodule Pleroma.Backup do alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.ActivityPub.UserView + alias Pleroma.Workers.BackupWorker schema "backups" do field(:content_type, :string) @@ -30,7 +31,7 @@ defmodule Pleroma.Backup do def create(user) do with :ok <- validate_limit(user), {:ok, backup} <- user |> new() |> Repo.insert() do - Pleroma.Workers.BackupWorker.enqueue("process", %{"backup_id" => backup.id}) + BackupWorker.process(backup) end end @@ -46,6 +47,14 @@ defmodule Pleroma.Backup do } end + def delete(backup) do + uploader = Pleroma.Config.get([Pleroma.Upload, :uploader]) + + with :ok <- uploader.delete_file("backups/" <> backup.file_name) do + Repo.delete(backup) + end + end + defp validate_limit(user) do case get_last(user.id) do %__MODULE__{inserted_at: inserted_at} -> @@ -75,7 +84,8 @@ defmodule Pleroma.Backup do __MODULE__ |> where(user_id: ^user_id) |> where([b], b.id != ^latest_id) - |> Repo.delete_all() + |> Repo.all() + |> Enum.each(&BackupWorker.delete/1) end def get(id), do: Repo.get(__MODULE__, id) diff --git a/lib/pleroma/workers/backup_worker.ex b/lib/pleroma/workers/backup_worker.ex index c982ffa3a..f40020794 100644 --- a/lib/pleroma/workers/backup_worker.ex +++ b/lib/pleroma/workers/backup_worker.ex @@ -3,15 +3,46 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.BackupWorker do + use Oban.Worker, queue: :backup, max_attempts: 1 + + alias Oban.Job alias Pleroma.Backup - use Pleroma.Workers.WorkerHelper, queue: "backup" + def process(backup) do + %{"op" => "process", "backup_id" => backup.id} + |> new() + |> Oban.insert() + end + + def schedule_deletion(backup) do + days = Pleroma.Config.get([Pleroma.Backup, :purge_after_days]) + time = 60 * 60 * 24 * days + scheduled_at = Calendar.NaiveDateTime.add!(backup.inserted_at, time) + + %{"op" => "delete", "backup_id" => backup.id} + |> new(scheduled_at: scheduled_at) + |> Oban.insert() + end + + def delete(backup) do + %{"op" => "delete", "backup_id" => backup.id} + |> new() + |> Oban.insert() + end - @impl Oban.Worker def perform(%Job{args: %{"op" => "process", "backup_id" => backup_id}}) do with {:ok, %Backup{} = backup} <- - backup_id |> Backup.get() |> Backup.process() do + backup_id |> Backup.get() |> Backup.process(), + {:ok, _job} <- schedule_deletion(backup), + :ok <- Backup.remove_outdated(backup) do {:ok, backup} end end + + def perform(%Job{args: %{"op" => "delete", "backup_id" => backup_id}}) do + case Backup.get(backup_id) do + %Backup{} = backup -> Backup.delete(backup) + nil -> :ok + end + end end diff --git a/test/backup_test.exs b/test/backup_test.exs index f343b0361..59aebe360 100644 --- a/test/backup_test.exs +++ b/test/backup_test.exs @@ -13,8 +13,12 @@ defmodule Pleroma.BackupTest do alias Pleroma.Bookmark alias Pleroma.Web.CommonAPI alias Pleroma.Workers.BackupWorker + alias Pleroma.Tests.ObanHelpers - setup do: clear_config([Pleroma.Upload, :uploader]) + setup do + clear_config([Pleroma.Upload, :uploader]) + clear_config([Pleroma.Backup, :limit_days]) + end test "it creates a backup record and an Oban job" do %{id: user_id} = user = insert(:user) @@ -38,10 +42,34 @@ defmodule Pleroma.BackupTest do test "it process a backup record" do Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) %{id: user_id} = user = insert(:user) - assert {:ok, %Oban.Job{args: %{"backup_id" => backup_id}} = job} = Backup.create(user) - assert {:ok, backup} = BackupWorker.perform(job) + + assert {:ok, %Oban.Job{args: %{"backup_id" => backup_id} = args}} = Backup.create(user) + assert {:ok, backup} = perform_job(BackupWorker, args) assert backup.file_size > 0 assert %Backup{id: ^backup_id, processed: true, user_id: ^user_id} = backup + + delete_job_args = %{"op" => "delete", "backup_id" => backup_id} + + assert_enqueued(worker: BackupWorker, args: delete_job_args) + assert {:ok, backup} = perform_job(BackupWorker, delete_job_args) + refute Backup.get(backup_id) + end + + test "it removes outdated backups after creating a fresh one" do + Pleroma.Config.put([Pleroma.Backup, :limit_days], -1) + Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) + user = insert(:user) + + assert {:ok, job1} = Backup.create(user) + + assert {:ok, %Backup{id: backup1_id}} = ObanHelpers.perform(job1) + assert {:ok, job2} = Backup.create(user) + assert Pleroma.Repo.aggregate(Backup, :count) == 2 + assert {:ok, backup2} = ObanHelpers.perform(job2) + + ObanHelpers.perform_all() + + assert [^backup2] = Pleroma.Repo.all(Backup) end test "it creates a zip archive with user data" do @@ -145,7 +173,7 @@ defmodule Pleroma.BackupTest do File.rm!(path) end - describe "it uploads a backup archive" do + describe "it uploads and deletes a backup archive" do setup do clear_config(Pleroma.Uploaders.S3, bucket: "test_bucket", @@ -173,8 +201,16 @@ defmodule Pleroma.BackupTest do test "S3", %{path: path, backup: backup} do Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.S3) - with_mock ExAws, request: fn _ -> {:ok, :ok} end do + with_mock ExAws, + request: fn + %{http_method: :put} -> {:ok, :ok} + %{http_method: :delete} -> {:ok, %{status_code: 204}} + end do assert {:ok, %Pleroma.Upload{}} = Backup.upload(backup, path) + assert {:ok, _backup} = Backup.delete(backup) + end + + with_mock ExAws, request: fn %{http_method: :delete} -> {:ok, %{status_code: 204}} end do end end @@ -182,6 +218,7 @@ defmodule Pleroma.BackupTest do Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) assert {:ok, %Pleroma.Upload{}} = Backup.upload(backup, path) + assert {:ok, _backup} = Backup.delete(backup) end end end diff --git a/test/support/oban_helpers.ex b/test/support/oban_helpers.ex index 9f90a821c..2468f66dc 100644 --- a/test/support/oban_helpers.ex +++ b/test/support/oban_helpers.ex @@ -7,6 +7,8 @@ defmodule Pleroma.Tests.ObanHelpers do Oban test helpers. """ + require Ecto.Query + alias Pleroma.Repo def wipe_all do @@ -15,6 +17,7 @@ defmodule Pleroma.Tests.ObanHelpers do def perform_all do Oban.Job + |> Ecto.Query.where(state: "available") |> Repo.all() |> perform() end From abdffc6b8c2eec8f81ffe89f943f11d1f90d7074 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Fri, 4 Sep 2020 22:00:26 +0400 Subject: [PATCH 11/45] Fix Credo warning --- test/backup_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/backup_test.exs b/test/backup_test.exs index 59aebe360..5fc519eab 100644 --- a/test/backup_test.exs +++ b/test/backup_test.exs @@ -11,9 +11,9 @@ defmodule Pleroma.BackupTest do alias Pleroma.Backup alias Pleroma.Bookmark + alias Pleroma.Tests.ObanHelpers alias Pleroma.Web.CommonAPI alias Pleroma.Workers.BackupWorker - alias Pleroma.Tests.ObanHelpers setup do clear_config([Pleroma.Upload, :uploader]) From 2c73bfe1227065fa203b0b78c9eb12cf86ab3948 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 9 Sep 2020 01:04:00 +0400 Subject: [PATCH 12/45] Add API endpoints for Backups --- lib/pleroma/backup.ex | 7 ++ .../operations/pleroma_backup_operation.ex | 79 +++++++++++++++++ .../controllers/backup_controller.ex | 27 ++++++ .../web/pleroma_api/views/backup_view.ex | 24 ++++++ lib/pleroma/web/router.ex | 3 + .../controllers/backup_controller_test.exs | 84 +++++++++++++++++++ 6 files changed, 224 insertions(+) create mode 100644 lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex create mode 100644 lib/pleroma/web/pleroma_api/controllers/backup_controller.ex create mode 100644 lib/pleroma/web/pleroma_api/views/backup_view.ex create mode 100644 test/web/pleroma_api/controllers/backup_controller_test.exs diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex index bd50fd910..348e537a8 100644 --- a/lib/pleroma/backup.ex +++ b/lib/pleroma/backup.ex @@ -80,6 +80,13 @@ defmodule Pleroma.Backup do |> Repo.one() end + def list(%User{id: user_id}) do + __MODULE__ + |> where(user_id: ^user_id) + |> order_by(desc: :id) + |> Repo.all() + end + def remove_outdated(%__MODULE__{id: latest_id, user_id: user_id}) do __MODULE__ |> where(user_id: ^user_id) diff --git a/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex new file mode 100644 index 000000000..f877ca31b --- /dev/null +++ b/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex @@ -0,0 +1,79 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.PleromaBackupOperation do + alias OpenApiSpex.Operation + alias OpenApiSpex.Schema + alias Pleroma.Web.ApiSpec.Schemas.ApiError + + def open_api_operation(action) do + operation = String.to_existing_atom("#{action}_operation") + apply(__MODULE__, operation, []) + end + + def index_operation do + %Operation{ + tags: ["Backups"], + summary: "List backups", + security: [%{"oAuth" => ["read:account"]}], + operationId: "PleromaAPI.BackupController.index", + responses: %{ + 200 => + Operation.response( + "An array of backups", + "application/json", + %Schema{ + type: :array, + items: backup() + } + ), + 400 => Operation.response("Bad Request", "application/json", ApiError) + } + } + end + + def create_operation do + %Operation{ + tags: ["Backups"], + summary: "Create a backup", + security: [%{"oAuth" => ["read:account"]}], + operationId: "PleromaAPI.BackupController.create", + responses: %{ + 200 => + Operation.response( + "An array of backups", + "application/json", + %Schema{ + type: :array, + items: backup() + } + ), + 400 => Operation.response("Bad Request", "application/json", ApiError) + } + } + end + + defp backup do + %Schema{ + title: "Backup", + description: "Response schema for a backup", + type: :object, + properties: %{ + inserted_at: %Schema{type: :string, format: :"date-time"}, + content_type: %Schema{type: :string}, + file_name: %Schema{type: :string}, + file_size: %Schema{type: :integer}, + processed: %Schema{type: :boolean} + }, + example: %{ + "content_type" => "application/zip", + "file_name" => + "archive-cofe-20200908T195819-1lWrJyJqpsj8-KuHFr7N03lfsYYa5nf2NL-7A9-ddFU.zip", + "file_size" => 1024, + "inserted_at" => "2020-09-08T19:58:20", + "processed" => true + } + } + end +end diff --git a/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex b/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex new file mode 100644 index 000000000..e52c77ff2 --- /dev/null +++ b/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex @@ -0,0 +1,27 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.PleromaAPI.BackupController do + use Pleroma.Web, :controller + + alias Pleroma.Plugs.OAuthScopesPlug + + action_fallback(Pleroma.Web.MastodonAPI.FallbackController) + plug(OAuthScopesPlug, %{scopes: ["read:accounts"]} when action in [:index, :create]) + plug(OpenApiSpex.Plug.CastAndValidate, render_error: Pleroma.Web.ApiSpec.RenderError) + + defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.PleromaBackupOperation + + def index(%{assigns: %{user: user}} = conn, _params) do + backups = Pleroma.Backup.list(user) + render(conn, "index.json", backups: backups) + end + + def create(%{assigns: %{user: user}} = conn, _params) do + with {:ok, _} <- Pleroma.Backup.create(user) do + backups = Pleroma.Backup.list(user) + render(conn, "index.json", backups: backups) + end + end +end diff --git a/lib/pleroma/web/pleroma_api/views/backup_view.ex b/lib/pleroma/web/pleroma_api/views/backup_view.ex new file mode 100644 index 000000000..02b94ce4f --- /dev/null +++ b/lib/pleroma/web/pleroma_api/views/backup_view.ex @@ -0,0 +1,24 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.PleromaAPI.BackupView do + use Pleroma.Web, :view + + alias Pleroma.Backup + alias Pleroma.Web.CommonAPI.Utils + + def render("show.json", %{backup: %Backup{} = backup}) do + %{ + content_type: backup.content_type, + file_name: backup.file_name, + file_size: backup.file_size, + processed: backup.processed, + inserted_at: Utils.to_masto_date(backup.inserted_at) + } + end + + def render("index.json", %{backups: backups}) do + render_many(backups, __MODULE__, "show.json") + end +end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index e22b31b4c..a1a5a1cb5 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -293,6 +293,9 @@ defmodule Pleroma.Web.Router do get("/accounts/mfa/setup/:method", TwoFactorAuthenticationController, :setup) post("/accounts/mfa/confirm/:method", TwoFactorAuthenticationController, :confirm) delete("/accounts/mfa/:method", TwoFactorAuthenticationController, :disable) + + get("/backups", BackupController, :index) + post("/backups", BackupController, :create) end scope "/oauth", Pleroma.Web.OAuth do diff --git a/test/web/pleroma_api/controllers/backup_controller_test.exs b/test/web/pleroma_api/controllers/backup_controller_test.exs new file mode 100644 index 000000000..1ad1b63c4 --- /dev/null +++ b/test/web/pleroma_api/controllers/backup_controller_test.exs @@ -0,0 +1,84 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.PleromaAPI.BackupControllerTest do + use Pleroma.Web.ConnCase + + alias Pleroma.Backup + + setup do + clear_config([Pleroma.Upload, :uploader]) + clear_config([Backup, :limit_days]) + oauth_access(["read:accounts"]) + end + + test "GET /api/pleroma/backups", %{user: user, conn: conn} do + assert {:ok, %Oban.Job{args: %{"backup_id" => backup_id}}} = Backup.create(user) + + backup = Backup.get(backup_id) + + response = + conn + |> get("/api/pleroma/backups") + |> json_response_and_validate_schema(:ok) + + assert [ + %{ + "content_type" => "application/zip", + "file_name" => file_name, + "file_size" => 0, + "processed" => false, + "inserted_at" => _ + } + ] = response + + assert file_name == backup.file_name + + Pleroma.Tests.ObanHelpers.perform_all() + + assert [ + %{ + "file_name" => ^file_name, + "processed" => true + } + ] = + conn + |> get("/api/pleroma/backups") + |> json_response_and_validate_schema(:ok) + end + + test "POST /api/pleroma/backups", %{user: _user, conn: conn} do + assert [ + %{ + "content_type" => "application/zip", + "file_name" => file_name, + "file_size" => 0, + "processed" => false, + "inserted_at" => _ + } + ] = + conn + |> post("/api/pleroma/backups") + |> json_response_and_validate_schema(:ok) + + Pleroma.Tests.ObanHelpers.perform_all() + + assert [ + %{ + "file_name" => ^file_name, + "processed" => true + } + ] = + conn + |> get("/api/pleroma/backups") + |> json_response_and_validate_schema(:ok) + + days = Pleroma.Config.get([Backup, :limit_days]) + + assert %{"error" => "Last export was less than #{days} days ago"} == + conn + |> post("/api/pleroma/backups") + |> json_response_and_validate_schema(400) + end +end From 86ce4afd9338d81f741fa57f962509a6f0f50aff Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 9 Sep 2020 20:02:20 +0400 Subject: [PATCH 13/45] Improve backup urls --- .../api_spec/operations/pleroma_backup_operation.ex | 6 +++--- lib/pleroma/web/pleroma_api/views/backup_view.ex | 6 +++++- .../controllers/backup_controller_test.exs | 11 ++++++----- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex index f877ca31b..6993794db 100644 --- a/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex +++ b/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex @@ -69,9 +69,9 @@ defmodule Pleroma.Web.ApiSpec.PleromaBackupOperation do example: %{ "content_type" => "application/zip", "file_name" => - "archive-cofe-20200908T195819-1lWrJyJqpsj8-KuHFr7N03lfsYYa5nf2NL-7A9-ddFU.zip", - "file_size" => 1024, - "inserted_at" => "2020-09-08T19:58:20", + "https://cofe.fe:4000/media/backups/archive-foobar-20200908T164207-Yr7vuT5Wycv-sN3kSN2iJ0k-9pMo60j9qmvRCdDqIew.zip", + "file_size" => 4105, + "inserted_at" => "2020-09-08T16:42:07.000Z", "processed" => true } } diff --git a/lib/pleroma/web/pleroma_api/views/backup_view.ex b/lib/pleroma/web/pleroma_api/views/backup_view.ex index 02b94ce4f..bf40a001e 100644 --- a/lib/pleroma/web/pleroma_api/views/backup_view.ex +++ b/lib/pleroma/web/pleroma_api/views/backup_view.ex @@ -11,7 +11,7 @@ defmodule Pleroma.Web.PleromaAPI.BackupView do def render("show.json", %{backup: %Backup{} = backup}) do %{ content_type: backup.content_type, - file_name: backup.file_name, + url: download_url(backup), file_size: backup.file_size, processed: backup.processed, inserted_at: Utils.to_masto_date(backup.inserted_at) @@ -21,4 +21,8 @@ defmodule Pleroma.Web.PleromaAPI.BackupView do def render("index.json", %{backups: backups}) do render_many(backups, __MODULE__, "show.json") end + + def download_url(%Backup{file_name: file_name}) do + Pleroma.Web.Endpoint.url() <> "/media/backups/" <> file_name + end end diff --git a/test/web/pleroma_api/controllers/backup_controller_test.exs b/test/web/pleroma_api/controllers/backup_controller_test.exs index 1ad1b63c4..5d2f1206e 100644 --- a/test/web/pleroma_api/controllers/backup_controller_test.exs +++ b/test/web/pleroma_api/controllers/backup_controller_test.exs @@ -6,6 +6,7 @@ defmodule Pleroma.Web.PleromaAPI.BackupControllerTest do use Pleroma.Web.ConnCase alias Pleroma.Backup + alias Pleroma.Web.PleromaAPI.BackupView setup do clear_config([Pleroma.Upload, :uploader]) @@ -26,20 +27,20 @@ defmodule Pleroma.Web.PleromaAPI.BackupControllerTest do assert [ %{ "content_type" => "application/zip", - "file_name" => file_name, + "url" => url, "file_size" => 0, "processed" => false, "inserted_at" => _ } ] = response - assert file_name == backup.file_name + assert url == BackupView.download_url(backup) Pleroma.Tests.ObanHelpers.perform_all() assert [ %{ - "file_name" => ^file_name, + "url" => ^url, "processed" => true } ] = @@ -52,7 +53,7 @@ defmodule Pleroma.Web.PleromaAPI.BackupControllerTest do assert [ %{ "content_type" => "application/zip", - "file_name" => file_name, + "url" => url, "file_size" => 0, "processed" => false, "inserted_at" => _ @@ -66,7 +67,7 @@ defmodule Pleroma.Web.PleromaAPI.BackupControllerTest do assert [ %{ - "file_name" => ^file_name, + "url" => ^url, "processed" => true } ] = From cd13613db3f675b6a9171dea56fc5b03e43ae6b0 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 10 Sep 2020 20:53:06 +0400 Subject: [PATCH 14/45] Fix query --- lib/pleroma/backup.ex | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex index 348e537a8..ce54a413a 100644 --- a/lib/pleroma/backup.ex +++ b/lib/pleroma/backup.ex @@ -8,6 +8,8 @@ defmodule Pleroma.Backup do import Ecto.Changeset import Ecto.Query + require Pleroma.Constants + alias Pleroma.Activity alias Pleroma.Bookmark alias Pleroma.Repo @@ -158,6 +160,7 @@ defmodule Pleroma.Backup do "id": "#{name}.json", "type": "OrderedCollection", "orderedItems": [ + """ ) end @@ -209,13 +212,13 @@ defmodule Pleroma.Backup do opts = %{} |> Map.put(:type, ["Create", "Announce"]) - |> Map.put(:blocking_user, user) - |> Map.put(:muting_user, user) - |> Map.put(:reply_filtering_user, user) - |> Map.put(:announce_filtering_user, user) - |> Map.put(:user, user) + |> Map.put(:actor_id, user.ap_id) - [[user.ap_id], User.following(user), Pleroma.List.memberships(user)] + [ + [Pleroma.Constants.as_public(), user.ap_id], + User.following(user), + Pleroma.List.memberships(user) + ] |> Enum.concat() |> ActivityPub.fetch_activities_query(opts) |> write(dir, "outbox", fn a -> From 386199063b9be9fc30ad403f6afb03bf6ca47298 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 10 Sep 2020 21:09:20 +0400 Subject: [PATCH 15/45] Document `/api/pleroma/backups` API endpoint --- docs/API/pleroma_api.md | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/docs/API/pleroma_api.md b/docs/API/pleroma_api.md index 3fd141bd2..aeb266159 100644 --- a/docs/API/pleroma_api.md +++ b/docs/API/pleroma_api.md @@ -615,3 +615,41 @@ Emoji reactions work a lot like favourites do. They make it possible to react to {"name": "😀", "count": 2, "me": true, "accounts": [{"id" => "xyz.."...}, {"id" => "zyx..."}]} ] ``` + +## `POST /api/pleroma/backups` +### Create a user backup archive + +* Method: `POST` +* Authentication: not required +* Params: none +* Response: JSON +* Example response: + +```json +[{ + "content_type": "application/zip", + "file_size": 0, + "inserted_at": "2020-09-10T16:18:03.000Z", + "processed": false, + "url": "https://example.com/media/backups/archive-foobar-20200910T161803-QUhx6VYDRQ2wfV0SdA2Pfj_2CLM_ATUlw-D5l5TJf4Q.zip" +}] +``` + +## `GET /api/pleroma/backups` +### Lists user backups + +* Method: `GET` +* Authentication: not required +* Params: none +* Response: JSON +* Example response: + +```json +[{ + "content_type": "application/zip", + "file_size": 55457, + "inserted_at": "2020-09-10T16:18:03.000Z", + "processed": true, + "url": "https://example.com/media/backups/archive-foobar-20200910T161803-QUhx6VYDRQ2wfV0SdA2Pfj_2CLM_ATUlw-D5l5TJf4Q.zip" +}] +``` From 27bc121ec00a7b088030d6fb36c7e731f5b072b6 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 15 Sep 2020 18:07:28 +0400 Subject: [PATCH 16/45] Require email --- docs/configuration/cheatsheet.md | 3 +++ lib/pleroma/backup.ex | 19 ++++++++++++++++--- test/backup_test.exs | 16 ++++++++++++++-- 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index cc4081f14..8da8a7bd6 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -1085,6 +1085,9 @@ Control favicons for instances. ## Account Backup +!!! note + Requires enabled email + * `:purge_after_days` an integer, remove backup achives after N days. * `:limit_days` an integer, limit user to export not more often than once per N days. diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex index ce54a413a..3b85dd1c1 100644 --- a/lib/pleroma/backup.ex +++ b/lib/pleroma/backup.ex @@ -31,7 +31,9 @@ defmodule Pleroma.Backup do end def create(user) do - with :ok <- validate_limit(user), + with :ok <- validate_email_enabled(), + :ok <- validate_user_email(user), + :ok <- validate_limit(user), {:ok, backup} <- user |> new() |> Repo.insert() do BackupWorker.process(backup) end @@ -74,6 +76,17 @@ defmodule Pleroma.Backup do end end + defp validate_email_enabled do + if Pleroma.Config.get([Pleroma.Emails.Mailer, :enabled]) do + :ok + else + {:error, "Backups require enabled email"} + end + end + + defp validate_user_email(%User{email: nil}), do: {:error, "Email is required"} + defp validate_user_email(%User{email: email}) when is_binary(email), do: :ok + def get_last(user_id) do __MODULE__ |> where(user_id: ^user_id) @@ -100,7 +113,7 @@ defmodule Pleroma.Backup do def get(id), do: Repo.get(__MODULE__, id) def process(%__MODULE__{} = backup) do - with {:ok, zip_file} <- zip(backup), + with {:ok, zip_file} <- export(backup), {:ok, %{size: size}} <- File.stat(zip_file), {:ok, _upload} <- upload(backup, zip_file) do backup @@ -110,7 +123,7 @@ defmodule Pleroma.Backup do end @files ['actor.json', 'outbox.json', 'likes.json', 'bookmarks.json'] - def zip(%__MODULE__{} = backup) do + def export(%__MODULE__{} = backup) do backup = Repo.preload(backup, :user) name = String.trim_trailing(backup.file_name, ".zip") dir = Path.join(System.tmp_dir!(), name) diff --git a/test/backup_test.exs b/test/backup_test.exs index 5fc519eab..318c8c419 100644 --- a/test/backup_test.exs +++ b/test/backup_test.exs @@ -18,6 +18,18 @@ defmodule Pleroma.BackupTest do setup do clear_config([Pleroma.Upload, :uploader]) clear_config([Pleroma.Backup, :limit_days]) + clear_config([Pleroma.Emails.Mailer, :enabled]) + end + + test "it requries enabled email" do + Pleroma.Config.put([Pleroma.Emails.Mailer, :enabled], false) + user = insert(:user) + assert {:error, "Backups require enabled email"} == Backup.create(user) + end + + test "it requries user's email" do + user = insert(:user, %{email: nil}) + assert {:error, "Email is required"} == Backup.create(user) end test "it creates a backup record and an Oban job" do @@ -91,7 +103,7 @@ defmodule Pleroma.BackupTest do Bookmark.create(user.id, status3.id) assert {:ok, backup} = user |> Backup.new() |> Repo.insert() - assert {:ok, path} = Backup.zip(backup) + assert {:ok, path} = Backup.export(backup) assert {:ok, zipfile} = :zip.zip_open(String.to_charlist(path), [:memory]) assert {:ok, {'actor.json', json}} = :zip.zip_get('actor.json', zipfile) @@ -193,7 +205,7 @@ defmodule Pleroma.BackupTest do Bookmark.create(user.id, status3.id) assert {:ok, backup} = user |> Backup.new() |> Repo.insert() - assert {:ok, path} = Backup.zip(backup) + assert {:ok, path} = Backup.export(backup) [path: path, backup: backup] end From e52dd62e14a956a28a706124464f3ac4b985080d Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 16 Sep 2020 23:21:13 +0400 Subject: [PATCH 17/45] Add configurable temporary directory --- config/config.exs | 3 ++- docs/configuration/cheatsheet.md | 6 ++++++ lib/pleroma/backup.ex | 7 ++++++- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/config/config.exs b/config/config.exs index 09023e2c3..0e12d6e15 100644 --- a/config/config.exs +++ b/config/config.exs @@ -820,7 +820,8 @@ config :pleroma, Pleroma.Web.Auth.Authenticator, Pleroma.Web.Auth.PleromaAuthent config :pleroma, Pleroma.Backup, purge_after_days: 30, - limit_days: 7 + limit_days: 7, + dir: nil # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 8da8a7bd6..9271964f1 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -1090,6 +1090,12 @@ Control favicons for instances. * `:purge_after_days` an integer, remove backup achives after N days. * `:limit_days` an integer, limit user to export not more often than once per N days. +* `:dir` a string with a path to backup temporary directory or `nil` to let Pleroma choose temporary directory in the following order: + 1. the directory named by the TMPDIR environment variable + 2. the directory named by the TEMP environment variable + 3. the directory named by the TMP environment variable + 4. C:\TMP on Windows or /tmp on Unix-like operating systems + 5. as a last resort, the current working directory ## Frontend management diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex index 3b85dd1c1..450dd5b84 100644 --- a/lib/pleroma/backup.ex +++ b/lib/pleroma/backup.ex @@ -126,7 +126,7 @@ defmodule Pleroma.Backup do def export(%__MODULE__{} = backup) do backup = Repo.preload(backup, :user) name = String.trim_trailing(backup.file_name, ".zip") - dir = Path.join(System.tmp_dir!(), name) + dir = dir(name) with :ok <- File.mkdir(dir), :ok <- actor(dir, backup.user), @@ -139,6 +139,11 @@ defmodule Pleroma.Backup do end end + def dir(name) do + dir = Pleroma.Config.get([__MODULE__, :dir]) || System.tmp_dir!() + Path.join(dir, name) + end + def upload(%__MODULE__{} = backup, zip_path) do uploader = Pleroma.Config.get([Pleroma.Upload, :uploader]) From 7fdd81d000d857cbcd5bf442f68c91b1c5b1cebb Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 17 Sep 2020 18:42:24 +0400 Subject: [PATCH 18/45] Add "Your backup is ready" email --- lib/pleroma/emails/user_email.ex | 16 ++++++++++++++++ lib/pleroma/workers/backup_worker.ex | 6 +++++- test/backup_test.exs | 5 ++++- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/emails/user_email.ex b/lib/pleroma/emails/user_email.ex index 1d8c72ae9..f943dda0d 100644 --- a/lib/pleroma/emails/user_email.ex +++ b/lib/pleroma/emails/user_email.ex @@ -189,4 +189,20 @@ defmodule Pleroma.Emails.UserEmail do Router.Helpers.subscription_url(Endpoint, :unsubscribe, token) end + + def backup_is_ready_email(backup) do + %{user: user} = Pleroma.Repo.preload(backup, :user) + download_url = Pleroma.Web.PleromaAPI.BackupView.download_url(backup) + + html_body = """ +

You requested a full backup of your Pleroma account. It's ready for download:

+

+ """ + + new() + |> to(recipient(user)) + |> from(sender()) + |> subject("Your account archive is ready") + |> html_body(html_body) + end end diff --git a/lib/pleroma/workers/backup_worker.ex b/lib/pleroma/workers/backup_worker.ex index f40020794..405d55269 100644 --- a/lib/pleroma/workers/backup_worker.ex +++ b/lib/pleroma/workers/backup_worker.ex @@ -34,7 +34,11 @@ defmodule Pleroma.Workers.BackupWorker do with {:ok, %Backup{} = backup} <- backup_id |> Backup.get() |> Backup.process(), {:ok, _job} <- schedule_deletion(backup), - :ok <- Backup.remove_outdated(backup) do + :ok <- Backup.remove_outdated(backup), + {:ok, _} <- + backup + |> Pleroma.Emails.UserEmail.backup_is_ready_email() + |> Pleroma.Emails.Mailer.deliver() do {:ok, backup} end end diff --git a/test/backup_test.exs b/test/backup_test.exs index 318c8c419..0ea40e6fd 100644 --- a/test/backup_test.exs +++ b/test/backup_test.exs @@ -6,8 +6,9 @@ defmodule Pleroma.BackupTest do use Oban.Testing, repo: Pleroma.Repo use Pleroma.DataCase - import Pleroma.Factory import Mock + import Pleroma.Factory + import Swoosh.TestAssertions alias Pleroma.Backup alias Pleroma.Bookmark @@ -65,6 +66,8 @@ defmodule Pleroma.BackupTest do assert_enqueued(worker: BackupWorker, args: delete_job_args) assert {:ok, backup} = perform_job(BackupWorker, delete_job_args) refute Backup.get(backup_id) + + assert_email_sent(Pleroma.Emails.UserEmail.backup_is_ready_email(backup)) end test "it removes outdated backups after creating a fresh one" do From 7c22c9afb410668d87dcd4a90651d62d9a1e9e4d Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Fri, 18 Sep 2020 22:18:34 +0400 Subject: [PATCH 19/45] Allow admins request user backups --- lib/pleroma/backup.ex | 4 ++-- lib/pleroma/emails/user_email.ex | 20 +++++++++++++----- .../controllers/admin_api_controller.ex | 12 ++++++++++- lib/pleroma/web/router.ex | 2 ++ lib/pleroma/workers/backup_worker.ex | 10 +++++---- .../controllers/admin_api_controller_test.exs | 21 +++++++++++++++++++ 6 files changed, 57 insertions(+), 12 deletions(-) diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex index 450dd5b84..d589f12f1 100644 --- a/lib/pleroma/backup.ex +++ b/lib/pleroma/backup.ex @@ -30,12 +30,12 @@ defmodule Pleroma.Backup do timestamps() end - def create(user) do + def create(user, admin_user_id \\ nil) do with :ok <- validate_email_enabled(), :ok <- validate_user_email(user), :ok <- validate_limit(user), {:ok, backup} <- user |> new() |> Repo.insert() do - BackupWorker.process(backup) + BackupWorker.process(backup, admin_user_id) end end diff --git a/lib/pleroma/emails/user_email.ex b/lib/pleroma/emails/user_email.ex index f943dda0d..5745794ec 100644 --- a/lib/pleroma/emails/user_email.ex +++ b/lib/pleroma/emails/user_email.ex @@ -190,14 +190,24 @@ defmodule Pleroma.Emails.UserEmail do Router.Helpers.subscription_url(Endpoint, :unsubscribe, token) end - def backup_is_ready_email(backup) do + def backup_is_ready_email(backup, admin_user_id \\ nil) do %{user: user} = Pleroma.Repo.preload(backup, :user) download_url = Pleroma.Web.PleromaAPI.BackupView.download_url(backup) - html_body = """ -

You requested a full backup of your Pleroma account. It's ready for download:

-

- """ + html_body = + if is_nil(admin_user_id) do + """ +

You requested a full backup of your Pleroma account. It's ready for download:

+

+ """ + else + admin = Pleroma.Repo.get(User, admin_user_id) + + """ +

Admin @#{admin.nickname} requested a full backup of your Pleroma account. It's ready for download:

+

+ """ + end new() |> to(recipient(user)) diff --git a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex index d5713c3dd..f7d2fe5b1 100644 --- a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex @@ -23,12 +23,14 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do alias Pleroma.Web.Endpoint alias Pleroma.Web.Router + require Logger + @users_page_size 50 plug( OAuthScopesPlug, %{scopes: ["read:accounts"], admin: true} - when action in [:list_users, :user_show, :right_get, :show_user_credentials] + when action in [:list_users, :user_show, :right_get, :show_user_credentials, :create_backup] ) plug( @@ -681,6 +683,14 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do json(conn, %{"status_visibility" => counters}) end + def create_backup(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do + with %User{} = user <- User.get_by_nickname(nickname), + {:ok, _} <- Pleroma.Backup.create(user, admin.id) do + Logger.info("Admin @#{admin.nickname} requested account backup for @{nickname}") + json(conn, "") + end + end + defp page_params(params) do {get_page(params["page"]), get_page_size(params["page_size"])} end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index a1a5a1cb5..e539eeeeb 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -129,6 +129,8 @@ defmodule Pleroma.Web.Router do scope "/api/pleroma/admin", Pleroma.Web.AdminAPI do pipe_through(:admin_api) + post("/backups", AdminAPIController, :create_backup) + post("/users/follow", AdminAPIController, :user_follow) post("/users/unfollow", AdminAPIController, :user_unfollow) diff --git a/lib/pleroma/workers/backup_worker.ex b/lib/pleroma/workers/backup_worker.ex index 405d55269..65754b6a2 100644 --- a/lib/pleroma/workers/backup_worker.ex +++ b/lib/pleroma/workers/backup_worker.ex @@ -8,8 +8,8 @@ defmodule Pleroma.Workers.BackupWorker do alias Oban.Job alias Pleroma.Backup - def process(backup) do - %{"op" => "process", "backup_id" => backup.id} + def process(backup, admin_user_id \\ nil) do + %{"op" => "process", "backup_id" => backup.id, "admin_user_id" => admin_user_id} |> new() |> Oban.insert() end @@ -30,14 +30,16 @@ defmodule Pleroma.Workers.BackupWorker do |> Oban.insert() end - def perform(%Job{args: %{"op" => "process", "backup_id" => backup_id}}) do + def perform(%Job{ + args: %{"op" => "process", "backup_id" => backup_id, "admin_user_id" => admin_user_id} + }) do with {:ok, %Backup{} = backup} <- backup_id |> Backup.get() |> Backup.process(), {:ok, _job} <- schedule_deletion(backup), :ok <- Backup.remove_outdated(backup), {:ok, _} <- backup - |> Pleroma.Emails.UserEmail.backup_is_ready_email() + |> Pleroma.Emails.UserEmail.backup_is_ready_email(admin_user_id) |> Pleroma.Emails.Mailer.deliver() do {:ok, backup} end diff --git a/test/web/admin_api/controllers/admin_api_controller_test.exs b/test/web/admin_api/controllers/admin_api_controller_test.exs index cba6b43d3..4d331779e 100644 --- a/test/web/admin_api/controllers/admin_api_controller_test.exs +++ b/test/web/admin_api/controllers/admin_api_controller_test.exs @@ -2024,6 +2024,27 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do response["status_visibility"] end end + + describe "/api/pleroma/backups" do + test "it creates a backup", %{conn: conn} do + admin = insert(:user, is_admin: true) + token = insert(:oauth_admin_token, user: admin) + user = insert(:user) + + assert "" == + conn + |> assign(:user, admin) + |> assign(:token, token) + |> post("/api/pleroma/admin/backups", %{nickname: user.nickname}) + |> json_response(200) + + assert [backup] = Repo.all(Pleroma.Backup) + + ObanHelpers.perform_all() + + assert_email_sent(Pleroma.Emails.UserEmail.backup_is_ready_email(backup, admin.id)) + end + end end # Needed for testing From 563801716a0aa54e30f680b4e985d4b8c79578fb Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Fri, 18 Sep 2020 22:01:46 +0400 Subject: [PATCH 20/45] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fc1750d1..04b49d80a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mix tasks for controlling user account confirmation status in bulk (`mix pleroma.user confirm_all` and `mix pleroma.user unconfirm_all`) - Mix task for sending confirmation emails to all unconfirmed users (`mix pleroma.email send_confirmation_mails`) - Mix task option for force-unfollowing relays +- Account backup ### Changed From e50314d9d342dbf9a03ca484654b07717592d4bd Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Fri, 18 Sep 2020 22:33:12 +0400 Subject: [PATCH 21/45] Fix export --- lib/pleroma/backup.ex | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex index d589f12f1..242773bdb 100644 --- a/lib/pleroma/backup.ex +++ b/lib/pleroma/backup.ex @@ -191,16 +191,13 @@ defmodule Pleroma.Backup do counter = :counters.new(1, []) query - |> Pleroma.RepoStreamer.chunk_stream(100) - |> Stream.each(fn items -> - Enum.each(items, fn i -> - with {:ok, str} <- fun.(i), - :ok <- IO.write(file, str <> ",\n") do - :counters.add(counter, 1, 1) - end - end) + |> Pleroma.Repo.chunk_stream(100) + |> Enum.each(fn i -> + with {:ok, str} <- fun.(i), + :ok <- IO.write(file, str <> ",\n") do + :counters.add(counter, 1, 1) + end end) - |> Stream.run() total = :counters.get(counter, 1) From a9efd441e242f1d8ac608b866d0cfafe4833243a Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Sun, 20 Sep 2020 19:57:09 +0400 Subject: [PATCH 22/45] Use `Pleroma.Repo.chunk_stream/2` instead of `Pleroma.RepoStreamer.chunk_stream/2` --- lib/pleroma/backup.ex | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex index 242773bdb..f5f39431d 100644 --- a/lib/pleroma/backup.ex +++ b/lib/pleroma/backup.ex @@ -188,18 +188,17 @@ defmodule Pleroma.Backup do with {:ok, file} <- File.open(path, [:write, :utf8]), :ok <- write_header(file, name) do - counter = :counters.new(1, []) - - query - |> Pleroma.Repo.chunk_stream(100) - |> Enum.each(fn i -> - with {:ok, str} <- fun.(i), - :ok <- IO.write(file, str <> ",\n") do - :counters.add(counter, 1, 1) - end - end) - - total = :counters.get(counter, 1) + total = + query + |> Pleroma.Repo.chunk_stream(100) + |> Enum.reduce(0, fn i, acc -> + with {:ok, str} <- fun.(i), + :ok <- IO.write(file, str <> ",\n") do + acc + 1 + else + _ -> acc + end + end) with :ok <- :file.pwrite(file, {:eof, -2}, "\n],\n \"totalItems\": #{total}}") do File.close(file) From 17562bf4147ab03e171b1f1d365a512f2e5b3202 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Sun, 20 Sep 2020 20:43:27 +0400 Subject: [PATCH 23/45] Move API endpoints to `/api/v1/pleroma/backups` --- docs/API/pleroma_api.md | 4 ++-- lib/pleroma/web/router.ex | 6 +++--- .../controllers/backup_controller_test.exs | 14 +++++++------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/API/pleroma_api.md b/docs/API/pleroma_api.md index aeb266159..fa3a9a449 100644 --- a/docs/API/pleroma_api.md +++ b/docs/API/pleroma_api.md @@ -616,7 +616,7 @@ Emoji reactions work a lot like favourites do. They make it possible to react to ] ``` -## `POST /api/pleroma/backups` +## `POST /api/v1/pleroma/backups` ### Create a user backup archive * Method: `POST` @@ -635,7 +635,7 @@ Emoji reactions work a lot like favourites do. They make it possible to react to }] ``` -## `GET /api/pleroma/backups` +## `GET /api/v1/pleroma/backups` ### Lists user backups * Method: `GET` diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index e539eeeeb..ad7e315c7 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -295,9 +295,6 @@ defmodule Pleroma.Web.Router do get("/accounts/mfa/setup/:method", TwoFactorAuthenticationController, :setup) post("/accounts/mfa/confirm/:method", TwoFactorAuthenticationController, :confirm) delete("/accounts/mfa/:method", TwoFactorAuthenticationController, :disable) - - get("/backups", BackupController, :index) - post("/backups", BackupController, :create) end scope "/oauth", Pleroma.Web.OAuth do @@ -358,6 +355,9 @@ defmodule Pleroma.Web.Router do put("/mascot", MascotController, :update) post("/scrobble", ScrobbleController, :create) + + get("/backups", BackupController, :index) + post("/backups", BackupController, :create) end scope [] do diff --git a/test/web/pleroma_api/controllers/backup_controller_test.exs b/test/web/pleroma_api/controllers/backup_controller_test.exs index 5d2f1206e..b2ac74c7d 100644 --- a/test/web/pleroma_api/controllers/backup_controller_test.exs +++ b/test/web/pleroma_api/controllers/backup_controller_test.exs @@ -14,14 +14,14 @@ defmodule Pleroma.Web.PleromaAPI.BackupControllerTest do oauth_access(["read:accounts"]) end - test "GET /api/pleroma/backups", %{user: user, conn: conn} do + test "GET /api/v1/pleroma/backups", %{user: user, conn: conn} do assert {:ok, %Oban.Job{args: %{"backup_id" => backup_id}}} = Backup.create(user) backup = Backup.get(backup_id) response = conn - |> get("/api/pleroma/backups") + |> get("/api/v1/pleroma/backups") |> json_response_and_validate_schema(:ok) assert [ @@ -45,11 +45,11 @@ defmodule Pleroma.Web.PleromaAPI.BackupControllerTest do } ] = conn - |> get("/api/pleroma/backups") + |> get("/api/v1/pleroma/backups") |> json_response_and_validate_schema(:ok) end - test "POST /api/pleroma/backups", %{user: _user, conn: conn} do + test "POST /api/v1/pleroma/backups", %{user: _user, conn: conn} do assert [ %{ "content_type" => "application/zip", @@ -60,7 +60,7 @@ defmodule Pleroma.Web.PleromaAPI.BackupControllerTest do } ] = conn - |> post("/api/pleroma/backups") + |> post("/api/v1/pleroma/backups") |> json_response_and_validate_schema(:ok) Pleroma.Tests.ObanHelpers.perform_all() @@ -72,14 +72,14 @@ defmodule Pleroma.Web.PleromaAPI.BackupControllerTest do } ] = conn - |> get("/api/pleroma/backups") + |> get("/api/v1/pleroma/backups") |> json_response_and_validate_schema(:ok) days = Pleroma.Config.get([Backup, :limit_days]) assert %{"error" => "Last export was less than #{days} days ago"} == conn - |> post("/api/pleroma/backups") + |> post("/api/v1/pleroma/backups") |> json_response_and_validate_schema(400) end end From e4792ce76af3094d378a3a201ca429ae38203696 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Sun, 20 Sep 2020 21:06:16 +0400 Subject: [PATCH 24/45] Do not limit admins --- lib/pleroma/backup.ex | 10 ++++---- .../controllers/admin_api_controller_test.exs | 24 +++++++++++++++++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex index f5f39431d..e2673db80 100644 --- a/lib/pleroma/backup.ex +++ b/lib/pleroma/backup.ex @@ -30,12 +30,12 @@ defmodule Pleroma.Backup do timestamps() end - def create(user, admin_user_id \\ nil) do + def create(user, admin_id \\ nil) do with :ok <- validate_email_enabled(), :ok <- validate_user_email(user), - :ok <- validate_limit(user), + :ok <- validate_limit(user, admin_id), {:ok, backup} <- user |> new() |> Repo.insert() do - BackupWorker.process(backup, admin_user_id) + BackupWorker.process(backup, admin_id) end end @@ -59,7 +59,9 @@ defmodule Pleroma.Backup do end end - defp validate_limit(user) do + defp validate_limit(_user, admin_id) when is_binary(admin_id), do: :ok + + defp validate_limit(user, nil) do case get_last(user.id) do %__MODULE__{inserted_at: inserted_at} -> days = Pleroma.Config.get([Pleroma.Backup, :limit_days]) diff --git a/test/web/admin_api/controllers/admin_api_controller_test.exs b/test/web/admin_api/controllers/admin_api_controller_test.exs index 4d331779e..4b3abce0d 100644 --- a/test/web/admin_api/controllers/admin_api_controller_test.exs +++ b/test/web/admin_api/controllers/admin_api_controller_test.exs @@ -2044,6 +2044,30 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do assert_email_sent(Pleroma.Emails.UserEmail.backup_is_ready_email(backup, admin.id)) end + + test "it doesn't limit admins", %{conn: conn} do + admin = insert(:user, is_admin: true) + token = insert(:oauth_admin_token, user: admin) + user = insert(:user) + + assert "" == + conn + |> assign(:user, admin) + |> assign(:token, token) + |> post("/api/pleroma/admin/backups", %{nickname: user.nickname}) + |> json_response(200) + + assert [_backup] = Repo.all(Pleroma.Backup) + + assert "" == + conn + |> assign(:user, admin) + |> assign(:token, token) + |> post("/api/pleroma/admin/backups", %{nickname: user.nickname}) + |> json_response(200) + + assert Repo.aggregate(Pleroma.Backup, :count) == 2 + end end end From 8baee855d90530def46dc62b81e6a0cb0c315914 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 21 Sep 2020 21:47:36 +0400 Subject: [PATCH 25/45] Fix emails --- lib/pleroma/emails/user_email.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/emails/user_email.ex b/lib/pleroma/emails/user_email.ex index 5745794ec..806a61fd2 100644 --- a/lib/pleroma/emails/user_email.ex +++ b/lib/pleroma/emails/user_email.ex @@ -198,14 +198,14 @@ defmodule Pleroma.Emails.UserEmail do if is_nil(admin_user_id) do """

You requested a full backup of your Pleroma account. It's ready for download:

-

+

#{download_url}

""" else admin = Pleroma.Repo.get(User, admin_user_id) """

Admin @#{admin.nickname} requested a full backup of your Pleroma account. It's ready for download:

-

+

#{download_url}

""" end From f1e4333dd7976e8cbef44a3bcfe5c96bef177c6f Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 23 Sep 2020 20:23:11 +0400 Subject: [PATCH 26/45] Fix test --- test/backup_test.exs | 7 ++++++- .../admin_api/controllers/admin_api_controller_test.exs | 5 ++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/test/backup_test.exs b/test/backup_test.exs index 0ea40e6fd..23c08b680 100644 --- a/test/backup_test.exs +++ b/test/backup_test.exs @@ -67,7 +67,12 @@ defmodule Pleroma.BackupTest do assert {:ok, backup} = perform_job(BackupWorker, delete_job_args) refute Backup.get(backup_id) - assert_email_sent(Pleroma.Emails.UserEmail.backup_is_ready_email(backup)) + email = Pleroma.Emails.UserEmail.backup_is_ready_email(backup) + + assert_email_sent( + to: {user.name, user.email}, + html_body: email.html_body + ) end test "it removes outdated backups after creating a fresh one" do diff --git a/test/web/admin_api/controllers/admin_api_controller_test.exs b/test/web/admin_api/controllers/admin_api_controller_test.exs index 4b3abce0d..a6dc4f62d 100644 --- a/test/web/admin_api/controllers/admin_api_controller_test.exs +++ b/test/web/admin_api/controllers/admin_api_controller_test.exs @@ -2042,7 +2042,10 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do ObanHelpers.perform_all() - assert_email_sent(Pleroma.Emails.UserEmail.backup_is_ready_email(backup, admin.id)) + email = Pleroma.Emails.UserEmail.backup_is_ready_email(backup, admin.id) + + assert String.contains?(email.html_body, "Admin @#{admin.nickname} requested a full backup") + assert_email_sent(to: {user.name, user.email}, html_body: email.html_body) end test "it doesn't limit admins", %{conn: conn} do From 6d5f02a1da81ed7693c5ae364a25bc0b54ee1a38 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Sat, 26 Sep 2020 20:34:44 +0400 Subject: [PATCH 27/45] Fix API documentation --- docs/API/pleroma_api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/API/pleroma_api.md b/docs/API/pleroma_api.md index fa3a9a449..7a0a80dad 100644 --- a/docs/API/pleroma_api.md +++ b/docs/API/pleroma_api.md @@ -620,7 +620,7 @@ Emoji reactions work a lot like favourites do. They make it possible to react to ### Create a user backup archive * Method: `POST` -* Authentication: not required +* Authentication: required * Params: none * Response: JSON * Example response: From d7a5291b4fa3b7568674c0f7643fe287fcd21eff Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Sat, 26 Sep 2020 21:24:35 +0400 Subject: [PATCH 28/45] Use `Jason.encode/1` for likes and bookmarks --- lib/pleroma/backup.ex | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex index e2673db80..b43dc94d6 100644 --- a/lib/pleroma/backup.ex +++ b/lib/pleroma/backup.ex @@ -194,7 +194,8 @@ defmodule Pleroma.Backup do query |> Pleroma.Repo.chunk_stream(100) |> Enum.reduce(0, fn i, acc -> - with {:ok, str} <- fun.(i), + with {:ok, data} <- fun.(i), + {:ok, str} <- Jason.encode(data), :ok <- IO.write(file, str <> ",\n") do acc + 1 else @@ -213,7 +214,7 @@ defmodule Pleroma.Backup do |> where(user_id: ^user_id) |> join(:inner, [b], activity in assoc(b, :activity)) |> select([b, a], %{id: b.id, object: fragment("(?)->>'object'", a.data)}) - |> write(dir, "bookmarks", fn a -> {:ok, "\"#{a.object}\""} end) + |> write(dir, "bookmarks", fn a -> {:ok, a.object} end) end defp likes(dir, user) do @@ -221,7 +222,7 @@ defmodule Pleroma.Backup do |> Activity.Queries.by_actor() |> Activity.Queries.by_type("Like") |> select([like], %{id: like.id, object: fragment("(?)->>'object'", like.data)}) - |> write(dir, "likes", fn a -> {:ok, "\"#{a.object}\""} end) + |> write(dir, "likes", fn a -> {:ok, a.object} end) end defp statuses(dir, user) do @@ -239,7 +240,7 @@ defmodule Pleroma.Backup do |> ActivityPub.fetch_activities_query(opts) |> write(dir, "outbox", fn a -> with {:ok, activity} <- Transmogrifier.prepare_outgoing(a.data) do - activity |> Map.delete("@context") |> Jason.encode() + {:ok, Map.delete(activity, "@context")} end end) end From 9af9f02f4b3c4eac859a69ab9b2f546a91110287 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Sat, 26 Sep 2020 21:45:03 +0400 Subject: [PATCH 29/45] Use Gettext for error messages --- lib/pleroma/backup.ex | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex index b43dc94d6..0ebaf02e5 100644 --- a/lib/pleroma/backup.ex +++ b/lib/pleroma/backup.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Backup do import Ecto.Changeset import Ecto.Query + import Pleroma.Web.Gettext require Pleroma.Constants @@ -70,7 +71,14 @@ defmodule Pleroma.Backup do if diff > days do :ok else - {:error, "Last export was less than #{days} days ago"} + {:error, + dngettext( + "errors", + "Last export was less than a day ago", + "Last export was less than %{days} days ago", + days, + days: days + )} end nil -> @@ -82,11 +90,14 @@ defmodule Pleroma.Backup do if Pleroma.Config.get([Pleroma.Emails.Mailer, :enabled]) do :ok else - {:error, "Backups require enabled email"} + {:error, dgettext("errors", "Backups require enabled email")} end end - defp validate_user_email(%User{email: nil}), do: {:error, "Email is required"} + defp validate_user_email(%User{email: nil}) do + {:error, dgettext("errors", "Email is required")} + end + defp validate_user_email(%User{email: email}) when is_binary(email), do: :ok def get_last(user_id) do From 08972dd135c200073f5de0c8731b886cc2e72eeb Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Sat, 26 Sep 2020 21:50:31 +0400 Subject: [PATCH 30/45] Use Path.join/2 --- lib/pleroma/backup.ex | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex index 0ebaf02e5..cee51d7c1 100644 --- a/lib/pleroma/backup.ex +++ b/lib/pleroma/backup.ex @@ -55,7 +55,7 @@ defmodule Pleroma.Backup do def delete(backup) do uploader = Pleroma.Config.get([Pleroma.Upload, :uploader]) - with :ok <- uploader.delete_file("backups/" <> backup.file_name) do + with :ok <- uploader.delete_file(Path.join("backups", backup.file_name)) do Repo.delete(backup) end end @@ -164,7 +164,7 @@ defmodule Pleroma.Backup do name: backup.file_name, tempfile: zip_path, content_type: backup.content_type, - path: "backups/" <> backup.file_name + path: Path.join("backups", backup.file_name) } with {:ok, _} <- Pleroma.Uploaders.Uploader.put_file(uploader, upload), @@ -178,7 +178,7 @@ defmodule Pleroma.Backup do UserView.render("user.json", %{user: user}) |> Map.merge(%{"likes" => "likes.json", "bookmarks" => "bookmarks.json"}) |> Jason.encode() do - File.write(dir <> "/actor.json", json) + File.write(Path.join(dir, "actor.json"), json) end end @@ -197,7 +197,7 @@ defmodule Pleroma.Backup do end defp write(query, dir, name, fun) do - path = dir <> "/#{name}.json" + path = Path.join(dir, "#{name}.json") with {:ok, file} <- File.open(path, [:write, :utf8]), :ok <- write_header(file, name) do From 8545d533ddee2978e9bf7f3284cc7dcb822a77e6 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Sat, 26 Sep 2020 21:53:04 +0400 Subject: [PATCH 31/45] Use to_string/1 instead of :binary.list_to_bin/1 --- lib/pleroma/backup.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex index cee51d7c1..629e879a7 100644 --- a/lib/pleroma/backup.ex +++ b/lib/pleroma/backup.ex @@ -148,7 +148,7 @@ defmodule Pleroma.Backup do :ok <- bookmarks(dir, backup.user), {:ok, zip_path} <- :zip.create(String.to_charlist(dir <> ".zip"), @files, cwd: dir), {:ok, _} <- File.rm_rf(dir) do - {:ok, :binary.list_to_bin(zip_path)} + {:ok, to_string(zip_path)} end end From bc3db724030707e9903d161a70b10fe217a83212 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Sat, 26 Sep 2020 23:16:56 +0400 Subject: [PATCH 32/45] Use ModerationLog instead of Logger --- lib/pleroma/moderation_log.ex | 10 ++++++++ .../controllers/admin_api_controller.ex | 3 ++- .../controllers/admin_api_controller_test.exs | 23 +++++++++++++++++-- 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/moderation_log.ex b/lib/pleroma/moderation_log.ex index 47036a6f6..be1e81467 100644 --- a/lib/pleroma/moderation_log.ex +++ b/lib/pleroma/moderation_log.ex @@ -651,6 +651,16 @@ defmodule Pleroma.ModerationLog do "@#{actor_nickname} deleted chat message ##{subject_id}" end + def get_log_entry_message(%ModerationLog{ + data: %{ + "actor" => %{"nickname" => actor_nickname}, + "action" => "create_backup", + "subject" => %{"nickname" => user_nickname} + } + }) do + "@#{actor_nickname} requested account backup for @#{user_nickname}" + end + defp nicknames_to_string(nicknames) do nicknames |> Enum.map(&"@#{&1}") diff --git a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex index f7d2fe5b1..8b5310d80 100644 --- a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex @@ -686,7 +686,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do def create_backup(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do with %User{} = user <- User.get_by_nickname(nickname), {:ok, _} <- Pleroma.Backup.create(user, admin.id) do - Logger.info("Admin @#{admin.nickname} requested account backup for @{nickname}") + ModerationLog.insert_log(%{actor: admin, subject: user, action: "create_backup"}) + json(conn, "") end end diff --git a/test/web/admin_api/controllers/admin_api_controller_test.exs b/test/web/admin_api/controllers/admin_api_controller_test.exs index a6dc4f62d..34d48c2c1 100644 --- a/test/web/admin_api/controllers/admin_api_controller_test.exs +++ b/test/web/admin_api/controllers/admin_api_controller_test.exs @@ -2027,9 +2027,9 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do describe "/api/pleroma/backups" do test "it creates a backup", %{conn: conn} do - admin = insert(:user, is_admin: true) + admin = %{id: admin_id, nickname: admin_nickname} = insert(:user, is_admin: true) token = insert(:oauth_admin_token, user: admin) - user = insert(:user) + user = %{id: user_id, nickname: user_nickname} = insert(:user) assert "" == conn @@ -2046,6 +2046,25 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do assert String.contains?(email.html_body, "Admin @#{admin.nickname} requested a full backup") assert_email_sent(to: {user.name, user.email}, html_body: email.html_body) + + log_message = "@#{admin_nickname} requested account backup for @#{user_nickname}" + + assert [ + %{ + data: %{ + "action" => "create_backup", + "actor" => %{ + "id" => ^admin_id, + "nickname" => ^admin_nickname + }, + "message" => ^log_message, + "subject" => %{ + "id" => ^user_id, + "nickname" => ^user_nickname + } + } + } + ] = Pleroma.ModerationLog |> Repo.all() end test "it doesn't limit admins", %{conn: conn} do From 98f32cf8204113c6d019653c22e446e558147248 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 19 Oct 2020 15:30:32 +0400 Subject: [PATCH 33/45] Fix tests --- lib/pleroma/web/pleroma_api/controllers/backup_controller.ex | 2 +- test/backup_test.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex b/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex index e52c77ff2..8e3d081f3 100644 --- a/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex @@ -5,7 +5,7 @@ defmodule Pleroma.Web.PleromaAPI.BackupController do use Pleroma.Web, :controller - alias Pleroma.Plugs.OAuthScopesPlug + alias Pleroma.Web.Plugs.OAuthScopesPlug action_fallback(Pleroma.Web.MastodonAPI.FallbackController) plug(OAuthScopesPlug, %{scopes: ["read:accounts"]} when action in [:index, :create]) diff --git a/test/backup_test.exs b/test/backup_test.exs index 23c08b680..078e03621 100644 --- a/test/backup_test.exs +++ b/test/backup_test.exs @@ -19,7 +19,7 @@ defmodule Pleroma.BackupTest do setup do clear_config([Pleroma.Upload, :uploader]) clear_config([Pleroma.Backup, :limit_days]) - clear_config([Pleroma.Emails.Mailer, :enabled]) + clear_config([Pleroma.Emails.Mailer, :enabled], true) end test "it requries enabled email" do From c1976d5b19fbceaecf1f52711fe35e1c7d5312aa Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 19 Oct 2020 18:14:39 +0400 Subject: [PATCH 34/45] Fix credo warnings --- test/{ => pleroma}/backup_test.exs | 0 .../web/pleroma_api/controllers/backup_controller_test.exs | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename test/{ => pleroma}/backup_test.exs (100%) rename test/{ => pleroma}/web/pleroma_api/controllers/backup_controller_test.exs (100%) diff --git a/test/backup_test.exs b/test/pleroma/backup_test.exs similarity index 100% rename from test/backup_test.exs rename to test/pleroma/backup_test.exs diff --git a/test/web/pleroma_api/controllers/backup_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/backup_controller_test.exs similarity index 100% rename from test/web/pleroma_api/controllers/backup_controller_test.exs rename to test/pleroma/web/pleroma_api/controllers/backup_controller_test.exs From ad605e3e16ba3f6ee3df7a0a3e6705036fef369f Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 20 Oct 2020 17:16:58 +0400 Subject: [PATCH 35/45] Rename `Pleroma.Backup` to `Pleroma.User.Backup` --- config/config.exs | 2 +- config/description.exs | 2 +- docs/configuration/cheatsheet.md | 2 +- lib/pleroma/{ => user}/backup.ex | 4 ++-- .../web/admin_api/controllers/admin_api_controller.ex | 2 +- .../web/pleroma_api/controllers/backup_controller.ex | 7 ++++--- lib/pleroma/web/pleroma_api/views/backup_view.ex | 2 +- lib/pleroma/workers/backup_worker.ex | 4 ++-- test/pleroma/{ => user}/backup_test.exs | 10 +++++----- .../controllers/admin_api_controller_test.exs | 6 +++--- .../pleroma_api/controllers/backup_controller_test.exs | 2 +- 11 files changed, 22 insertions(+), 21 deletions(-) rename lib/pleroma/{ => user}/backup.ex (98%) rename test/pleroma/{ => user}/backup_test.exs (97%) diff --git a/config/config.exs b/config/config.exs index 63e386250..c758c818c 100644 --- a/config/config.exs +++ b/config/config.exs @@ -831,7 +831,7 @@ config :floki, :html_parser, Floki.HTMLParser.FastHtml config :pleroma, Pleroma.Web.Auth.Authenticator, Pleroma.Web.Auth.PleromaAuthenticator -config :pleroma, Pleroma.Backup, +config :pleroma, Pleroma.User.Backup, purge_after_days: 30, limit_days: 7, dir: nil diff --git a/config/description.exs b/config/description.exs index 88f2a6133..9f23b6d3d 100644 --- a/config/description.exs +++ b/config/description.exs @@ -3731,7 +3731,7 @@ config :pleroma, :config_description, [ }, %{ group: :pleroma, - key: Pleroma.Backup, + key: Pleroma.User.Backup, type: :group, description: "Account Backup", children: [ diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index aafc43f3d..b40a2aebf 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -1077,7 +1077,7 @@ Control favicons for instances. * `enabled`: Allow/disallow displaying and getting instances favicons -## Account Backup +## Pleroma.User.Backup !!! note Requires enabled email diff --git a/lib/pleroma/backup.ex b/lib/pleroma/user/backup.ex similarity index 98% rename from lib/pleroma/backup.ex rename to lib/pleroma/user/backup.ex index 629e879a7..a9041fd94 100644 --- a/lib/pleroma/backup.ex +++ b/lib/pleroma/user/backup.ex @@ -2,7 +2,7 @@ # Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Backup do +defmodule Pleroma.User.Backup do use Ecto.Schema import Ecto.Changeset @@ -65,7 +65,7 @@ defmodule Pleroma.Backup do defp validate_limit(user, nil) do case get_last(user.id) do %__MODULE__{inserted_at: inserted_at} -> - days = Pleroma.Config.get([Pleroma.Backup, :limit_days]) + days = Pleroma.Config.get([__MODULE__, :limit_days]) diff = Timex.diff(NaiveDateTime.utc_now(), inserted_at, :days) if diff > days do diff --git a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex index a4f0d7d34..0a27c5861 100644 --- a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex @@ -685,7 +685,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do def create_backup(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do with %User{} = user <- User.get_by_nickname(nickname), - {:ok, _} <- Pleroma.Backup.create(user, admin.id) do + {:ok, _} <- Pleroma.User.Backup.create(user, admin.id) do ModerationLog.insert_log(%{actor: admin, subject: user, action: "create_backup"}) json(conn, "") diff --git a/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex b/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex index 8e3d081f3..bd7b36880 100644 --- a/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex @@ -6,6 +6,7 @@ defmodule Pleroma.Web.PleromaAPI.BackupController do use Pleroma.Web, :controller alias Pleroma.Web.Plugs.OAuthScopesPlug + alias Pleroma.User.Backup action_fallback(Pleroma.Web.MastodonAPI.FallbackController) plug(OAuthScopesPlug, %{scopes: ["read:accounts"]} when action in [:index, :create]) @@ -14,13 +15,13 @@ defmodule Pleroma.Web.PleromaAPI.BackupController do defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.PleromaBackupOperation def index(%{assigns: %{user: user}} = conn, _params) do - backups = Pleroma.Backup.list(user) + backups = Backup.list(user) render(conn, "index.json", backups: backups) end def create(%{assigns: %{user: user}} = conn, _params) do - with {:ok, _} <- Pleroma.Backup.create(user) do - backups = Pleroma.Backup.list(user) + with {:ok, _} <- Backup.create(user) do + backups = Backup.list(user) render(conn, "index.json", backups: backups) end end diff --git a/lib/pleroma/web/pleroma_api/views/backup_view.ex b/lib/pleroma/web/pleroma_api/views/backup_view.ex index bf40a001e..af75876aa 100644 --- a/lib/pleroma/web/pleroma_api/views/backup_view.ex +++ b/lib/pleroma/web/pleroma_api/views/backup_view.ex @@ -5,7 +5,7 @@ defmodule Pleroma.Web.PleromaAPI.BackupView do use Pleroma.Web, :view - alias Pleroma.Backup + alias Pleroma.User.Backup alias Pleroma.Web.CommonAPI.Utils def render("show.json", %{backup: %Backup{} = backup}) do diff --git a/lib/pleroma/workers/backup_worker.ex b/lib/pleroma/workers/backup_worker.ex index 65754b6a2..5b4985983 100644 --- a/lib/pleroma/workers/backup_worker.ex +++ b/lib/pleroma/workers/backup_worker.ex @@ -6,7 +6,7 @@ defmodule Pleroma.Workers.BackupWorker do use Oban.Worker, queue: :backup, max_attempts: 1 alias Oban.Job - alias Pleroma.Backup + alias Pleroma.User.Backup def process(backup, admin_user_id \\ nil) do %{"op" => "process", "backup_id" => backup.id, "admin_user_id" => admin_user_id} @@ -15,7 +15,7 @@ defmodule Pleroma.Workers.BackupWorker do end def schedule_deletion(backup) do - days = Pleroma.Config.get([Pleroma.Backup, :purge_after_days]) + days = Pleroma.Config.get([Backup, :purge_after_days]) time = 60 * 60 * 24 * days scheduled_at = Calendar.NaiveDateTime.add!(backup.inserted_at, time) diff --git a/test/pleroma/backup_test.exs b/test/pleroma/user/backup_test.exs similarity index 97% rename from test/pleroma/backup_test.exs rename to test/pleroma/user/backup_test.exs index 078e03621..5ad587833 100644 --- a/test/pleroma/backup_test.exs +++ b/test/pleroma/user/backup_test.exs @@ -2,7 +2,7 @@ # Copyright © 2017-2020 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.BackupTest do +defmodule Pleroma.User.BackupTest do use Oban.Testing, repo: Pleroma.Repo use Pleroma.DataCase @@ -10,7 +10,7 @@ defmodule Pleroma.BackupTest do import Pleroma.Factory import Swoosh.TestAssertions - alias Pleroma.Backup + alias Pleroma.User.Backup alias Pleroma.Bookmark alias Pleroma.Tests.ObanHelpers alias Pleroma.Web.CommonAPI @@ -18,7 +18,7 @@ defmodule Pleroma.BackupTest do setup do clear_config([Pleroma.Upload, :uploader]) - clear_config([Pleroma.Backup, :limit_days]) + clear_config([Backup, :limit_days]) clear_config([Pleroma.Emails.Mailer, :enabled], true) end @@ -44,7 +44,7 @@ defmodule Pleroma.BackupTest do test "it return an error if the export limit is over" do %{id: user_id} = user = insert(:user) - limit_days = Pleroma.Config.get([Pleroma.Backup, :limit_days]) + limit_days = Pleroma.Config.get([Backup, :limit_days]) assert {:ok, %Oban.Job{args: args}} = Backup.create(user) backup = Backup.get(args["backup_id"]) assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup @@ -76,7 +76,7 @@ defmodule Pleroma.BackupTest do end test "it removes outdated backups after creating a fresh one" do - Pleroma.Config.put([Pleroma.Backup, :limit_days], -1) + Pleroma.Config.put([Backup, :limit_days], -1) Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) user = insert(:user) diff --git a/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs b/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs index 34d48c2c1..5efe8ef71 100644 --- a/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs @@ -2038,7 +2038,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do |> post("/api/pleroma/admin/backups", %{nickname: user.nickname}) |> json_response(200) - assert [backup] = Repo.all(Pleroma.Backup) + assert [backup] = Repo.all(Pleroma.User.Backup) ObanHelpers.perform_all() @@ -2079,7 +2079,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do |> post("/api/pleroma/admin/backups", %{nickname: user.nickname}) |> json_response(200) - assert [_backup] = Repo.all(Pleroma.Backup) + assert [_backup] = Repo.all(Pleroma.User.Backup) assert "" == conn @@ -2088,7 +2088,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do |> post("/api/pleroma/admin/backups", %{nickname: user.nickname}) |> json_response(200) - assert Repo.aggregate(Pleroma.Backup, :count) == 2 + assert Repo.aggregate(Pleroma.User.Backup, :count) == 2 end end end diff --git a/test/pleroma/web/pleroma_api/controllers/backup_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/backup_controller_test.exs index b2ac74c7d..f1941f6dd 100644 --- a/test/pleroma/web/pleroma_api/controllers/backup_controller_test.exs +++ b/test/pleroma/web/pleroma_api/controllers/backup_controller_test.exs @@ -5,7 +5,7 @@ defmodule Pleroma.Web.PleromaAPI.BackupControllerTest do use Pleroma.Web.ConnCase - alias Pleroma.Backup + alias Pleroma.User.Backup alias Pleroma.Web.PleromaAPI.BackupView setup do From 034ac43f3a91178694d3c621c52ce68207ec4f69 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 20 Oct 2020 17:47:04 +0400 Subject: [PATCH 36/45] Fix credo warnings --- lib/pleroma/web/pleroma_api/controllers/backup_controller.ex | 2 +- test/pleroma/user/backup_test.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex b/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex index bd7b36880..dd0a2e22f 100644 --- a/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex @@ -5,8 +5,8 @@ defmodule Pleroma.Web.PleromaAPI.BackupController do use Pleroma.Web, :controller - alias Pleroma.Web.Plugs.OAuthScopesPlug alias Pleroma.User.Backup + alias Pleroma.Web.Plugs.OAuthScopesPlug action_fallback(Pleroma.Web.MastodonAPI.FallbackController) plug(OAuthScopesPlug, %{scopes: ["read:accounts"]} when action in [:index, :create]) diff --git a/test/pleroma/user/backup_test.exs b/test/pleroma/user/backup_test.exs index 5ad587833..513798911 100644 --- a/test/pleroma/user/backup_test.exs +++ b/test/pleroma/user/backup_test.exs @@ -10,9 +10,9 @@ defmodule Pleroma.User.BackupTest do import Pleroma.Factory import Swoosh.TestAssertions - alias Pleroma.User.Backup alias Pleroma.Bookmark alias Pleroma.Tests.ObanHelpers + alias Pleroma.User.Backup alias Pleroma.Web.CommonAPI alias Pleroma.Workers.BackupWorker From 4f90077767b416f3469fe7c8acfaa6932c579ec2 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 28 Oct 2020 15:32:44 +0400 Subject: [PATCH 37/45] Fix warning --- test/pleroma/user/backup_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/pleroma/user/backup_test.exs b/test/pleroma/user/backup_test.exs index 513798911..f68e4a029 100644 --- a/test/pleroma/user/backup_test.exs +++ b/test/pleroma/user/backup_test.exs @@ -82,7 +82,7 @@ defmodule Pleroma.User.BackupTest do assert {:ok, job1} = Backup.create(user) - assert {:ok, %Backup{id: backup1_id}} = ObanHelpers.perform(job1) + assert {:ok, %Backup{}} = ObanHelpers.perform(job1) assert {:ok, job2} = Backup.create(user) assert Pleroma.Repo.aggregate(Backup, :count) == 2 assert {:ok, backup2} = ObanHelpers.perform(job2) From 1a98476f48db380b5d26b882dad7fba745b3f160 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Fri, 30 Oct 2020 16:39:14 +0400 Subject: [PATCH 38/45] Remove unused aliases --- .../web/admin_api/controllers/admin_api_controller_test.exs | 1 - test/pleroma/web/admin_api/controllers/chat_controller_test.exs | 1 - .../admin_api/controllers/instance_document_controller_test.exs | 1 - .../web/admin_api/controllers/o_auth_app_controller_test.exs | 1 - test/pleroma/web/admin_api/controllers/relay_controller_test.exs | 1 - .../pleroma/web/admin_api/controllers/report_controller_test.exs | 1 - .../pleroma/web/admin_api/controllers/status_controller_test.exs | 1 - test/pleroma/web/admin_api/controllers/user_controller_test.exs | 1 - .../web/mastodon_api/controllers/timeline_controller_test.exs | 1 - .../web/pleroma_api/controllers/user_import_controller_test.exs | 1 - test/pleroma/web/plugs/http_security_plug_test.exs | 1 - test/pleroma/web/twitter_api/remote_follow_controller_test.exs | 1 - 12 files changed, 12 deletions(-) diff --git a/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs b/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs index 34b26dddf..e2d6d5f6e 100644 --- a/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs @@ -11,7 +11,6 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do import Swoosh.TestAssertions alias Pleroma.Activity - alias Pleroma.Config alias Pleroma.MFA alias Pleroma.ModerationLog alias Pleroma.Repo diff --git a/test/pleroma/web/admin_api/controllers/chat_controller_test.exs b/test/pleroma/web/admin_api/controllers/chat_controller_test.exs index bd4c9c9d1..5aefa1e60 100644 --- a/test/pleroma/web/admin_api/controllers/chat_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/chat_controller_test.exs @@ -9,7 +9,6 @@ defmodule Pleroma.Web.AdminAPI.ChatControllerTest do alias Pleroma.Chat alias Pleroma.Chat.MessageReference - alias Pleroma.Config alias Pleroma.ModerationLog alias Pleroma.Object alias Pleroma.Repo diff --git a/test/pleroma/web/admin_api/controllers/instance_document_controller_test.exs b/test/pleroma/web/admin_api/controllers/instance_document_controller_test.exs index 5f7b042f6..ce867dd0e 100644 --- a/test/pleroma/web/admin_api/controllers/instance_document_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/instance_document_controller_test.exs @@ -5,7 +5,6 @@ defmodule Pleroma.Web.AdminAPI.InstanceDocumentControllerTest do use Pleroma.Web.ConnCase, async: true import Pleroma.Factory - alias Pleroma.Config @dir "test/tmp/instance_static" @default_instance_panel ~s(

Welcome to Pleroma!

) diff --git a/test/pleroma/web/admin_api/controllers/o_auth_app_controller_test.exs b/test/pleroma/web/admin_api/controllers/o_auth_app_controller_test.exs index ed7c4172c..f388375d1 100644 --- a/test/pleroma/web/admin_api/controllers/o_auth_app_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/o_auth_app_controller_test.exs @@ -8,7 +8,6 @@ defmodule Pleroma.Web.AdminAPI.OAuthAppControllerTest do import Pleroma.Factory - alias Pleroma.Config alias Pleroma.Web setup do diff --git a/test/pleroma/web/admin_api/controllers/relay_controller_test.exs b/test/pleroma/web/admin_api/controllers/relay_controller_test.exs index adadf2b5c..b4c5e7567 100644 --- a/test/pleroma/web/admin_api/controllers/relay_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/relay_controller_test.exs @@ -7,7 +7,6 @@ defmodule Pleroma.Web.AdminAPI.RelayControllerTest do import Pleroma.Factory - alias Pleroma.Config alias Pleroma.ModerationLog alias Pleroma.Repo alias Pleroma.User diff --git a/test/pleroma/web/admin_api/controllers/report_controller_test.exs b/test/pleroma/web/admin_api/controllers/report_controller_test.exs index 57946e6bb..fa746d6ea 100644 --- a/test/pleroma/web/admin_api/controllers/report_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/report_controller_test.exs @@ -8,7 +8,6 @@ defmodule Pleroma.Web.AdminAPI.ReportControllerTest do import Pleroma.Factory alias Pleroma.Activity - alias Pleroma.Config alias Pleroma.ModerationLog alias Pleroma.Repo alias Pleroma.ReportNote diff --git a/test/pleroma/web/admin_api/controllers/status_controller_test.exs b/test/pleroma/web/admin_api/controllers/status_controller_test.exs index eff78fb0a..a18ef9e4b 100644 --- a/test/pleroma/web/admin_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/status_controller_test.exs @@ -8,7 +8,6 @@ defmodule Pleroma.Web.AdminAPI.StatusControllerTest do import Pleroma.Factory alias Pleroma.Activity - alias Pleroma.Config alias Pleroma.ModerationLog alias Pleroma.Repo alias Pleroma.User diff --git a/test/pleroma/web/admin_api/controllers/user_controller_test.exs b/test/pleroma/web/admin_api/controllers/user_controller_test.exs index da26caf25..5705306c7 100644 --- a/test/pleroma/web/admin_api/controllers/user_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/user_controller_test.exs @@ -9,7 +9,6 @@ defmodule Pleroma.Web.AdminAPI.UserControllerTest do import Mock import Pleroma.Factory - alias Pleroma.Config alias Pleroma.HTML alias Pleroma.ModerationLog alias Pleroma.Repo diff --git a/test/pleroma/web/mastodon_api/controllers/timeline_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/timeline_controller_test.exs index c6e0268fd..9f1ee0424 100644 --- a/test/pleroma/web/mastodon_api/controllers/timeline_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/timeline_controller_test.exs @@ -8,7 +8,6 @@ defmodule Pleroma.Web.MastodonAPI.TimelineControllerTest do import Pleroma.Factory import Tesla.Mock - alias Pleroma.Config alias Pleroma.User alias Pleroma.Web.CommonAPI diff --git a/test/pleroma/web/pleroma_api/controllers/user_import_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/user_import_controller_test.exs index 433c97e81..68723de71 100644 --- a/test/pleroma/web/pleroma_api/controllers/user_import_controller_test.exs +++ b/test/pleroma/web/pleroma_api/controllers/user_import_controller_test.exs @@ -6,7 +6,6 @@ defmodule Pleroma.Web.PleromaAPI.UserImportControllerTest do use Pleroma.Web.ConnCase use Oban.Testing, repo: Pleroma.Repo - alias Pleroma.Config alias Pleroma.Tests.ObanHelpers import Pleroma.Factory diff --git a/test/pleroma/web/plugs/http_security_plug_test.exs b/test/pleroma/web/plugs/http_security_plug_test.exs index 2297e3dac..df2b5ebb3 100644 --- a/test/pleroma/web/plugs/http_security_plug_test.exs +++ b/test/pleroma/web/plugs/http_security_plug_test.exs @@ -5,7 +5,6 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlugTest do use Pleroma.Web.ConnCase - alias Pleroma.Config alias Plug.Conn describe "http security enabled" do diff --git a/test/pleroma/web/twitter_api/remote_follow_controller_test.exs b/test/pleroma/web/twitter_api/remote_follow_controller_test.exs index 3852c7ce9..a3e784d13 100644 --- a/test/pleroma/web/twitter_api/remote_follow_controller_test.exs +++ b/test/pleroma/web/twitter_api/remote_follow_controller_test.exs @@ -5,7 +5,6 @@ defmodule Pleroma.Web.TwitterAPI.RemoteFollowControllerTest do use Pleroma.Web.ConnCase - alias Pleroma.Config alias Pleroma.MFA alias Pleroma.MFA.TOTP alias Pleroma.User From d1698267a27bd5084916f5f6f36d66b1ff2ffc5f Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Sat, 31 Oct 2020 00:26:11 +0400 Subject: [PATCH 39/45] Fix credo warning --- lib/pleroma/web/router.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 9592d0f38..efe67ad7a 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -148,7 +148,7 @@ defmodule Pleroma.Web.Router do scope "/api/pleroma/admin", Pleroma.Web.AdminAPI do pipe_through(:admin_api) - + put("/users/disable_mfa", AdminAPIController, :disable_mfa) put("/users/tag", AdminAPIController, :tag_users) delete("/users/tag", AdminAPIController, :untag_users) From 8e41baff40555ef7c74c8842d6fbfebc2368631a Mon Sep 17 00:00:00 2001 From: eugenijm Date: Sat, 31 Oct 2020 05:50:48 +0300 Subject: [PATCH 40/45] Add idempotency_key to the chat_message entity. --- CHANGELOG.md | 1 + docs/API/chats.md | 5 ++++- lib/pleroma/application.ex | 9 ++++++++- lib/pleroma/web/activity_pub/side_effects.ex | 6 ++++++ lib/pleroma/web/common_api.ex | 3 ++- .../web/pleroma_api/controllers/chat_controller.ex | 10 +++++++++- .../pleroma_api/views/chat/message_reference_view.ex | 11 +++++++++++ .../pleroma_api/controllers/chat_controller_test.exs | 2 ++ .../views/chat_message_reference_view_test.exs | 5 ++++- test/pleroma/web/streamer_test.exs | 4 +++- 10 files changed, 50 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 11820d313..bb02d7b32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Pleroma API: Pagination for remote/local packs and emoji. - Admin API: (`GET /api/pleroma/admin/users`) added filters user by `unconfirmed` status - Admin API: (`GET /api/pleroma/admin/users`) added filters user by `actor_type` +- Pleroma API: Add `idempotency_key` to the chat message entity that can be used for optimistic message sending. diff --git a/docs/API/chats.md b/docs/API/chats.md index aa6119670..9857aac67 100644 --- a/docs/API/chats.md +++ b/docs/API/chats.md @@ -173,11 +173,14 @@ Returned data: "created_at": "2020-04-21T15:06:45.000Z", "emojis": [], "id": "12", - "unread": false + "unread": false, + "idempotency_key": "75442486-0874-440c-9db1-a7006c25a31f" } ] ``` +- idempotency_key: The copy of the `idempotency-key` HTTP request header that can be used for optimistic message sending. Included only during the first few minutes after the message creation. + ### Posting a chat message Posting a chat message for given Chat id works like this: diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 51e9dda3b..7c4cd9626 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -168,7 +168,11 @@ defmodule Pleroma.Application do build_cachex("web_resp", limit: 2500), build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10), build_cachex("failed_proxy_url", limit: 2500), - build_cachex("banned_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000) + build_cachex("banned_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000), + build_cachex("chat_message_id_idempotency_key", + expiration: chat_message_id_idempotency_key_expiration(), + limit: 500_000 + ) ] end @@ -178,6 +182,9 @@ defmodule Pleroma.Application do defp idempotency_expiration, do: expiration(default: :timer.seconds(6 * 60 * 60), interval: :timer.seconds(60)) + defp chat_message_id_idempotency_key_expiration, + do: expiration(default: :timer.minutes(2), interval: :timer.seconds(60)) + defp seconds_valid_interval, do: :timer.seconds(Config.get!([Pleroma.Captcha, :seconds_valid])) diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 0fff5faf2..d552e91fc 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -312,6 +312,12 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do {:ok, chat} = Chat.bump_or_create(user.id, other_user.ap_id) {:ok, cm_ref} = MessageReference.create(chat, object, user.ap_id != actor.ap_id) + Cachex.put( + :chat_message_id_idempotency_key_cache, + cm_ref.id, + meta[:idempotency_key] + ) + { ["user", "user:pleroma_chat"], {user, %{cm_ref | chat: chat, object: object}} diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index 60a50b027..318ffc5d0 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -45,7 +45,8 @@ defmodule Pleroma.Web.CommonAPI do {_, {:ok, %Activity{} = activity, _meta}} <- {:common_pipeline, Pipeline.common_pipeline(create_activity_data, - local: true + local: true, + idempotency_key: opts[:idempotency_key] )} do {:ok, activity} else diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex index 6357148d0..2c4d3f135 100644 --- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex @@ -80,7 +80,8 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do %User{} = recipient <- User.get_cached_by_ap_id(chat.recipient), {:ok, activity} <- CommonAPI.post_chat_message(user, recipient, params[:content], - media_id: params[:media_id] + media_id: params[:media_id], + idempotency_key: idempotency_key(conn) ), message <- Object.normalize(activity, false), cm_ref <- MessageReference.for_chat_and_object(chat, message) do @@ -169,4 +170,11 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do |> render("show.json", chat: chat) end end + + defp idempotency_key(conn) do + case get_req_header(conn, "idempotency-key") do + [key] -> key + _ -> nil + end + end end diff --git a/lib/pleroma/web/pleroma_api/views/chat/message_reference_view.ex b/lib/pleroma/web/pleroma_api/views/chat/message_reference_view.ex index d4e08b50d..c058fb340 100644 --- a/lib/pleroma/web/pleroma_api/views/chat/message_reference_view.ex +++ b/lib/pleroma/web/pleroma_api/views/chat/message_reference_view.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Web.PleromaAPI.Chat.MessageReferenceView do use Pleroma.Web, :view + alias Pleroma.Maps alias Pleroma.User alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.MastodonAPI.StatusView @@ -37,6 +38,7 @@ defmodule Pleroma.Web.PleromaAPI.Chat.MessageReferenceView do Pleroma.Web.RichMedia.Helpers.fetch_data_for_object(object) ) } + |> put_idempotency_key() end def render("index.json", opts) do @@ -47,4 +49,13 @@ defmodule Pleroma.Web.PleromaAPI.Chat.MessageReferenceView do Map.put(opts, :as, :chat_message_reference) ) end + + defp put_idempotency_key(data) do + with {:ok, idempotency_key} <- Cachex.get(:chat_message_id_idempotency_key_cache, data.id) do + data + |> Maps.put_if_present(:idempotency_key, idempotency_key) + else + _ -> data + end + end end diff --git a/test/pleroma/web/pleroma_api/controllers/chat_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/chat_controller_test.exs index 6381f9757..fa6b9db65 100644 --- a/test/pleroma/web/pleroma_api/controllers/chat_controller_test.exs +++ b/test/pleroma/web/pleroma_api/controllers/chat_controller_test.exs @@ -82,11 +82,13 @@ defmodule Pleroma.Web.PleromaAPI.ChatControllerTest do result = conn |> put_req_header("content-type", "application/json") + |> put_req_header("idempotency-key", "123") |> post("/api/v1/pleroma/chats/#{chat.id}/messages", %{"content" => "Hallo!!"}) |> json_response_and_validate_schema(200) assert result["content"] == "Hallo!!" assert result["chat_id"] == chat.id |> to_string() + assert result["idempotency_key"] == "123" end test "it fails if there is no content", %{conn: conn, user: user} do diff --git a/test/pleroma/web/pleroma_api/views/chat_message_reference_view_test.exs b/test/pleroma/web/pleroma_api/views/chat_message_reference_view_test.exs index f171a1e55..ae8257870 100644 --- a/test/pleroma/web/pleroma_api/views/chat_message_reference_view_test.exs +++ b/test/pleroma/web/pleroma_api/views/chat_message_reference_view_test.exs @@ -25,7 +25,9 @@ defmodule Pleroma.Web.PleromaAPI.ChatMessageReferenceViewTest do } {:ok, upload} = ActivityPub.upload(file, actor: user.ap_id) - {:ok, activity} = CommonAPI.post_chat_message(user, recipient, "kippis :firefox:") + + {:ok, activity} = + CommonAPI.post_chat_message(user, recipient, "kippis :firefox:", idempotency_key: "123") chat = Chat.get(user.id, recipient.ap_id) @@ -42,6 +44,7 @@ defmodule Pleroma.Web.PleromaAPI.ChatMessageReferenceViewTest do assert chat_message[:created_at] assert chat_message[:unread] == false assert match?([%{shortcode: "firefox"}], chat_message[:emojis]) + assert chat_message[:idempotency_key] == "123" clear_config([:rich_media, :enabled], true) diff --git a/test/pleroma/web/streamer_test.exs b/test/pleroma/web/streamer_test.exs index 185724a9f..395016da2 100644 --- a/test/pleroma/web/streamer_test.exs +++ b/test/pleroma/web/streamer_test.exs @@ -255,7 +255,9 @@ defmodule Pleroma.Web.StreamerTest do } do other_user = insert(:user) - {:ok, create_activity} = CommonAPI.post_chat_message(other_user, user, "hey cirno") + {:ok, create_activity} = + CommonAPI.post_chat_message(other_user, user, "hey cirno", idempotency_key: "123") + object = Object.normalize(create_activity, false) chat = Chat.get(user.id, other_user.ap_id) cm_ref = MessageReference.for_chat_and_object(chat, object) From 8f00d90f9199e384fb1befb677c1c0595a0c854c Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 1 Nov 2020 12:05:39 +0300 Subject: [PATCH 41/45] Use Pleroma.HTTP instead of Tesla Closes #2275 As discovered in the issue, captcha used Tesla.get instead of Pleroma.HTTP. I've also grep'ed the repo and changed the other place where this was used. --- lib/pleroma/captcha/kocaptcha.ex | 2 +- lib/pleroma/emoji/pack.ex | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/captcha/kocaptcha.ex b/lib/pleroma/captcha/kocaptcha.ex index 337506647..201b55ab4 100644 --- a/lib/pleroma/captcha/kocaptcha.ex +++ b/lib/pleroma/captcha/kocaptcha.ex @@ -10,7 +10,7 @@ defmodule Pleroma.Captcha.Kocaptcha do def new do endpoint = Pleroma.Config.get!([__MODULE__, :endpoint]) - case Tesla.get(endpoint <> "/new") do + case Pleroma.HTTP.get(endpoint <> "/new") do {:error, _} -> %{error: :kocaptcha_service_unavailable} diff --git a/lib/pleroma/emoji/pack.ex b/lib/pleroma/emoji/pack.ex index 0670f29f1..ca58e5432 100644 --- a/lib/pleroma/emoji/pack.ex +++ b/lib/pleroma/emoji/pack.ex @@ -594,7 +594,7 @@ defmodule Pleroma.Emoji.Pack do end defp download_archive(url, sha) do - with {:ok, %{body: archive}} <- Tesla.get(url) do + with {:ok, %{body: archive}} <- Pleroma.HTTP.get(url) do if Base.decode16!(sha) == :crypto.hash(:sha256, archive) do {:ok, archive} else @@ -617,7 +617,7 @@ defmodule Pleroma.Emoji.Pack do end defp update_sha_and_save_metadata(pack, data) do - with {:ok, %{body: zip}} <- Tesla.get(data[:"fallback-src"]), + with {:ok, %{body: zip}} <- Pleroma.HTTP.get(data[:"fallback-src"]), :ok <- validate_has_all_files(pack, zip) do fallback_sha = :sha256 |> :crypto.hash(zip) |> Base.encode16() From 4caad4e9101c34debfa90d2e89850d4125a471b3 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Mon, 2 Nov 2020 05:43:06 +0100 Subject: [PATCH 42/45] =?UTF-8?q?side=5Feffects:=20Don=E2=80=99t=20increas?= =?UTF-8?q?e=5Freplies=5Fcount=20when=20it=E2=80=99s=20an=20Answer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/pleroma/web/activity_pub/side_effects.ex | 2 +- .../web/activity_pub/transmogrifier/answer_handling_test.exs | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 0fff5faf2..9b1171d07 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -187,7 +187,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do {:ok, notifications} = Notification.create_notifications(activity, do_send: false) {:ok, _user} = ActivityPub.increase_note_count_if_public(user, object) - if in_reply_to = object.data["inReplyTo"] do + if in_reply_to = object.data["inReplyTo"] && object.data["type"] != "Answer" do Object.increase_replies_count(in_reply_to) end diff --git a/test/pleroma/web/activity_pub/transmogrifier/answer_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/answer_handling_test.exs index 0f6605c3f..e7d85a2c5 100644 --- a/test/pleroma/web/activity_pub/transmogrifier/answer_handling_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier/answer_handling_test.exs @@ -27,6 +27,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.AnswerHandlingTest do }) object = Object.normalize(activity) + assert object.data["repliesCount"] == nil data = File.read!("test/fixtures/mastodon-vote.json") @@ -41,7 +42,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.AnswerHandlingTest do assert answer_object.data["inReplyTo"] == object.data["id"] new_object = Object.get_by_ap_id(object.data["id"]) - assert new_object.data["replies_count"] == object.data["replies_count"] + assert new_object.data["repliesCount"] == nil assert Enum.any?( new_object.data["oneOf"], From 7efc074eadae9b3d6d351e769ead0661f1f4c89c Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 2 Nov 2020 12:19:44 -0600 Subject: [PATCH 43/45] Permit fetching individual reports with notes preloaded --- lib/pleroma/activity.ex | 13 +++++++++++++ .../web/admin_api/controllers/report_controller.ex | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 17af04257..553834da0 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -14,6 +14,7 @@ defmodule Pleroma.Activity do alias Pleroma.ReportNote alias Pleroma.ThreadMute alias Pleroma.User + alias Pleroma.Web.ActivityPub.ActivityPub import Ecto.Changeset import Ecto.Query @@ -153,6 +154,18 @@ defmodule Pleroma.Activity do def get_bookmark(_, _), do: nil + def get_report(activity_id) do + opts = %{ + type: "Flag", + skip_preload: true, + preload_report_notes: true + } + + ActivityPub.fetch_activities_query([], opts) + |> where(id: ^activity_id) + |> Repo.one() + end + def change(struct, params \\ %{}) do struct |> cast(params, [:data, :recipients]) diff --git a/lib/pleroma/web/admin_api/controllers/report_controller.ex b/lib/pleroma/web/admin_api/controllers/report_controller.ex index 86da93893..6a0e56f5f 100644 --- a/lib/pleroma/web/admin_api/controllers/report_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/report_controller.ex @@ -38,7 +38,7 @@ defmodule Pleroma.Web.AdminAPI.ReportController do end def show(conn, %{id: id}) do - with %Activity{} = report <- Activity.get_by_id(id) do + with %Activity{} = report <- Activity.get_report(id) do render(conn, "show.json", Report.extract_report_info(report)) else _ -> {:error, :not_found} From 53dd048590b93da67f9d4abac8cc111424c4d5c0 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 2 Nov 2020 15:49:07 -0600 Subject: [PATCH 44/45] Test the note is returned when fetching a single report --- .../web/admin_api/controllers/report_controller_test.exs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/pleroma/web/admin_api/controllers/report_controller_test.exs b/test/pleroma/web/admin_api/controllers/report_controller_test.exs index fa746d6ea..958e1d3ab 100644 --- a/test/pleroma/web/admin_api/controllers/report_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/report_controller_test.exs @@ -37,12 +37,21 @@ defmodule Pleroma.Web.AdminAPI.ReportControllerTest do status_ids: [activity.id] }) + conn + |> put_req_header("content-type", "application/json") + |> post("/api/pleroma/admin/reports/#{report_id}/notes", %{ + content: "this is an admin note" + }) + response = conn |> get("/api/pleroma/admin/reports/#{report_id}") |> json_response_and_validate_schema(:ok) assert response["id"] == report_id + + [notes] = response["notes"] + assert notes["content"] == "this is an admin note" end test "returns 404 when report id is invalid", %{conn: conn} do From 2f2281fdf1bd3fbd5d82bb437ce3d43ff9043862 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 2 Nov 2020 17:09:56 -0600 Subject: [PATCH 45/45] Ensure URLs for git repos end in .git for older git clients like on CentOS 7 --- mix.exs | 4 ++-- mix.lock | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mix.exs b/mix.exs index e0da696ce..0691902a6 100644 --- a/mix.exs +++ b/mix.exs @@ -134,7 +134,7 @@ defmodule Pleroma.Mixfile do {:cachex, "~> 3.2"}, {:poison, "~> 3.0", override: true}, {:tesla, - git: "https://github.com/teamon/tesla/", + git: "https://github.com/teamon/tesla.git", ref: "9f7261ca49f9f901ceb73b60219ad6f8a9f6aa30", override: true}, {:castore, "~> 0.1"}, @@ -196,7 +196,7 @@ defmodule Pleroma.Mixfile do ref: "e0f16822d578866e186a0974d65ad58cddc1e2ab"}, {:restarter, path: "./restarter"}, {:majic, - git: "https://git.pleroma.social/pleroma/elixir-libraries/majic", branch: "develop"}, + git: "https://git.pleroma.social/pleroma/elixir-libraries/majic.git", branch: "develop"}, {:open_api_spex, git: "https://git.pleroma.social/pleroma/elixir-libraries/open_api_spex.git", ref: "f296ac0924ba3cf79c7a588c4c252889df4c2edd"}, diff --git a/mix.lock b/mix.lock index 07238f550..e5d9bc693 100644 --- a/mix.lock +++ b/mix.lock @@ -66,7 +66,7 @@ "jumper": {:hex, :jumper, "1.0.1", "3c00542ef1a83532b72269fab9f0f0c82bf23a35e27d278bfd9ed0865cecabff", [:mix], [], "hexpm", "318c59078ac220e966d27af3646026db9b5a5e6703cb2aa3e26bcfaba65b7433"}, "libring": {:hex, :libring, "1.4.0", "41246ba2f3fbc76b3971f6bce83119dfec1eee17e977a48d8a9cfaaf58c2a8d6", [:mix], [], "hexpm"}, "linkify": {:hex, :linkify, "0.2.0", "2518bbbea21d2caa9d372424e1ad845b640c6630e2d016f1bd1f518f9ebcca28", [:mix], [], "hexpm", "b8ca8a68b79e30b7938d6c996085f3db14939f29538a59ca5101988bb7f917f6"}, - "majic": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/majic", "4c692e544b28d1f5e543fb8a44be090f8cd96f80", [branch: "develop"]}, + "majic": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/majic.git", "4c692e544b28d1f5e543fb8a44be090f8cd96f80", [branch: "develop"]}, "makeup": {:hex, :makeup, "1.0.3", "e339e2f766d12e7260e6672dd4047405963c5ec99661abdc432e6ec67d29ef95", [:mix], [{:nimble_parsec, "~> 0.5", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "2e9b4996d11832947731f7608fed7ad2f9443011b3b479ae288011265cdd3dad"}, "makeup_elixir": {:hex, :makeup_elixir, "0.14.1", "4f0e96847c63c17841d42c08107405a005a2680eb9c7ccadfd757bd31dabccfb", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "f2438b1a80eaec9ede832b5c41cd4f373b38fd7aa33e3b22d9db79e640cbde11"}, "meck": {:hex, :meck, "0.8.13", "ffedb39f99b0b99703b8601c6f17c7f76313ee12de6b646e671e3188401f7866", [:rebar3], [], "hexpm", "d34f013c156db51ad57cc556891b9720e6a1c1df5fe2e15af999c84d6cebeb1a"}, @@ -115,7 +115,7 @@ "swoosh": {:hex, :swoosh, "1.0.6", "6765e334c67dacabe721f0d701c7e5a6f06e4595c90df6f91e73ebd54d555833", [:mix], [{:cowboy, "~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13 or ~> 1.0", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm", "7c50ef78e4acfd1cbd4907dc1fa87b5540675a6be9dc979d04890f49d7ec1830"}, "syslog": {:hex, :syslog, "1.1.0", "6419a232bea84f07b56dc575225007ffe34d9fdc91abe6f1b2f254fd71d8efc2", [:rebar3], [], "hexpm", "4c6a41373c7e20587be33ef841d3de6f3beba08519809329ecc4d27b15b659e1"}, "telemetry": {:hex, :telemetry, "0.4.2", "2808c992455e08d6177322f14d3bdb6b625fbcfd233a73505870d8738a2f4599", [:rebar3], [], "hexpm", "2d1419bd9dda6a206d7b5852179511722e2b18812310d304620c7bd92a13fcef"}, - "tesla": {:git, "https://github.com/teamon/tesla/", "9f7261ca49f9f901ceb73b60219ad6f8a9f6aa30", [ref: "9f7261ca49f9f901ceb73b60219ad6f8a9f6aa30"]}, + "tesla": {:git, "https://github.com/teamon/tesla.git", "9f7261ca49f9f901ceb73b60219ad6f8a9f6aa30", [ref: "9f7261ca49f9f901ceb73b60219ad6f8a9f6aa30"]}, "timex": {:hex, :timex, "3.6.2", "845cdeb6119e2fef10751c0b247b6c59d86d78554c83f78db612e3290f819bc2", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "26030b46199d02a590be61c2394b37ea25a3664c02fafbeca0b24c972025d47a"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "bd4fde4c15f3e993a999e019d64347489b91b7a9096af68b2bdadd192afa693f"}, "tzdata": {:hex, :tzdata, "1.0.4", "a3baa4709ea8dba552dca165af6ae97c624a2d6ac14bd265165eaa8e8af94af6", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "b02637db3df1fd66dd2d3c4f194a81633d0e4b44308d36c1b2fdfd1e4e6f169b"},