Add backups table

This commit is contained in:
Egor Kislitsyn 2020-09-02 20:21:33 +04:00
parent 75e07ba206
commit 4f3a633745
No known key found for this signature in database
GPG key ID: 1B49CB15B71E7805
3 changed files with 141 additions and 34 deletions

View file

@ -2,41 +2,110 @@
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Export do defmodule Pleroma.Backup do
use Ecto.Schema
import Ecto.Changeset
import Ecto.Query
alias Pleroma.Activity alias Pleroma.Activity
alias Pleroma.Bookmark alias Pleroma.Bookmark
alias Pleroma.Repo
alias Pleroma.User alias Pleroma.User
alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.ActivityPub
alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.ActivityPub.Transmogrifier
alias Pleroma.Web.ActivityPub.UserView alias Pleroma.Web.ActivityPub.UserView
import Ecto.Query schema "backups" do
field(:content_type, :string)
field(:file_name, :string)
field(:file_size, :integer, default: 0)
field(:processed, :boolean, default: false)
belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
timestamps()
end
def create(user) do
with :ok <- validate_limit(user),
{:ok, backup} <- user |> new() |> Repo.insert() do
{:ok, backup}
end
end
def new(user) do
rand_str = :crypto.strong_rand_bytes(32) |> Base.url_encode64(padding: false)
datetime = Calendar.NaiveDateTime.Format.iso8601_basic(NaiveDateTime.utc_now())
name = "archive-#{user.nickname}-#{datetime}-#{rand_str}.zip"
%__MODULE__{
user_id: user.id,
content_type: "application/zip",
file_name: name
}
end
defp validate_limit(user) do
case get_last(user.id) do
%__MODULE__{inserted_at: inserted_at} ->
days = 7
diff = Timex.diff(NaiveDateTime.utc_now(), inserted_at, :days)
if diff > days do
:ok
else
{:error, "Last export was less than #{days} days ago"}
end
nil ->
:ok
end
end
def get_last(user_id) do
__MODULE__
|> where(user_id: ^user_id)
|> order_by(desc: :id)
|> limit(1)
|> Repo.one()
end
def process(%__MODULE__{} = backup) do
with {:ok, zip_file} <- zip(backup),
{:ok, %{size: size}} <- File.stat(zip_file),
{:ok, _upload} <- upload(backup, zip_file) do
backup
|> cast(%{file_size: size, processed: true}, [:file_size, :processed])
|> Repo.update()
end
end
@files ['actor.json', 'outbox.json', 'likes.json', 'bookmarks.json'] @files ['actor.json', 'outbox.json', 'likes.json', 'bookmarks.json']
def zip(%__MODULE__{} = backup) do
backup = Repo.preload(backup, :user)
name = String.trim_trailing(backup.file_name, ".zip")
dir = Path.join(System.tmp_dir!(), name)
def run(user) do with :ok <- File.mkdir(dir),
with {:ok, path} <- create_dir(user), :ok <- actor(dir, backup.user),
:ok <- actor(path, user), :ok <- statuses(dir, backup.user),
:ok <- statuses(path, user), :ok <- likes(dir, backup.user),
:ok <- likes(path, user), :ok <- bookmarks(dir, backup.user),
:ok <- bookmarks(path, user), {:ok, zip_path} <- :zip.create(String.to_charlist(dir <> ".zip"), @files, cwd: dir),
{:ok, zip_path} <- :zip.create('#{path}.zip', @files, cwd: path), {:ok, _} <- File.rm_rf(dir) do
{:ok, _} <- File.rm_rf(path) do
{:ok, :binary.list_to_bin(zip_path)} {:ok, :binary.list_to_bin(zip_path)}
end end
end end
def upload(zip_path) do def upload(%__MODULE__{} = backup, zip_path) do
uploader = Pleroma.Config.get([Pleroma.Upload, :uploader]) uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
file_name = zip_path |> String.split("/") |> List.last()
id = Ecto.UUID.generate()
upload = %Pleroma.Upload{ upload = %Pleroma.Upload{
id: id, name: backup.file_name,
name: file_name,
tempfile: zip_path, tempfile: zip_path,
content_type: "application/zip", content_type: backup.content_type,
path: id <> "/" <> file_name path: "backups/" <> backup.file_name
} }
with {:ok, _} <- Pleroma.Uploaders.Uploader.put_file(uploader, upload), with {:ok, _} <- Pleroma.Uploaders.Uploader.put_file(uploader, upload),
@ -54,13 +123,6 @@ defp actor(dir, user) do
end end
end end
defp create_dir(user) do
datetime = Calendar.NaiveDateTime.Format.iso8601_basic(NaiveDateTime.utc_now())
dir = Path.join(System.tmp_dir!(), "archive-#{user.id}-#{datetime}")
with :ok <- File.mkdir(dir), do: {:ok, dir}
end
defp write_header(file, name) do defp write_header(file, name) do
IO.write( IO.write(
file, file,

View file

@ -0,0 +1,17 @@
defmodule Pleroma.Repo.Migrations.CreateBackups do
use Ecto.Migration
def change do
create_if_not_exists table(:backups) do
add(:user_id, references(:users, type: :uuid, on_delete: :delete_all))
add(:file_name, :string, null: false)
add(:content_type, :string, null: false)
add(:processed, :boolean, null: false, default: false)
add(:file_size, :bigint)
timestamps()
end
create_if_not_exists(index(:backups, [:user_id]))
end
end

View file

@ -2,15 +2,41 @@
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.ExportTest do defmodule Pleroma.BackupTest do
use Pleroma.DataCase use Pleroma.DataCase
import Pleroma.Factory import Pleroma.Factory
import Mock import Mock
alias Pleroma.Backup
alias Pleroma.Bookmark alias Pleroma.Bookmark
alias Pleroma.Web.CommonAPI alias Pleroma.Web.CommonAPI
test "it exports user data" do test "it creates a backup record" do
%{id: user_id} = user = insert(:user)
assert {:ok, backup} = Backup.create(user)
assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup
end
test "it return an error if the export limit is over" do
%{id: user_id} = user = insert(:user)
limit_days = 7
assert {:ok, backup} = Backup.create(user)
assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup
assert Backup.create(user) == {:error, "Last export was less than #{limit_days} days ago"}
end
test "it process a backup record" do
%{id: user_id} = user = insert(:user)
assert {:ok, %{id: backup_id} = backup} = Backup.create(user)
assert {:ok, %Backup{} = backup} = Backup.process(backup)
assert backup.file_size > 0
assert %Backup{id: ^backup_id, processed: true, user_id: ^user_id} = backup
end
test "it creates a zip archive with user data" do
user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"}) user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"})
{:ok, %{object: %{data: %{"id" => id1}}} = status1} = {:ok, %{object: %{data: %{"id" => id1}}} = status1} =
@ -28,7 +54,8 @@ test "it exports user data" do
Bookmark.create(user.id, status2.id) Bookmark.create(user.id, status2.id)
Bookmark.create(user.id, status3.id) Bookmark.create(user.id, status3.id)
assert {:ok, path} = Pleroma.Export.run(user) assert {:ok, backup} = user |> Backup.new() |> Repo.insert()
assert {:ok, path} = Backup.zip(backup)
assert {:ok, zipfile} = :zip.zip_open(String.to_charlist(path), [:memory]) assert {:ok, zipfile} = :zip.zip_open(String.to_charlist(path), [:memory])
assert {:ok, {'actor.json', json}} = :zip.zip_get('actor.json', zipfile) assert {:ok, {'actor.json', json}} = :zip.zip_get('actor.json', zipfile)
@ -110,7 +137,7 @@ test "it exports user data" do
File.rm!(path) File.rm!(path)
end end
describe "it uploads an exported backup archive" do describe "it uploads a backup archive" do
setup do setup do
clear_config(Pleroma.Uploaders.S3, clear_config(Pleroma.Uploaders.S3,
bucket: "test_bucket", bucket: "test_bucket",
@ -129,23 +156,24 @@ test "it exports user data" do
Bookmark.create(user.id, status2.id) Bookmark.create(user.id, status2.id)
Bookmark.create(user.id, status3.id) Bookmark.create(user.id, status3.id)
assert {:ok, path} = Pleroma.Export.run(user) assert {:ok, backup} = user |> Backup.new() |> Repo.insert()
assert {:ok, path} = Backup.zip(backup)
[path: path] [path: path, backup: backup]
end end
test "S3", %{path: path} do test "S3", %{path: path, backup: backup} do
Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.S3) Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.S3)
with_mock ExAws, request: fn _ -> {:ok, :ok} end do with_mock ExAws, request: fn _ -> {:ok, :ok} end do
assert {:ok, %Pleroma.Upload{}} = Pleroma.Export.upload(path) assert {:ok, %Pleroma.Upload{}} = Backup.upload(backup, path)
end end
end end
test "Local", %{path: path} do test "Local", %{path: path, backup: backup} do
Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
assert {:ok, %Pleroma.Upload{}} = Pleroma.Export.upload(path) assert {:ok, %Pleroma.Upload{}} = Backup.upload(backup, path)
end end
end end
end end