Add backups table

This commit is contained in:
Egor Kislitsyn 2020-09-02 20:21:33 +04:00
parent 75e07ba206
commit 4f3a633745
No known key found for this signature in database
GPG key ID: 1B49CB15B71E7805
3 changed files with 141 additions and 34 deletions

View file

@ -2,41 +2,110 @@
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Export do
defmodule Pleroma.Backup do
use Ecto.Schema
import Ecto.Changeset
import Ecto.Query
alias Pleroma.Activity
alias Pleroma.Bookmark
alias Pleroma.Repo
alias Pleroma.User
alias Pleroma.Web.ActivityPub.ActivityPub
alias Pleroma.Web.ActivityPub.Transmogrifier
alias Pleroma.Web.ActivityPub.UserView
import Ecto.Query
schema "backups" do
field(:content_type, :string)
field(:file_name, :string)
field(:file_size, :integer, default: 0)
field(:processed, :boolean, default: false)
belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
timestamps()
end
def create(user) do
with :ok <- validate_limit(user),
{:ok, backup} <- user |> new() |> Repo.insert() do
{:ok, backup}
end
end
def new(user) do
rand_str = :crypto.strong_rand_bytes(32) |> Base.url_encode64(padding: false)
datetime = Calendar.NaiveDateTime.Format.iso8601_basic(NaiveDateTime.utc_now())
name = "archive-#{user.nickname}-#{datetime}-#{rand_str}.zip"
%__MODULE__{
user_id: user.id,
content_type: "application/zip",
file_name: name
}
end
defp validate_limit(user) do
case get_last(user.id) do
%__MODULE__{inserted_at: inserted_at} ->
days = 7
diff = Timex.diff(NaiveDateTime.utc_now(), inserted_at, :days)
if diff > days do
:ok
else
{:error, "Last export was less than #{days} days ago"}
end
nil ->
:ok
end
end
def get_last(user_id) do
__MODULE__
|> where(user_id: ^user_id)
|> order_by(desc: :id)
|> limit(1)
|> Repo.one()
end
def process(%__MODULE__{} = backup) do
with {:ok, zip_file} <- zip(backup),
{:ok, %{size: size}} <- File.stat(zip_file),
{:ok, _upload} <- upload(backup, zip_file) do
backup
|> cast(%{file_size: size, processed: true}, [:file_size, :processed])
|> Repo.update()
end
end
@files ['actor.json', 'outbox.json', 'likes.json', 'bookmarks.json']
def zip(%__MODULE__{} = backup) do
backup = Repo.preload(backup, :user)
name = String.trim_trailing(backup.file_name, ".zip")
dir = Path.join(System.tmp_dir!(), name)
def run(user) do
with {:ok, path} <- create_dir(user),
:ok <- actor(path, user),
:ok <- statuses(path, user),
:ok <- likes(path, user),
:ok <- bookmarks(path, user),
{:ok, zip_path} <- :zip.create('#{path}.zip', @files, cwd: path),
{:ok, _} <- File.rm_rf(path) do
with :ok <- File.mkdir(dir),
:ok <- actor(dir, backup.user),
:ok <- statuses(dir, backup.user),
:ok <- likes(dir, backup.user),
:ok <- bookmarks(dir, backup.user),
{:ok, zip_path} <- :zip.create(String.to_charlist(dir <> ".zip"), @files, cwd: dir),
{:ok, _} <- File.rm_rf(dir) do
{:ok, :binary.list_to_bin(zip_path)}
end
end
def upload(zip_path) do
def upload(%__MODULE__{} = backup, zip_path) do
uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
file_name = zip_path |> String.split("/") |> List.last()
id = Ecto.UUID.generate()
upload = %Pleroma.Upload{
id: id,
name: file_name,
name: backup.file_name,
tempfile: zip_path,
content_type: "application/zip",
path: id <> "/" <> file_name
content_type: backup.content_type,
path: "backups/" <> backup.file_name
}
with {:ok, _} <- Pleroma.Uploaders.Uploader.put_file(uploader, upload),
@ -54,13 +123,6 @@ defmodule Pleroma.Export do
end
end
defp create_dir(user) do
datetime = Calendar.NaiveDateTime.Format.iso8601_basic(NaiveDateTime.utc_now())
dir = Path.join(System.tmp_dir!(), "archive-#{user.id}-#{datetime}")
with :ok <- File.mkdir(dir), do: {:ok, dir}
end
defp write_header(file, name) do
IO.write(
file,

View file

@ -0,0 +1,17 @@
defmodule Pleroma.Repo.Migrations.CreateBackups do
use Ecto.Migration
def change do
create_if_not_exists table(:backups) do
add(:user_id, references(:users, type: :uuid, on_delete: :delete_all))
add(:file_name, :string, null: false)
add(:content_type, :string, null: false)
add(:processed, :boolean, null: false, default: false)
add(:file_size, :bigint)
timestamps()
end
create_if_not_exists(index(:backups, [:user_id]))
end
end

View file

@ -2,15 +2,41 @@
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.ExportTest do
defmodule Pleroma.BackupTest do
use Pleroma.DataCase
import Pleroma.Factory
import Mock
alias Pleroma.Backup
alias Pleroma.Bookmark
alias Pleroma.Web.CommonAPI
test "it exports user data" do
test "it creates a backup record" do
%{id: user_id} = user = insert(:user)
assert {:ok, backup} = Backup.create(user)
assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup
end
test "it return an error if the export limit is over" do
%{id: user_id} = user = insert(:user)
limit_days = 7
assert {:ok, backup} = Backup.create(user)
assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup
assert Backup.create(user) == {:error, "Last export was less than #{limit_days} days ago"}
end
test "it process a backup record" do
%{id: user_id} = user = insert(:user)
assert {:ok, %{id: backup_id} = backup} = Backup.create(user)
assert {:ok, %Backup{} = backup} = Backup.process(backup)
assert backup.file_size > 0
assert %Backup{id: ^backup_id, processed: true, user_id: ^user_id} = backup
end
test "it creates a zip archive with user data" do
user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"})
{:ok, %{object: %{data: %{"id" => id1}}} = status1} =
@ -28,7 +54,8 @@ defmodule Pleroma.ExportTest do
Bookmark.create(user.id, status2.id)
Bookmark.create(user.id, status3.id)
assert {:ok, path} = Pleroma.Export.run(user)
assert {:ok, backup} = user |> Backup.new() |> Repo.insert()
assert {:ok, path} = Backup.zip(backup)
assert {:ok, zipfile} = :zip.zip_open(String.to_charlist(path), [:memory])
assert {:ok, {'actor.json', json}} = :zip.zip_get('actor.json', zipfile)
@ -110,7 +137,7 @@ defmodule Pleroma.ExportTest do
File.rm!(path)
end
describe "it uploads an exported backup archive" do
describe "it uploads a backup archive" do
setup do
clear_config(Pleroma.Uploaders.S3,
bucket: "test_bucket",
@ -129,23 +156,24 @@ defmodule Pleroma.ExportTest do
Bookmark.create(user.id, status2.id)
Bookmark.create(user.id, status3.id)
assert {:ok, path} = Pleroma.Export.run(user)
assert {:ok, backup} = user |> Backup.new() |> Repo.insert()
assert {:ok, path} = Backup.zip(backup)
[path: path]
[path: path, backup: backup]
end
test "S3", %{path: path} do
test "S3", %{path: path, backup: backup} do
Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.S3)
with_mock ExAws, request: fn _ -> {:ok, :ok} end do
assert {:ok, %Pleroma.Upload{}} = Pleroma.Export.upload(path)
assert {:ok, %Pleroma.Upload{}} = Backup.upload(backup, path)
end
end
test "Local", %{path: path} do
test "Local", %{path: path, backup: backup} do
Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
assert {:ok, %Pleroma.Upload{}} = Pleroma.Export.upload(path)
assert {:ok, %Pleroma.Upload{}} = Backup.upload(backup, path)
end
end
end