Add backups table

lib/pleroma/backup.ex

@@ -2,41 +2,110 @@
 # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.Export do
+defmodule Pleroma.Backup do
+  use Ecto.Schema
+
+  import Ecto.Changeset
+  import Ecto.Query
+
   alias Pleroma.Activity
   alias Pleroma.Bookmark
+  alias Pleroma.Repo
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Transmogrifier
   alias Pleroma.Web.ActivityPub.UserView
 
-  import Ecto.Query
+  schema "backups" do
+    field(:content_type, :string)
+    field(:file_name, :string)
+    field(:file_size, :integer, default: 0)
+    field(:processed, :boolean, default: false)
+
+    belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
+
+    timestamps()
+  end
+
+  def create(user) do
+    with :ok <- validate_limit(user),
+         {:ok, backup} <- user |> new() |> Repo.insert() do
+      {:ok, backup}
+    end
+  end
+
+  def new(user) do
+    rand_str = :crypto.strong_rand_bytes(32) |> Base.url_encode64(padding: false)
+    datetime = Calendar.NaiveDateTime.Format.iso8601_basic(NaiveDateTime.utc_now())
+    name = "archive-#{user.nickname}-#{datetime}-#{rand_str}.zip"
+
+    %__MODULE__{
+      user_id: user.id,
+      content_type: "application/zip",
+      file_name: name
+    }
+  end
+
+  defp validate_limit(user) do
+    case get_last(user.id) do
+      %__MODULE__{inserted_at: inserted_at} ->
+        days = 7
+        diff = Timex.diff(NaiveDateTime.utc_now(), inserted_at, :days)
+
+        if diff > days do
+          :ok
+        else
+          {:error, "Last export was less than #{days} days ago"}
+        end
+
+      nil ->
+        :ok
+    end
+  end
+
+  def get_last(user_id) do
+    __MODULE__
+    |> where(user_id: ^user_id)
+    |> order_by(desc: :id)
+    |> limit(1)
+    |> Repo.one()
+  end
+
+  def process(%__MODULE__{} = backup) do
+    with {:ok, zip_file} <- zip(backup),
+         {:ok, %{size: size}} <- File.stat(zip_file),
+         {:ok, _upload} <- upload(backup, zip_file) do
+      backup
+      |> cast(%{file_size: size, processed: true}, [:file_size, :processed])
+      |> Repo.update()
+    end
+  end
 
   @files ['actor.json', 'outbox.json', 'likes.json', 'bookmarks.json']
+  def zip(%__MODULE__{} = backup) do
+    backup = Repo.preload(backup, :user)
+    name = String.trim_trailing(backup.file_name, ".zip")
+    dir = Path.join(System.tmp_dir!(), name)
 
-  def run(user) do
+    with :ok <- File.mkdir(dir),
-    with {:ok, path} <- create_dir(user),
+         :ok <- actor(dir, backup.user),
-         :ok <- actor(path, user),
+         :ok <- statuses(dir, backup.user),
-         :ok <- statuses(path, user),
+         :ok <- likes(dir, backup.user),
-         :ok <- likes(path, user),
+         :ok <- bookmarks(dir, backup.user),
-         :ok <- bookmarks(path, user),
+         {:ok, zip_path} <- :zip.create(String.to_charlist(dir <> ".zip"), @files, cwd: dir),
-         {:ok, zip_path} <- :zip.create('#{path}.zip', @files, cwd: path),
+         {:ok, _} <- File.rm_rf(dir) do
-         {:ok, _} <- File.rm_rf(path) do
       {:ok, :binary.list_to_bin(zip_path)}
     end
   end
 
-  def upload(zip_path) do
+  def upload(%__MODULE__{} = backup, zip_path) do
     uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
-    file_name = zip_path |> String.split("/") |> List.last()
-    id = Ecto.UUID.generate()
 
     upload = %Pleroma.Upload{
-      id: id,
+      name: backup.file_name,
-      name: file_name,
       tempfile: zip_path,
-      content_type: "application/zip",
+      content_type: backup.content_type,
-      path: id <> "/" <> file_name
+      path: "backups/" <> backup.file_name
     }
 
     with {:ok, _} <- Pleroma.Uploaders.Uploader.put_file(uploader, upload),
@@ -54,13 +123,6 @@ defp actor(dir, user) do
     end
   end
 
-  defp create_dir(user) do
-    datetime = Calendar.NaiveDateTime.Format.iso8601_basic(NaiveDateTime.utc_now())
-    dir = Path.join(System.tmp_dir!(), "archive-#{user.id}-#{datetime}")
-
-    with :ok <- File.mkdir(dir), do: {:ok, dir}
-  end
-
   defp write_header(file, name) do
     IO.write(
       file,

priv/repo/migrations/20200831192323_create_backups.exs

@@ -0,0 +1,17 @@
+defmodule Pleroma.Repo.Migrations.CreateBackups do
+  use Ecto.Migration
+
+  def change do
+    create_if_not_exists table(:backups) do
+      add(:user_id, references(:users, type: :uuid, on_delete: :delete_all))
+      add(:file_name, :string, null: false)
+      add(:content_type, :string, null: false)
+      add(:processed, :boolean, null: false, default: false)
+      add(:file_size, :bigint)
+
+      timestamps()
+    end
+
+    create_if_not_exists(index(:backups, [:user_id]))
+  end
+end

test/backup_test.exs

@@ -2,15 +2,41 @@
 # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.ExportTest do
+defmodule Pleroma.BackupTest do
   use Pleroma.DataCase
   import Pleroma.Factory
   import Mock
 
+  alias Pleroma.Backup
   alias Pleroma.Bookmark
   alias Pleroma.Web.CommonAPI
 
-  test "it exports user data" do
+  test "it creates a backup record" do
+    %{id: user_id} = user = insert(:user)
+    assert {:ok, backup} = Backup.create(user)
+
+    assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup
+  end
+
+  test "it return an error if the export limit is over" do
+    %{id: user_id} = user = insert(:user)
+    limit_days = 7
+
+    assert {:ok, backup} = Backup.create(user)
+    assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup
+
+    assert Backup.create(user) == {:error, "Last export was less than #{limit_days} days ago"}
+  end
+
+  test "it process a backup record" do
+    %{id: user_id} = user = insert(:user)
+    assert {:ok, %{id: backup_id} = backup} = Backup.create(user)
+    assert {:ok, %Backup{} = backup} = Backup.process(backup)
+    assert backup.file_size > 0
+    assert %Backup{id: ^backup_id, processed: true, user_id: ^user_id} = backup
+  end
+
+  test "it creates a zip archive with user data" do
     user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"})
 
     {:ok, %{object: %{data: %{"id" => id1}}} = status1} =
@@ -28,7 +54,8 @@ test "it exports user data" do
     Bookmark.create(user.id, status2.id)
     Bookmark.create(user.id, status3.id)
 
-    assert {:ok, path} = Pleroma.Export.run(user)
+    assert {:ok, backup} = user |> Backup.new() |> Repo.insert()
+    assert {:ok, path} = Backup.zip(backup)
     assert {:ok, zipfile} = :zip.zip_open(String.to_charlist(path), [:memory])
     assert {:ok, {'actor.json', json}} = :zip.zip_get('actor.json', zipfile)
 
@@ -110,7 +137,7 @@ test "it exports user data" do
     File.rm!(path)
   end
 
-  describe "it uploads an exported backup archive" do
+  describe "it uploads a backup archive" do
     setup do
       clear_config(Pleroma.Uploaders.S3,
         bucket: "test_bucket",
@@ -129,23 +156,24 @@ test "it exports user data" do
       Bookmark.create(user.id, status2.id)
       Bookmark.create(user.id, status3.id)
 
-      assert {:ok, path} = Pleroma.Export.run(user)
+      assert {:ok, backup} = user |> Backup.new() |> Repo.insert()
+      assert {:ok, path} = Backup.zip(backup)
 
-      [path: path]
+      [path: path, backup: backup]
     end
 
-    test "S3", %{path: path} do
+    test "S3", %{path: path, backup: backup} do
       Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.S3)
 
       with_mock ExAws, request: fn _ -> {:ok, :ok} end do
-        assert {:ok, %Pleroma.Upload{}} = Pleroma.Export.upload(path)
+        assert {:ok, %Pleroma.Upload{}} = Backup.upload(backup, path)
       end
     end
 
-    test "Local", %{path: path} do
+    test "Local", %{path: path, backup: backup} do
       Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
 
-      assert {:ok, %Pleroma.Upload{}} = Pleroma.Export.upload(path)
+      assert {:ok, %Pleroma.Upload{}} = Backup.upload(backup, path)
     end
   end
 end