Merge branch 'develop' into gun

CHANGELOG.md

@@ -39,6 +39,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Logger: default log level changed from `warn` to `info`.
 - Config mix task `migrate_to_db` truncates `config` table before migrating the config file.
 - Default to `prepare: :unnamed` in the database configuration.
+- Instance stats are now loaded on startup instead of being empty until next hourly job.
 <details>
   <summary>API Changes</summary>
 

docs/API/admin_api.md

@@ -278,6 +278,19 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
   - On failure: `Not found`
   - On success: JSON array of instance's latest statuses
 
+## `GET /api/pleroma/admin/statuses`
+
+### Retrives all latest statuses
+
+- Params:
+  - *optional* `page_size`: number of statuses to return (default is `20`)
+  - *optional* `local_only`: excludes remote statuses
+  - *optional* `godmode`: `true`/`false` – allows to see private statuses
+  - *optional* `with_reblogs`: `true`/`false` – allows to see reblogs (default is false)
+- Response:
+  - On failure: `Not found`
+  - On success: JSON array of user's latest statuses
+
 ## `POST /api/pleroma/admin/relay`
 
 ### Follow a Relay

lib/mix/pleroma.ex

@@ -12,6 +12,19 @@ def start_pleroma do
     end
 
     {:ok, _} = Application.ensure_all_started(:pleroma)
+
+    if Pleroma.Config.get(:env) not in [:test, :benchmark] do
+      pleroma_rebooted?()
+    end
+  end
+
+  defp pleroma_rebooted? do
+    if Restarter.Pleroma.rebooted?() do
+      :ok
+    else
+      Process.sleep(10)
+      pleroma_rebooted?()
+    end
   end
 
   def load_pleroma do

lib/pleroma/config/transfer_task.ex

@@ -45,7 +45,8 @@ def start_link(_) do
 
   @spec load_and_update_env([ConfigDB.t()]) :: :ok | false
   def load_and_update_env(deleted \\ [], restart_pleroma? \\ true) do
-    with true <- Pleroma.Config.get(:configurable_from_database),
+    with {:configurable, true} <-
+           {:configurable, Pleroma.Config.get(:configurable_from_database)},
          true <- Ecto.Adapters.SQL.table_exists?(Repo, "config"),
          started_applications <- Application.started_applications() do
       # We need to restart applications for loaded settings take effect
@@ -68,12 +69,15 @@ def load_and_update_env(deleted \\ [], restart_pleroma? \\ true) do
         if :pleroma in applications do
           List.delete(applications, :pleroma) ++ [:pleroma]
         else
+          Restarter.Pleroma.rebooted()
           applications
         end
 
       Enum.each(applications, &restart(started_applications, &1, Pleroma.Config.get(:env)))
 
       :ok
+    else
+      {:configurable, false} -> Restarter.Pleroma.rebooted()
     end
   end
 

lib/pleroma/stats.ex

@@ -10,19 +10,10 @@ defmodule Pleroma.Stats do
 
   use GenServer
 
-  @init_state %{
-    peers: [],
-    stats: %{
-      domain_count: 0,
-      status_count: 0,
-      user_count: 0
-    }
-  }
-
   def start_link(_) do
     GenServer.start_link(
       __MODULE__,
-      @init_state,
+      nil,
       name: __MODULE__
     )
   end
@@ -53,8 +44,8 @@ def get_peers do
     peers
   end
 
-  def init(args) do
+  def init(_args) do
-    {:ok, args}
+    {:ok, get_stat_data()}
   end
 
   def handle_call(:force_update, _from, _state) do

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -660,7 +660,9 @@ def flag(
          {:ok, activity} <- insert(flag_data, local),
          {:ok, stripped_activity} <- strip_report_status_data(activity),
          :ok <- maybe_federate(stripped_activity) do
-      Enum.each(User.all_superusers(), fn superuser ->
+      User.all_superusers()
+      |> Enum.filter(fn user -> not is_nil(user.email) end)
+      |> Enum.each(fn superuser ->
         superuser
         |> Pleroma.Emails.AdminEmail.report(actor, account, statuses, content)
         |> Pleroma.Emails.Mailer.deliver_async()

lib/pleroma/web/admin_api/admin_api_controller.ex

@@ -748,6 +748,7 @@ def report_notes_delete(%{assigns: %{user: user}} = conn, %{
   def list_statuses(%{assigns: %{user: admin}} = conn, params) do
     godmode = params["godmode"] == "true" || params["godmode"] == true
     local_only = params["local_only"] == "true" || params["local_only"] == true
+    with_reblogs = params["with_reblogs"] == "true" || params["with_reblogs"] == true
     {page, page_size} = page_params(params)
 
     activities =
@@ -755,7 +756,8 @@ def list_statuses(%{assigns: %{user: admin}} = conn, params) do
         "godmode" => godmode,
         "local_only" => local_only,
         "limit" => page_size,
-        "offset" => (page - 1) * page_size
+        "offset" => (page - 1) * page_size,
+        "exclude_reblogs" => !with_reblogs && "true"
       })
 
     conn

lib/pleroma/web/common_api/common_api.ex

@@ -70,20 +70,21 @@ def reject_follow_request(follower, followed) do
   end
 
   def delete(activity_id, user) do
-    with %Activity{data: %{"object" => _}} = activity <-
+    with {_, %Activity{data: %{"object" => _}} = activity} <-
-           Activity.get_by_id_with_object(activity_id),
+           {:find_activity, Activity.get_by_id_with_object(activity_id)},
          %Object{} = object <- Object.normalize(activity),
          true <- User.superuser?(user) || user.ap_id == object.data["actor"],
          {:ok, _} <- unpin(activity_id, user),
          {:ok, delete} <- ActivityPub.delete(object) do
       {:ok, delete}
     else
+      {:find_activity, _} -> {:error, :not_found}
       _ -> {:error, dgettext("errors", "Could not delete")}
     end
   end
 
   def repeat(id_or_ap_id, user, params \\ %{}) do
-    with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id),
+    with {_, %Activity{} = activity} <- {:find_activity, get_by_id_or_ap_id(id_or_ap_id)},
          object <- Object.normalize(activity),
          announce_activity <- Utils.get_existing_announce(user.ap_id, object),
          public <- public_announce?(object, params) do
@@ -93,21 +94,23 @@ def repeat(id_or_ap_id, user, params \\ %{}) do
         ActivityPub.announce(user, object, nil, true, public)
       end
     else
+      {:find_activity, _} -> {:error, :not_found}
       _ -> {:error, dgettext("errors", "Could not repeat")}
     end
   end
 
   def unrepeat(id_or_ap_id, user) do
-    with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id) do
+    with {_, %Activity{} = activity} <- {:find_activity, get_by_id_or_ap_id(id_or_ap_id)} do
       object = Object.normalize(activity)
       ActivityPub.unannounce(user, object)
     else
+      {:find_activity, _} -> {:error, :not_found}
       _ -> {:error, dgettext("errors", "Could not unrepeat")}
     end
   end
 
   def favorite(id_or_ap_id, user) do
-    with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id),
+    with {_, %Activity{} = activity} <- {:find_activity, get_by_id_or_ap_id(id_or_ap_id)},
          object <- Object.normalize(activity),
          like_activity <- Utils.get_existing_like(user.ap_id, object) do
       if like_activity do
@@ -116,15 +119,17 @@ def favorite(id_or_ap_id, user) do
         ActivityPub.like(user, object)
       end
     else
+      {:find_activity, _} -> {:error, :not_found}
       _ -> {:error, dgettext("errors", "Could not favorite")}
     end
   end
 
   def unfavorite(id_or_ap_id, user) do
-    with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id) do
+    with {_, %Activity{} = activity} <- {:find_activity, get_by_id_or_ap_id(id_or_ap_id)} do
       object = Object.normalize(activity)
       ActivityPub.unlike(user, object)
     else
+      {:find_activity, _} -> {:error, :not_found}
       _ -> {:error, dgettext("errors", "Could not unfavorite")}
     end
   end

lib/pleroma/web/mastodon_api/controllers/status_controller.ex

@@ -175,6 +175,8 @@ def show(%{assigns: %{user: user}} = conn, %{"id" => id}) do
         for: user,
         with_direct_conversation_id: true
       )
+    else
+      _ -> {:error, :not_found}
     end
   end
 
@@ -183,6 +185,7 @@ def delete(%{assigns: %{user: user}} = conn, %{"id" => id}) do
     with {:ok, %Activity{}} <- CommonAPI.delete(id, user) do
       json(conn, %{})
     else
+      {:error, :not_found} = e -> e
       _e -> render_error(conn, :forbidden, "Can't delete this post")
     end
   end

lib/pleroma/web/twitter_api/twitter_api.ex

@@ -99,7 +99,8 @@ defp create_user(params, opts) do
 
   def password_reset(nickname_or_email) do
     with true <- is_binary(nickname_or_email),
-         %User{local: true} = user <- User.get_by_nickname_or_email(nickname_or_email),
+         %User{local: true, email: email} = user when not is_nil(email) <-
+           User.get_by_nickname_or_email(nickname_or_email),
          {:ok, token_record} <- Pleroma.PasswordResetToken.create_token(user) do
       user
       |> UserEmail.password_reset_email(token_record.token)
@@ -110,6 +111,9 @@ def password_reset(nickname_or_email) do
       false ->
         {:error, "bad user identifier"}
 
+      %User{local: true, email: nil} ->
+        {:ok, :noop}
+
       %User{local: false} ->
         {:error, "remote user"}
 

lib/pleroma/workers/cron/digest_emails_worker.ex

@@ -31,6 +31,7 @@ def perform(_opts, _job) do
 
       from(u in inactive_users_query,
         where: fragment(~s(? ->'digest' @> 'true'), u.email_notifications),
+        where: not is_nil(u.email),
         where: u.last_digest_emailed_at < datetime_add(^now, ^negative_interval, "day"),
         select: u
       )

lib/pleroma/workers/cron/new_users_digest_worker.ex

@@ -51,6 +51,7 @@ def perform(_args, _job) do
       if users_and_statuses != [] do
         %{is_admin: true}
         |> User.Query.build()
+        |> where([u], not is_nil(u.email))
         |> Repo.all()
         |> Enum.map(&Pleroma.Emails.NewUsersDigestEmail.new_users(&1, users_and_statuses))
         |> Enum.each(&Pleroma.Emails.Mailer.deliver/1)

restarter/lib/pleroma.ex

@@ -3,11 +3,21 @@ defmodule Restarter.Pleroma do
 
   require Logger
 
+  @init_state %{need_reboot: false, rebooted: false, after_boot: false}
+
   def start_link(_) do
     GenServer.start_link(__MODULE__, [], name: __MODULE__)
   end
 
-  def init(_), do: {:ok, %{need_reboot?: false}}
+  def init(_), do: {:ok, @init_state}
+
+  def rebooted? do
+    GenServer.call(__MODULE__, :rebooted?)
+  end
+
+  def rebooted do
+    GenServer.cast(__MODULE__, :rebooted)
+  end
 
   def need_reboot? do
     GenServer.call(__MODULE__, :need_reboot?)
@@ -29,41 +39,51 @@ def restart_after_boot(env) do
     GenServer.cast(__MODULE__, {:after_boot, env})
   end
 
+  def handle_call(:rebooted?, _from, state) do
+    {:reply, state[:rebooted], state}
+  end
+
   def handle_call(:need_reboot?, _from, state) do
-    {:reply, state[:need_reboot?], state}
+    {:reply, state[:need_reboot], state}
+  end
+
+  def handle_cast(:rebooted, state) do
+    {:noreply, Map.put(state, :rebooted, true)}
+  end
+
+  def handle_cast(:need_reboot, %{need_reboot: true} = state), do: {:noreply, state}
+
+  def handle_cast(:need_reboot, state) do
+    {:noreply, Map.put(state, :need_reboot, true)}
   end
 
   def handle_cast(:refresh, _state) do
-    {:noreply, %{need_reboot?: false}}
+    {:noreply, @init_state}
-  end
-
-  def handle_cast(:need_reboot, %{need_reboot?: true} = state), do: {:noreply, state}
-
-  def handle_cast(:need_reboot, state) do
-    {:noreply, Map.put(state, :need_reboot?, true)}
   end
 
   def handle_cast({:restart, :test, _}, state) do
     Logger.debug("pleroma manually restarted")
-    {:noreply, Map.put(state, :need_reboot?, false)}
+    {:noreply, Map.put(state, :need_reboot, false)}
   end
 
   def handle_cast({:restart, _, delay}, state) do
     Process.sleep(delay)
     do_restart(:pleroma)
-    {:noreply, Map.put(state, :need_reboot?, false)}
+    {:noreply, Map.put(state, :need_reboot, false)}
   end
 
   def handle_cast({:after_boot, _}, %{after_boot: true} = state), do: {:noreply, state}
 
   def handle_cast({:after_boot, :test}, state) do
     Logger.debug("pleroma restarted after boot")
-    {:noreply, Map.put(state, :after_boot, true)}
+    state = %{state | after_boot: true, rebooted: true}
+    {:noreply, state}
   end
 
   def handle_cast({:after_boot, _}, state) do
     do_restart(:pleroma)
-    {:noreply, Map.put(state, :after_boot, true)}
+    state = %{state | after_boot: true, rebooted: true}
+    {:noreply, state}
   end
 
   defp do_restart(app) do

test/web/admin_api/admin_api_controller_test.exs

@@ -1880,10 +1880,10 @@ test "deletes status", %{conn: conn, id: id, admin: admin} do
                "@#{admin.nickname} deleted status ##{id}"
     end
 
-    test "returns error when status is not exist", %{conn: conn} do
+    test "returns 404 when the status does not exist", %{conn: conn} do
       conn = delete(conn, "/api/pleroma/admin/statuses/test")
 
-      assert json_response(conn, :bad_request) == "Could not delete"
+      assert json_response(conn, :not_found) == "Not found"
     end
   end
 

test/web/mastodon_api/controllers/auth_controller_test.exs

@@ -85,6 +85,37 @@ test "it sends an email to user", %{user: user} do
     end
   end
 
+  describe "POST /auth/password, with nickname" do
+    test "it returns 204", %{conn: conn} do
+      user = insert(:user)
+
+      assert conn
+             |> post("/auth/password?nickname=#{user.nickname}")
+             |> json_response(:no_content)
+
+      ObanHelpers.perform_all()
+      token_record = Repo.get_by(Pleroma.PasswordResetToken, user_id: user.id)
+
+      email = Pleroma.Emails.UserEmail.password_reset_email(user, token_record.token)
+      notify_email = Config.get([:instance, :notify_email])
+      instance_name = Config.get([:instance, :name])
+
+      assert_email_sent(
+        from: {instance_name, notify_email},
+        to: {user.name, user.email},
+        html_body: email.html_body
+      )
+    end
+
+    test "it doesn't fail when a user has no email", %{conn: conn} do
+      user = insert(:user, %{email: nil})
+
+      assert conn
+             |> post("/auth/password?nickname=#{user.nickname}")
+             |> json_response(:no_content)
+    end
+  end
+
   describe "POST /auth/password, with invalid parameters" do
     setup do
       user = insert(:user)

test/web/mastodon_api/controllers/report_controller_test.exs

@@ -75,4 +75,13 @@ test "returns error when account is not exist", %{
 
     assert json_response(conn, 400) == %{"error" => "Account not found"}
   end
+
+  test "doesn't fail if an admin has no email", %{conn: conn, target_user: target_user} do
+    insert(:user, %{is_admin: true, email: nil})
+
+    assert %{"action_taken" => false, "id" => _} =
+             conn
+             |> post("/api/v1/reports", %{"account_id" => target_user.id})
+             |> json_response(200)
+  end
 end

test/web/mastodon_api/controllers/status_controller_test.exs

@@ -476,6 +476,15 @@ test "get a status" do
     assert id == to_string(activity.id)
   end
 
+  test "getting a status that doesn't exist returns 404" do
+    %{conn: conn} = oauth_access(["read:statuses"])
+    activity = insert(:note_activity)
+
+    conn = get(conn, "/api/v1/statuses/#{String.downcase(activity.id)}")
+
+    assert json_response(conn, 404) == %{"error" => "Record not found"}
+  end
+
   test "get a direct status" do
     %{user: user, conn: conn} = oauth_access(["read:statuses"])
     other_user = insert(:user)
@@ -520,6 +529,18 @@ test "when you created it" do
       refute Activity.get_by_id(activity.id)
     end
 
+    test "when it doesn't exist" do
+      %{user: author, conn: conn} = oauth_access(["write:statuses"])
+      activity = insert(:note_activity, user: author)
+
+      conn =
+        conn
+        |> assign(:user, author)
+        |> delete("/api/v1/statuses/#{String.downcase(activity.id)}")
+
+      assert %{"error" => "Record not found"} == json_response(conn, 404)
+    end
+
     test "when you didn't create it" do
       %{conn: conn} = oauth_access(["write:statuses"])
       activity = insert(:note_activity)
@@ -574,6 +595,14 @@ test "reblogs and returns the reblogged status", %{conn: conn} do
       assert to_string(activity.id) == id
     end
 
+    test "returns 404 if the reblogged status doesn't exist", %{conn: conn} do
+      activity = insert(:note_activity)
+
+      conn = post(conn, "/api/v1/statuses/#{String.downcase(activity.id)}/reblog")
+
+      assert %{"error" => "Record not found"} = json_response(conn, 404)
+    end
+
     test "reblogs privately and returns the reblogged status", %{conn: conn} do
       activity = insert(:note_activity)
 
@@ -626,12 +655,6 @@ test "reblogged status for another user" do
 
       assert to_string(activity.id) == id
     end
-
-    test "returns 400 error when activity is not exist", %{conn: conn} do
-      conn = post(conn, "/api/v1/statuses/foo/reblog")
-
-      assert json_response(conn, 400) == %{"error" => "Could not repeat"}
-    end
   end
 
   describe "unreblogging" do
@@ -649,10 +672,10 @@ test "unreblogs and returns the unreblogged status", %{user: user, conn: conn} d
       assert to_string(activity.id) == id
     end
 
-    test "returns 400 error when activity is not exist", %{conn: conn} do
+    test "returns 404 error when activity does not exist", %{conn: conn} do
       conn = post(conn, "/api/v1/statuses/foo/unreblog")
 
-      assert json_response(conn, 400) == %{"error" => "Could not unrepeat"}
+      assert json_response(conn, 404) == %{"error" => "Record not found"}
     end
   end
 
@@ -677,10 +700,10 @@ test "favoriting twice will just return 200", %{conn: conn} do
       assert post(conn, "/api/v1/statuses/#{activity.id}/favourite") |> json_response(200)
     end
 
-    test "returns 400 error for a wrong id", %{conn: conn} do
+    test "returns 404 error for a wrong id", %{conn: conn} do
       conn = post(conn, "/api/v1/statuses/1/favourite")
 
-      assert json_response(conn, 400) == %{"error" => "Could not favorite"}
+      assert json_response(conn, 404) == %{"error" => "Record not found"}
     end
   end
 
@@ -700,10 +723,10 @@ test "unfavorites a status and returns it", %{user: user, conn: conn} do
       assert to_string(activity.id) == id
     end
 
-    test "returns 400 error for a wrong id", %{conn: conn} do
+    test "returns 404 error for a wrong id", %{conn: conn} do
       conn = post(conn, "/api/v1/statuses/1/unfavourite")
 
-      assert json_response(conn, 400) == %{"error" => "Could not unfavorite"}
+      assert json_response(conn, 404) == %{"error" => "Record not found"}
     end
   end
 

test/workers/cron/digest_emails_worker_test.exs

@@ -13,7 +13,7 @@ defmodule Pleroma.Workers.Cron.DigestEmailsWorkerTest do
 
   clear_config([:email_notifications, :digest])
 
-  test "it sends digest emails" do
+  setup do
     Pleroma.Config.put([:email_notifications, :digest], %{
       active: true,
       inactivity_threshold: 7,
@@ -31,6 +31,10 @@ test "it sends digest emails" do
     {:ok, _} = User.switch_email_notifications(user2, "digest", true)
     CommonAPI.post(user, %{"status" => "hey @#{user2.nickname}!"})
 
+    {:ok, user2: user2}
+  end
+
+  test "it sends digest emails", %{user2: user2} do
     Pleroma.Workers.Cron.DigestEmailsWorker.perform(:opts, :pid)
     # Performing job(s) enqueued at previous step
     ObanHelpers.perform_all()
@@ -39,4 +43,12 @@ test "it sends digest emails" do
     assert email.to == [{user2.name, user2.email}]
     assert email.subject == "Your digest from #{Pleroma.Config.get(:instance)[:name]}"
   end
+
+  test "it doesn't fail when a user has no email", %{user2: user2} do
+    {:ok, _} = user2 |> Ecto.Changeset.change(%{email: nil}) |> Pleroma.Repo.update()
+
+    Pleroma.Workers.Cron.DigestEmailsWorker.perform(:opts, :pid)
+    # Performing job(s) enqueued at previous step
+    ObanHelpers.perform_all()
+  end
 end

test/workers/cron/new_users_digest_worker_test.exs

@@ -29,4 +29,16 @@ test "it sends new users digest emails" do
     assert email.html_body =~ user2.nickname
     assert email.html_body =~ "cofe"
   end
+
+  test "it doesn't fail when admin has no email" do
+    yesterday = NaiveDateTime.utc_now() |> Timex.shift(days: -1)
+    insert(:user, %{is_admin: true, email: nil})
+    insert(:user, %{inserted_at: yesterday})
+    user = insert(:user, %{inserted_at: yesterday})
+
+    CommonAPI.post(user, %{"status" => "cofe"})
+
+    NewUsersDigestWorker.perform(nil, nil)
+    ObanHelpers.perform_all()
+  end
 end