Merge branch 'develop' of https://git.pleroma.social/pleroma/pleroma into develop

CHANGELOG.md

@@ -21,6 +21,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Deprecated `User.Info` embedded schema (fields moved to `User`)
 - Store status data inside Flag activity
 - Deprecated (reorganized as `UserRelationship` entity) User fields with user AP IDs (`blocks`, `mutes`, `muted_reblogs`, `muted_notifications`, `subscribers`).
+- Logger: default log level changed from `warn` to `info`.
 <details>
   <summary>API Changes</summary>
 
@@ -51,6 +52,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Support for `X-Forwarded-For` and similar HTTP headers which used by reverse proxies to pass a real user IP address to the backend. Must not be enabled unless your instance is behind at least one reverse proxy (such as Nginx, Apache HTTPD or Varnish Cache).
 - MRF: New module which handles incoming posts based on their age. By default, all incoming posts that are older than 2 days will be unlisted and not shown to their followers.
 - User notification settings: Add `privacy_option` option.
+- Support for custom Elixir modules (such as MRF policies)
 - User settings: Add _This account is a_ option.
 - OAuth: admin scopes support (relevant setting: `[:auth, :enforce_oauth_admin_scope_usage]`).
 <details>

config/config.exs

@@ -67,7 +67,7 @@
 
 config :pleroma, Pleroma.Captcha,
   enabled: true,
-  seconds_valid: 60,
+  seconds_valid: 3000,
   method: Pleroma.Captcha.Native
 
 config :pleroma, Pleroma.Captcha.Kocaptcha, endpoint: "https://captcha.kotobank.ch"
@@ -621,6 +621,8 @@
   activity_pub: nil,
   activity_pub_question: 30_000
 
+config :pleroma, :modules, runtime_dir: "instance/modules"
+
 config :swarm, node_blacklist: [~r/myhtml_.*$/]
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.

config/prod.exs

@@ -20,8 +20,8 @@
 config :phoenix, serve_endpoints: true
 
 # Do not print debug messages in production
-config :logger, :console, level: :warn
+config :logger, :console, level: :info
-config :logger, :ex_syslogger, level: :warn
+config :logger, :ex_syslogger, level: :info
 
 # ## SSL Support
 #

config/releases.exs

@@ -2,6 +2,7 @@
 
 config :pleroma, :instance, static_dir: "/var/lib/pleroma/static"
 config :pleroma, Pleroma.Uploaders.Local, uploads: "/var/lib/pleroma/uploads"
+config :pleroma, :modules, runtime_dir: "/var/lib/pleroma/modules"
 
 config_path = System.get_env("PLEROMA_CONFIG_PATH") || "/etc/pleroma/config.exs"
 

config/test.exs

@@ -95,6 +95,8 @@
 
 config :pleroma, Pleroma.ReverseProxy.Client, Pleroma.ReverseProxy.ClientMock
 
+config :pleroma, :modules, runtime_dir: "test/fixtures/modules"
+
 if File.exists?("./config/test.secret.exs") do
   import_config "test.secret.exs"
 else

docs/API/pleroma_api.md

@@ -70,59 +70,6 @@ Request parameters can be passed via [query strings](https://en.wikipedia.org/wi
 * Response: JSON. Returns `{"status": "success"}` if the account was successfully disabled, `{"error": "[error message]"}` otherwise
 * Example response: `{"error": "Invalid password."}`
 
-## `/api/account/register`
-### Register a new user
-* Method `POST`
-* Authentication: not required
-* Params:
-    * `nickname`
-    * `fullname`
-    * `bio`
-    * `email`
-    * `password`
-    * `confirm`
-    * `captcha_solution`: optional, contains provider-specific captcha solution,
-    * `captcha_token`: optional, contains provider-specific captcha token
-    * `token`: invite token required when the registrations aren't public.
-* Response: JSON. Returns a user object on success, otherwise returns `{"error": "error_msg"}`
-* Example response:
-```json
-{
-	"background_image": null,
-	"cover_photo": "https://pleroma.soykaf.com/images/banner.png",
-	"created_at": "Tue Dec 18 16:55:56 +0000 2018",
-	"default_scope": "public",
-	"description": "blushy-crushy fediverse idol + pleroma dev\nlet's be friends \nぷれろまの生徒会長。謎の外人。日本語OK. \n公主病.",
-	"description_html": "blushy-crushy fediverse idol + pleroma dev.<br />let's be friends <br />ぷれろまの生徒会長。謎の外人。日本語OK. <br />公主病.",
-	"favourites_count": 0,
-	"fields": [],
-	"followers_count": 0,
-	"following": false,
-	"follows_you": false,
-	"friends_count": 0,
-	"id": 6,
-	"is_local": true,
-	"locked": false,
-	"name": "lain",
-	"name_html": "lain",
-	"no_rich_text": false,
-	"pleroma": {
-		"tags": []
-	},
-	"profile_image_url": "https://pleroma.soykaf.com/images/avi.png",
-	"profile_image_url_https": "https://pleroma.soykaf.com/images/avi.png",
-	"profile_image_url_original": "https://pleroma.soykaf.com/images/avi.png",
-	"profile_image_url_profile_size": "https://pleroma.soykaf.com/images/avi.png",
-	"rights": {
-		"delete_others_notice": false
-	},
-	"screen_name": "lain",
-	"statuses_count": 0,
-	"statusnet_blocking": false,
-	"statusnet_profile_url": "https://pleroma.soykaf.com/users/lain"
-}
-```
-
 ## `/api/pleroma/admin/`…
 See [Admin-API](admin_api.md)
 

docs/configuration/cheatsheet.md

@@ -836,3 +836,7 @@ config :auto_linker,
     rel: "ugc"
   ]
 ```
+
+## Custom Runtime Modules (`:modules`)
+
+* `runtime_dir`: A path to custom Elixir modules (such as MRF policies).

lib/pleroma/application.ex

@@ -5,6 +5,7 @@
 defmodule Pleroma.Application do
   import Cachex.Spec
   use Application
+  require Logger
 
   @name Mix.Project.config()[:name]
   @version Mix.Project.config()[:version]
@@ -33,6 +34,7 @@ def start(_type, _args) do
     Pleroma.HTML.compile_scrubbers()
     Pleroma.Config.DeprecationWarnings.warn()
     setup_instrumenters()
+    load_custom_modules()
 
     # Define workers and child supervisors to be supervised
     children =
@@ -68,6 +70,28 @@ def start(_type, _args) do
     Supervisor.start_link(children, opts)
   end
 
+  def load_custom_modules do
+    dir = Pleroma.Config.get([:modules, :runtime_dir])
+
+    if dir && File.exists?(dir) do
+      dir
+      |> Pleroma.Utils.compile_dir()
+      |> case do
+        {:error, _errors, _warnings} ->
+          raise "Invalid custom modules"
+
+        {:ok, modules, _warnings} ->
+          if @env != :test do
+            Enum.each(modules, fn mod ->
+              Logger.info("Custom module loaded: #{inspect(mod)}")
+            end)
+          end
+
+          :ok
+      end
+    end
+  end
+
   defp setup_instrumenters do
     require Prometheus.Registry
 

lib/pleroma/html.ex

@@ -10,9 +10,7 @@ def compile_scrubbers do
     dir = Path.join(:code.priv_dir(:pleroma), "scrubbers")
 
     dir
-    |> File.ls!()
+    |> Pleroma.Utils.compile_dir()
-    |> Enum.map(&Path.join(dir, &1))
-    |> Kernel.ParallelCompiler.compile()
     |> case do
       {:error, _errors, _warnings} ->
         raise "Compiling scrubbers failed"

lib/pleroma/object/fetcher.ex

@@ -154,7 +154,7 @@ defp maybe_date_fetch(headers, date) do
   end
 
   def fetch_and_contain_remote_object_from_id(id) when is_binary(id) do
-    Logger.info("Fetching object #{id} via AP")
+    Logger.debug("Fetching object #{id} via AP")
 
     date = Pleroma.Signature.signed_date()
 

lib/pleroma/utils.ex

@@ -0,0 +1,12 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Utils do
+  def compile_dir(dir) when is_binary(dir) do
+    dir
+    |> File.ls!()
+    |> Enum.map(&Path.join(dir, &1))
+    |> Kernel.ParallelCompiler.compile()
+  end
+end

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -1298,28 +1298,26 @@ defp object_to_user_data(data) do
   def fetch_follow_information_for_user(user) do
     with {:ok, following_data} <-
            Fetcher.fetch_and_contain_remote_object_from_id(user.following_address),
-         following_count when is_integer(following_count) <- following_data["totalItems"],
          {:ok, hide_follows} <- collection_private(following_data),
          {:ok, followers_data} <-
            Fetcher.fetch_and_contain_remote_object_from_id(user.follower_address),
-         followers_count when is_integer(followers_count) <- followers_data["totalItems"],
          {:ok, hide_followers} <- collection_private(followers_data) do
       {:ok,
        %{
          hide_follows: hide_follows,
-         follower_count: followers_count,
+         follower_count: normalize_counter(followers_data["totalItems"]),
-         following_count: following_count,
+         following_count: normalize_counter(following_data["totalItems"]),
          hide_followers: hide_followers
        }}
     else
-      {:error, _} = e ->
+      {:error, _} = e -> e
-        e
+      e -> {:error, e}
-
-      e ->
-        {:error, e}
     end
   end
 
+  defp normalize_counter(counter) when is_integer(counter), do: counter
+  defp normalize_counter(_), do: 0
+
   defp maybe_update_follow_information(data) do
     with {:enabled, true} <-
            {:enabled, Pleroma.Config.get([:instance, :external_user_synchronization])},
@@ -1339,24 +1337,18 @@ defp maybe_update_follow_information(data) do
     end
   end
 
+  defp collection_private(%{"first" => %{"type" => type}})
+       when type in ["CollectionPage", "OrderedCollectionPage"],
+       do: {:ok, false}
+
   defp collection_private(%{"first" => first}) do
-    if is_map(first) and
+    with {:ok, %{"type" => type}} when type in ["CollectionPage", "OrderedCollectionPage"] <-
-         first["type"] in ["CollectionPage", "OrderedCollectionPage"] do
+           Fetcher.fetch_and_contain_remote_object_from_id(first) do
       {:ok, false}
     else
-      with {:ok, %{"type" => type}} when type in ["CollectionPage", "OrderedCollectionPage"] <-
+      {:error, {:ok, %{status: code}}} when code in [401, 403] -> {:ok, true}
-             Fetcher.fetch_and_contain_remote_object_from_id(first) do
+      {:error, _} = e -> e
-        {:ok, false}
+      e -> {:error, e}
-      else
-        {:error, {:ok, %{status: code}}} when code in [401, 403] ->
-          {:ok, true}
-
-        {:error, _} = e ->
-          e
-
-        e ->
-          {:error, e}
-      end
     end
   end
 

lib/pleroma/web/activity_pub/activity_pub_controller.ex

@@ -257,7 +257,7 @@ def inbox(%{assigns: %{valid_signature: true}} = conn, params) do
 
   # only accept relayed Creates
   def inbox(conn, %{"type" => "Create"} = params) do
-    Logger.info(
+    Logger.debug(
       "Signature missing or not from author, relayed Create message, fetching object from source"
     )
 
@@ -270,11 +270,11 @@ def inbox(conn, params) do
     headers = Enum.into(conn.req_headers, %{})
 
     if String.contains?(headers["signature"], params["actor"]) do
-      Logger.info(
+      Logger.debug(
         "Signature validation error for: #{params["actor"]}, make sure you are forwarding the HTTP Host header!"
       )
 
-      Logger.info(inspect(conn.req_headers))
+      Logger.debug(inspect(conn.req_headers))
     end
 
     json(conn, dgettext("errors", "error"))

lib/pleroma/web/activity_pub/mrf/drop_policy.ex

@@ -9,7 +9,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.DropPolicy do
 
   @impl true
   def filter(object) do
-    Logger.info("REJECTING #{inspect(object)}")
+    Logger.debug("REJECTING #{inspect(object)}")
     {:reject, object}
   end
 

lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex

@@ -18,7 +18,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
   ]
 
   def perform(:prefetch, url) do
-    Logger.info("Prefetching #{inspect(url)}")
+    Logger.debug("Prefetching #{inspect(url)}")
 
     url
     |> MediaProxy.url()

lib/pleroma/web/activity_pub/publisher.ex

@@ -48,7 +48,7 @@ def is_representable?(%Activity{} = activity) do
   * `id`: the ActivityStreams URI of the message
   """
   def publish_one(%{inbox: inbox, json: json, actor: %User{} = actor, id: id} = params) do
-    Logger.info("Federating #{id} to #{inbox}")
+    Logger.debug("Federating #{id} to #{inbox}")
     %{host: host, path: path} = URI.parse(inbox)
 
     digest = "SHA-256=" <> (:crypto.hash(:sha256, json) |> Base.encode64())
@@ -228,7 +228,7 @@ def publish(%User{} = actor, %Activity{} = activity) do
     public = is_public?(activity)
 
     if public && Config.get([:instance, :allow_relay]) do
-      Logger.info(fn -> "Relaying #{activity.data["id"]} out" end)
+      Logger.debug(fn -> "Relaying #{activity.data["id"]} out" end)
       Relay.publish(activity)
     end
 

lib/pleroma/web/endpoint.ex

@@ -59,7 +59,7 @@ defmodule Pleroma.Web.Endpoint do
 
   plug(Pleroma.Plugs.TrailingFormatPlug)
   plug(Plug.RequestId)
-  plug(Plug.Logger)
+  plug(Plug.Logger, log: :debug)
 
   plug(Pleroma.Plugs.Parsers)
 

lib/pleroma/web/federator/federator.ex

@@ -58,7 +58,7 @@ def perform(:publish, activity) do
   end
 
   def perform(:incoming_ap_doc, params) do
-    Logger.info("Handling incoming AP activity")
+    Logger.debug("Handling incoming AP activity")
 
     params = Utils.normalize_params(params)
 
@@ -71,13 +71,13 @@ def perform(:incoming_ap_doc, params) do
       {:ok, activity}
     else
       %Activity{} ->
-        Logger.info("Already had #{params["id"]}")
+        Logger.debug("Already had #{params["id"]}")
         :error
 
       _e ->
         # Just drop those for now
-        Logger.info("Unhandled activity")
+        Logger.debug("Unhandled activity")
-        Logger.info(Jason.encode!(params, pretty: true))
+        Logger.debug(Jason.encode!(params, pretty: true))
         :error
     end
   end

lib/pleroma/web/federator/publisher.ex

@@ -47,7 +47,7 @@ def publish(%User{} = user, %Activity{} = activity) do
     Config.get([:instance, :federation_publisher_modules])
     |> Enum.each(fn module ->
       if module.is_representable?(activity) do
-        Logger.info("Publishing #{activity.data["id"]} using #{inspect(module)}")
+        Logger.debug("Publishing #{activity.data["id"]} using #{inspect(module)}")
         module.publish(user, activity)
       end
     end)

lib/pleroma/web/metadata/twitter_card.ex

@@ -31,7 +31,7 @@ def build_tags(%{activity_id: id, object: object, user: user}) do
       if attachments == [] or Metadata.activity_nsfw?(object) do
         [
           image_tag(user),
-          {:meta, [property: "twitter:card", content: "summary_large_image"], []}
+          {:meta, [property: "twitter:card", content: "summary"], []}
         ]
       else
         attachments

test/fixtures/modules/runtime_module.ex

@@ -0,0 +1,9 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule RuntimeModule do
+  @moduledoc """
+  This is a dummy module to test custom runtime modules.
+  """
+end

test/runtime_test.exs

@@ -0,0 +1,11 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.RuntimeTest do
+  use ExUnit.Case, async: true
+
+  test "it loads custom runtime modules" do
+    assert Code.ensure_compiled?(RuntimeModule)
+  end
+end

test/web/activity_pub/activity_pub_test.exs

@@ -1623,6 +1623,44 @@ test "detects hidden follows/followers for friendica" do
       assert follow_info.following_count == 32
       assert follow_info.hide_follows == true
     end
+
+    test "doesn't crash when follower and following counters are hidden" do
+      mock(fn env ->
+        case env.url do
+          "http://localhost:4001/users/masto_hidden_counters/following" ->
+            json(%{
+              "@context" => "https://www.w3.org/ns/activitystreams",
+              "id" => "http://localhost:4001/users/masto_hidden_counters/followers"
+            })
+
+          "http://localhost:4001/users/masto_hidden_counters/following?page=1" ->
+            %Tesla.Env{status: 403, body: ""}
+
+          "http://localhost:4001/users/masto_hidden_counters/followers" ->
+            json(%{
+              "@context" => "https://www.w3.org/ns/activitystreams",
+              "id" => "http://localhost:4001/users/masto_hidden_counters/following"
+            })
+
+          "http://localhost:4001/users/masto_hidden_counters/followers?page=1" ->
+            %Tesla.Env{status: 403, body: ""}
+        end
+      end)
+
+      user =
+        insert(:user,
+          local: false,
+          follower_address: "http://localhost:4001/users/masto_hidden_counters/followers",
+          following_address: "http://localhost:4001/users/masto_hidden_counters/following"
+        )
+
+      {:ok, follow_info} = ActivityPub.fetch_follow_information_for_user(user)
+
+      assert follow_info.hide_followers == true
+      assert follow_info.follower_count == 0
+      assert follow_info.hide_follows == true
+      assert follow_info.following_count == 0
+    end
   end
 
   describe "fetch_favourites/3" do

test/web/metadata/twitter_card_test.exs

@@ -26,7 +26,32 @@ test "it renders twitter card for user info" do
            ]
   end
 
-  test "it does not render attachments if post is nsfw" do
+  test "it uses summary twittercard if post has no attachment" do
+    user = insert(:user, name: "Jimmy Hendriks", bio: "born 19 March 1994")
+    {:ok, activity} = CommonAPI.post(user, %{"status" => "HI"})
+
+    note =
+      insert(:note, %{
+        data: %{
+          "actor" => user.ap_id,
+          "tag" => [],
+          "id" => "https://pleroma.gov/objects/whatever",
+          "content" => "pleroma in a nutshell"
+        }
+      })
+
+    result = TwitterCard.build_tags(%{object: note, user: user, activity_id: activity.id})
+
+    assert [
+             {:meta, [property: "twitter:title", content: Utils.user_name_string(user)], []},
+             {:meta, [property: "twitter:description", content: "“pleroma in a nutshell”"], []},
+             {:meta, [property: "twitter:image", content: "http://localhost:4001/images/avi.png"],
+              []},
+             {:meta, [property: "twitter:card", content: "summary"], []}
+           ] == result
+  end
+
+  test "it renders avatar not attachment if post is nsfw and unfurl_nsfw is disabled" do
     Pleroma.Config.put([Pleroma.Web.Metadata, :unfurl_nsfw], false)
     user = insert(:user, name: "Jimmy Hendriks", bio: "born 19 March 1994")
     {:ok, activity} = CommonAPI.post(user, %{"status" => "HI"})
@@ -67,7 +92,7 @@ test "it does not render attachments if post is nsfw" do
              {:meta, [property: "twitter:description", content: "“pleroma in a nutshell”"], []},
              {:meta, [property: "twitter:image", content: "http://localhost:4001/images/avi.png"],
               []},
-             {:meta, [property: "twitter:card", content: "summary_large_image"], []}
+             {:meta, [property: "twitter:card", content: "summary"], []}
            ] == result
   end