Merge branch 'develop' into issue/1383

.gitlab-ci.yml

@@ -1,13 +1,13 @@
 image: elixir:1.8.1
 
-variables:
+variables: &global_variables
   POSTGRES_DB: pleroma_test
   POSTGRES_USER: postgres
   POSTGRES_PASSWORD: postgres
   DB_HOST: postgres
   MIX_ENV: test
 
-cache:
+cache: &global_cache_policy
   key: ${CI_COMMIT_REF_SLUG}
   paths:
           - deps
@@ -46,6 +46,10 @@ benchmark:
 
 unit-testing:
   stage: test
+  cache: &testing_cache_policy
+    <<: *global_cache_policy
+    policy: pull
+
   services:
   - name: postgres:9.6
     alias: postgres
@@ -58,6 +62,7 @@ unit-testing:
 
 federated-testing:
   stage: test
+  cache: *testing_cache_policy
   services:
   - name: minibikini/postgres-with-rum:12
     alias: postgres
@@ -71,11 +76,13 @@ federated-testing:
 
 unit-testing-rum:
   stage: test
+  cache: *testing_cache_policy
   services:
   - name: minibikini/postgres-with-rum:12
     alias: postgres
     command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
   variables:
+    <<: *global_variables
     RUM_ENABLED: "true"
   script:
     - mix deps.get
@@ -86,17 +93,20 @@ unit-testing-rum:
 
 lint:
   stage: test
+  cache: *testing_cache_policy
   script:
     - mix format --check-formatted
 
 analysis:
   stage: test
+  cache: *testing_cache_policy
   script:
     - mix deps.get
     - mix credo --strict --only=warnings,todo,fixme,consistency,readability
 
 docs-deploy:
   stage: deploy
+  cache: *testing_cache_policy
   image: alpine:latest
   only:
   - stable@pleroma/pleroma

CHANGELOG.md

@@ -21,6 +21,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Deprecated `User.Info` embedded schema (fields moved to `User`)
 - Store status data inside Flag activity
 - Deprecated (reorganized as `UserRelationship` entity) User fields with user AP IDs (`blocks`, `mutes`, `muted_reblogs`, `muted_notifications`, `subscribers`).
+- Logger: default log level changed from `warn` to `info`.
 <details>
   <summary>API Changes</summary>
 
@@ -51,6 +52,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Support for `X-Forwarded-For` and similar HTTP headers which used by reverse proxies to pass a real user IP address to the backend. Must not be enabled unless your instance is behind at least one reverse proxy (such as Nginx, Apache HTTPD or Varnish Cache).
 - MRF: New module which handles incoming posts based on their age. By default, all incoming posts that are older than 2 days will be unlisted and not shown to their followers.
 - User notification settings: Add `privacy_option` option.
+- Support for custom Elixir modules (such as MRF policies)
 - User settings: Add _This account is a_ option.
 - OAuth: admin scopes support (relevant setting: `[:auth, :enforce_oauth_admin_scope_usage]`).
 <details>

config/config.exs

@@ -56,6 +56,8 @@
   seconds_valid: 60,
   method: Pleroma.Captcha.Native
 
+config :pleroma, Pleroma.Captcha.Kocaptcha, endpoint: "https://captcha.kotobank.ch"
+
 config :pleroma, :hackney_pools,
   federation: [
     max_connections: 50,
@@ -609,6 +611,8 @@
   activity_pub: nil,
   activity_pub_question: 30_000
 
+config :pleroma, :modules, runtime_dir: "instance/modules"
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"

config/prod.exs

@@ -20,8 +20,8 @@
 config :phoenix, serve_endpoints: true
 
 # Do not print debug messages in production
-config :logger, :console, level: :warn
+config :logger, :console, level: :info
-config :logger, :ex_syslogger, level: :warn
+config :logger, :ex_syslogger, level: :info
 
 # ## SSL Support
 #

config/releases.exs

@@ -2,6 +2,7 @@
 
 config :pleroma, :instance, static_dir: "/var/lib/pleroma/static"
 config :pleroma, Pleroma.Uploaders.Local, uploads: "/var/lib/pleroma/uploads"
+config :pleroma, :modules, runtime_dir: "/var/lib/pleroma/modules"
 
 config_path = System.get_env("PLEROMA_CONFIG_PATH") || "/etc/pleroma/config.exs"
 

config/test.exs

@@ -91,7 +91,7 @@
 
 config :pleroma, Pleroma.ReverseProxy.Client, Pleroma.ReverseProxy.ClientMock
 
-config :pleroma, Pleroma.Captcha.Kocaptcha, endpoint: "https://captcha.kotobank.ch"
+config :pleroma, :modules, runtime_dir: "test/fixtures/modules"
 
 if File.exists?("./config/test.secret.exs") do
   import_config "test.secret.exs"

docs/API/pleroma_api.md

@@ -70,59 +70,6 @@ Request parameters can be passed via [query strings](https://en.wikipedia.org/wi
 * Response: JSON. Returns `{"status": "success"}` if the account was successfully disabled, `{"error": "[error message]"}` otherwise
 * Example response: `{"error": "Invalid password."}`
 
-## `/api/account/register`
-### Register a new user
-* Method `POST`
-* Authentication: not required
-* Params:
-    * `nickname`
-    * `fullname`
-    * `bio`
-    * `email`
-    * `password`
-    * `confirm`
-    * `captcha_solution`: optional, contains provider-specific captcha solution,
-    * `captcha_token`: optional, contains provider-specific captcha token
-    * `token`: invite token required when the registrations aren't public.
-* Response: JSON. Returns a user object on success, otherwise returns `{"error": "error_msg"}`
-* Example response:
-```json
-{
-	"background_image": null,
-	"cover_photo": "https://pleroma.soykaf.com/images/banner.png",
-	"created_at": "Tue Dec 18 16:55:56 +0000 2018",
-	"default_scope": "public",
-	"description": "blushy-crushy fediverse idol + pleroma dev\nlet's be friends \nぷれろまの生徒会長。謎の外人。日本語OK. \n公主病.",
-	"description_html": "blushy-crushy fediverse idol + pleroma dev.<br />let's be friends <br />ぷれろまの生徒会長。謎の外人。日本語OK. <br />公主病.",
-	"favourites_count": 0,
-	"fields": [],
-	"followers_count": 0,
-	"following": false,
-	"follows_you": false,
-	"friends_count": 0,
-	"id": 6,
-	"is_local": true,
-	"locked": false,
-	"name": "lain",
-	"name_html": "lain",
-	"no_rich_text": false,
-	"pleroma": {
-		"tags": []
-	},
-	"profile_image_url": "https://pleroma.soykaf.com/images/avi.png",
-	"profile_image_url_https": "https://pleroma.soykaf.com/images/avi.png",
-	"profile_image_url_original": "https://pleroma.soykaf.com/images/avi.png",
-	"profile_image_url_profile_size": "https://pleroma.soykaf.com/images/avi.png",
-	"rights": {
-		"delete_others_notice": false
-	},
-	"screen_name": "lain",
-	"statuses_count": 0,
-	"statusnet_blocking": false,
-	"statusnet_profile_url": "https://pleroma.soykaf.com/users/lain"
-}
-```
-
 ## `/api/pleroma/admin/`…
 See [Admin-API](admin_api.md)
 

docs/configuration/cheatsheet.md

@@ -845,3 +845,7 @@ config :auto_linker,
     rel: "ugc"
   ]
 ```
+
+## Custom Runtime Modules (`:modules`)
+
+* `runtime_dir`: A path to custom Elixir modules (such as MRF policies).

lib/pleroma/application.ex

@@ -5,6 +5,7 @@
 defmodule Pleroma.Application do
   import Cachex.Spec
   use Application
+  require Logger
 
   @name Mix.Project.config()[:name]
   @version Mix.Project.config()[:version]
@@ -33,6 +34,7 @@ def start(_type, _args) do
     Pleroma.HTML.compile_scrubbers()
     Pleroma.Config.DeprecationWarnings.warn()
     setup_instrumenters()
+    load_custom_modules()
 
     # Define workers and child supervisors to be supervised
     children =
@@ -64,6 +66,28 @@ def start(_type, _args) do
     Supervisor.start_link(children, opts)
   end
 
+  def load_custom_modules do
+    dir = Pleroma.Config.get([:modules, :runtime_dir])
+
+    if dir && File.exists?(dir) do
+      dir
+      |> Pleroma.Utils.compile_dir()
+      |> case do
+        {:error, _errors, _warnings} ->
+          raise "Invalid custom modules"
+
+        {:ok, modules, _warnings} ->
+          if @env != :test do
+            Enum.each(modules, fn mod ->
+              Logger.info("Custom module loaded: #{inspect(mod)}")
+            end)
+          end
+
+          :ok
+      end
+    end
+  end
+
   defp setup_instrumenters do
     require Prometheus.Registry
 

lib/pleroma/html.ex

@@ -10,9 +10,7 @@ def compile_scrubbers do
     dir = Path.join(:code.priv_dir(:pleroma), "scrubbers")
 
     dir
-    |> File.ls!()
+    |> Pleroma.Utils.compile_dir()
-    |> Enum.map(&Path.join(dir, &1))
-    |> Kernel.ParallelCompiler.compile()
     |> case do
       {:error, _errors, _warnings} ->
         raise "Compiling scrubbers failed"

lib/pleroma/object/fetcher.ex

@@ -154,7 +154,7 @@ defp maybe_date_fetch(headers, date) do
   end
 
   def fetch_and_contain_remote_object_from_id(id) when is_binary(id) do
-    Logger.info("Fetching object #{id} via AP")
+    Logger.debug("Fetching object #{id} via AP")
 
     date = Pleroma.Signature.signed_date()
 

lib/pleroma/utils.ex

@@ -0,0 +1,12 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Utils do
+  def compile_dir(dir) when is_binary(dir) do
+    dir
+    |> File.ls!()
+    |> Enum.map(&Path.join(dir, &1))
+    |> Kernel.ParallelCompiler.compile()
+  end
+end

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -1298,28 +1298,26 @@ defp object_to_user_data(data) do
   def fetch_follow_information_for_user(user) do
     with {:ok, following_data} <-
            Fetcher.fetch_and_contain_remote_object_from_id(user.following_address),
-         following_count when is_integer(following_count) <- following_data["totalItems"],
          {:ok, hide_follows} <- collection_private(following_data),
          {:ok, followers_data} <-
            Fetcher.fetch_and_contain_remote_object_from_id(user.follower_address),
-         followers_count when is_integer(followers_count) <- followers_data["totalItems"],
          {:ok, hide_followers} <- collection_private(followers_data) do
       {:ok,
        %{
          hide_follows: hide_follows,
-         follower_count: followers_count,
+         follower_count: normalize_counter(followers_data["totalItems"]),
-         following_count: following_count,
+         following_count: normalize_counter(following_data["totalItems"]),
          hide_followers: hide_followers
        }}
     else
-      {:error, _} = e ->
+      {:error, _} = e -> e
-        e
+      e -> {:error, e}
-
-      e ->
-        {:error, e}
     end
   end
 
+  defp normalize_counter(counter) when is_integer(counter), do: counter
+  defp normalize_counter(_), do: 0
+
   defp maybe_update_follow_information(data) do
     with {:enabled, true} <-
            {:enabled, Pleroma.Config.get([:instance, :external_user_synchronization])},
@@ -1339,24 +1337,18 @@ defp maybe_update_follow_information(data) do
     end
   end
 
+  defp collection_private(%{"first" => %{"type" => type}})
+       when type in ["CollectionPage", "OrderedCollectionPage"],
+       do: {:ok, false}
+
   defp collection_private(%{"first" => first}) do
-    if is_map(first) and
+    with {:ok, %{"type" => type}} when type in ["CollectionPage", "OrderedCollectionPage"] <-
-         first["type"] in ["CollectionPage", "OrderedCollectionPage"] do
+           Fetcher.fetch_and_contain_remote_object_from_id(first) do
       {:ok, false}
     else
-      with {:ok, %{"type" => type}} when type in ["CollectionPage", "OrderedCollectionPage"] <-
+      {:error, {:ok, %{status: code}}} when code in [401, 403] -> {:ok, true}
-             Fetcher.fetch_and_contain_remote_object_from_id(first) do
+      {:error, _} = e -> e
-        {:ok, false}
+      e -> {:error, e}
-      else
-        {:error, {:ok, %{status: code}}} when code in [401, 403] ->
-          {:ok, true}
-
-        {:error, _} = e ->
-          e
-
-        e ->
-          {:error, e}
-      end
     end
   end
 

lib/pleroma/web/activity_pub/activity_pub_controller.ex

@@ -257,7 +257,7 @@ def inbox(%{assigns: %{valid_signature: true}} = conn, params) do
 
   # only accept relayed Creates
   def inbox(conn, %{"type" => "Create"} = params) do
-    Logger.info(
+    Logger.debug(
       "Signature missing or not from author, relayed Create message, fetching object from source"
     )
 
@@ -270,11 +270,11 @@ def inbox(conn, params) do
     headers = Enum.into(conn.req_headers, %{})
 
     if String.contains?(headers["signature"], params["actor"]) do
-      Logger.info(
+      Logger.debug(
         "Signature validation error for: #{params["actor"]}, make sure you are forwarding the HTTP Host header!"
       )
 
-      Logger.info(inspect(conn.req_headers))
+      Logger.debug(inspect(conn.req_headers))
     end
 
     json(conn, dgettext("errors", "error"))

lib/pleroma/web/activity_pub/mrf/drop_policy.ex

@@ -9,7 +9,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.DropPolicy do
 
   @impl true
   def filter(object) do
-    Logger.info("REJECTING #{inspect(object)}")
+    Logger.debug("REJECTING #{inspect(object)}")
     {:reject, object}
   end
 

lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex

@@ -18,7 +18,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
   ]
 
   def perform(:prefetch, url) do
-    Logger.info("Prefetching #{inspect(url)}")
+    Logger.debug("Prefetching #{inspect(url)}")
 
     url
     |> MediaProxy.url()

lib/pleroma/web/activity_pub/publisher.ex

@@ -48,7 +48,7 @@ def is_representable?(%Activity{} = activity) do
   * `id`: the ActivityStreams URI of the message
   """
   def publish_one(%{inbox: inbox, json: json, actor: %User{} = actor, id: id} = params) do
-    Logger.info("Federating #{id} to #{inbox}")
+    Logger.debug("Federating #{id} to #{inbox}")
     %{host: host, path: path} = URI.parse(inbox)
 
     digest = "SHA-256=" <> (:crypto.hash(:sha256, json) |> Base.encode64())
@@ -228,7 +228,7 @@ def publish(%User{} = actor, %Activity{} = activity) do
     public = is_public?(activity)
 
     if public && Config.get([:instance, :allow_relay]) do
-      Logger.info(fn -> "Relaying #{activity.data["id"]} out" end)
+      Logger.debug(fn -> "Relaying #{activity.data["id"]} out" end)
       Relay.publish(activity)
     end
 

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -397,7 +397,7 @@ def handle_incoming(
         %{"type" => "Create", "object" => %{"type" => objtype} = object} = data,
         options
       )
-      when objtype in ["Article", "Note", "Video", "Page", "Question", "Answer"] do
+      when objtype in ["Article", "Event", "Note", "Video", "Page", "Question", "Answer"] do
     actor = Containment.get_actor(data)
 
     data =

lib/pleroma/web/activity_pub/utils.ex

@@ -22,7 +22,16 @@ defmodule Pleroma.Web.ActivityPub.Utils do
   require Logger
   require Pleroma.Constants
 
-  @supported_object_types ["Article", "Note", "Video", "Page", "Question", "Answer", "Audio"]
+  @supported_object_types [
+    "Article",
+    "Note",
+    "Event",
+    "Video",
+    "Page",
+    "Question",
+    "Answer",
+    "Audio"
+  ]
   @strip_status_report_states ~w(closed resolved)
   @supported_report_states ~w(open closed resolved)
   @valid_visibilities ~w(public unlisted private direct)

lib/pleroma/web/activity_pub/views/user_view.ex

@@ -201,7 +201,6 @@ def render("followers.json", %{user: user} = opts) do
     %{
       "id" => "#{user.ap_id}/followers",
       "type" => "OrderedCollection",
-      "totalItems" => total,
       "first" =>
         if showing_items do
           collection(followers, "#{user.ap_id}/followers", 1, showing_items, total)
@@ -209,6 +208,7 @@ def render("followers.json", %{user: user} = opts) do
           "#{user.ap_id}/followers?page=1"
         end
     }
+    |> maybe_put_total_items(showing_count, total)
     |> Map.merge(Utils.make_json_ld_header())
   end
 
@@ -251,6 +251,12 @@ def render("activity_collection_page.json", %{activities: activities, iri: iri})
     |> Map.merge(Utils.make_json_ld_header())
   end
 
+  defp maybe_put_total_items(map, false, _total), do: map
+
+  defp maybe_put_total_items(map, true, total) do
+    Map.put(map, "totalItems", total)
+  end
+
   def collection(collection, iri, page, show_items \\ true, total \\ nil) do
     offset = (page - 1) * 10
     items = Enum.slice(collection, offset, 10)

lib/pleroma/web/endpoint.ex

@@ -59,7 +59,7 @@ defmodule Pleroma.Web.Endpoint do
 
   plug(Pleroma.Plugs.TrailingFormatPlug)
   plug(Plug.RequestId)
-  plug(Plug.Logger)
+  plug(Plug.Logger, log: :debug)
 
   plug(Pleroma.Plugs.Parsers)
 

lib/pleroma/web/federator/federator.ex

@@ -58,7 +58,7 @@ def perform(:publish, activity) do
   end
 
   def perform(:incoming_ap_doc, params) do
-    Logger.info("Handling incoming AP activity")
+    Logger.debug("Handling incoming AP activity")
 
     params = Utils.normalize_params(params)
 
@@ -71,13 +71,13 @@ def perform(:incoming_ap_doc, params) do
       {:ok, activity}
     else
       %Activity{} ->
-        Logger.info("Already had #{params["id"]}")
+        Logger.debug("Already had #{params["id"]}")
         :error
 
       _e ->
         # Just drop those for now
-        Logger.info("Unhandled activity")
+        Logger.debug("Unhandled activity")
-        Logger.info(Jason.encode!(params, pretty: true))
+        Logger.debug(Jason.encode!(params, pretty: true))
         :error
     end
   end

lib/pleroma/web/federator/publisher.ex

@@ -47,7 +47,7 @@ def publish(%User{} = user, %Activity{} = activity) do
     Config.get([:instance, :federation_publisher_modules])
     |> Enum.each(fn module ->
       if module.is_representable?(activity) do
-        Logger.info("Publishing #{activity.data["id"]} using #{inspect(module)}")
+        Logger.debug("Publishing #{activity.data["id"]} using #{inspect(module)}")
         module.publish(user, activity)
       end
     end)

lib/pleroma/web/mastodon_api/views/status_view.ex

@@ -421,7 +421,8 @@ def get_reply_to(%{data: %{"object" => _object}} = activity, _) do
     end
   end
 
-  def render_content(%{data: %{"type" => "Video"}} = object) do
+  def render_content(%{data: %{"type" => object_type}} = object)
+      when object_type in ["Video", "Event"] do
     with name when not is_nil(name) and name != "" <- object.data["name"] do
       "<p><a href=\"#{object.data["id"]}\">#{name}</a></p>#{object.data["content"]}"
     else

lib/pleroma/web/metadata/twitter_card.ex

@@ -31,7 +31,7 @@ def build_tags(%{activity_id: id, object: object, user: user}) do
       if attachments == [] or Metadata.activity_nsfw?(object) do
         [
           image_tag(user),
-          {:meta, [property: "twitter:card", content: "summary_large_image"], []}
+          {:meta, [property: "twitter:card", content: "summary"], []}
         ]
       else
         attachments

lib/pleroma/web/twitter_api/controllers/util_controller.ex

@@ -104,7 +104,8 @@ defp follow_template(_), do: "follow_login.html"
 
   defp is_status?(acct) do
     case Pleroma.Object.Fetcher.fetch_and_contain_remote_object_from_id(acct) do
-      {:ok, %{"type" => type}} when type in ["Article", "Note", "Video", "Page", "Question"] ->
+      {:ok, %{"type" => type}}
+      when type in ["Article", "Event", "Note", "Video", "Page", "Question"] ->
         true
 
       _ ->

test/conversation/participation_test.exs

@@ -5,7 +5,9 @@
 defmodule Pleroma.Conversation.ParticipationTest do
   use Pleroma.DataCase
   import Pleroma.Factory
+  alias Pleroma.Conversation
   alias Pleroma.Conversation.Participation
+  alias Pleroma.Repo
   alias Pleroma.User
   alias Pleroma.Web.CommonAPI
 
@@ -98,7 +100,9 @@ test "it creates a participation for a conversation and a user" do
     assert participation.user_id == user.id
     assert participation.conversation_id == conversation.id
 
+    # Needed because updated_at is accurate down to a second
     :timer.sleep(1000)
+
     # Creating again returns the same participation
     {:ok, %Participation{} = participation_two} =
       Participation.create_for_user_and_conversation(user, conversation)
@@ -150,9 +154,7 @@ test "it marks all the user's participations as read" do
   test "gets all the participations for a user, ordered by updated at descending" do
     user = insert(:user)
     {:ok, activity_one} = CommonAPI.post(user, %{"status" => "x", "visibility" => "direct"})
-    :timer.sleep(1000)
     {:ok, activity_two} = CommonAPI.post(user, %{"status" => "x", "visibility" => "direct"})
-    :timer.sleep(1000)
 
     {:ok, activity_three} =
       CommonAPI.post(user, %{
@@ -161,6 +163,17 @@ test "gets all the participations for a user, ordered by updated at descending"
         "in_reply_to_status_id" => activity_one.id
       })
 
+    # Offset participations because the accuracy of updated_at is down to a second
+
+    for {activity, offset} <- [{activity_two, 1}, {activity_three, 2}] do
+      conversation = Conversation.get_for_ap_id(activity.data["context"])
+      participation = Participation.for_user_and_conversation(user, conversation)
+      updated_at = NaiveDateTime.add(Map.get(participation, :updated_at), offset)
+
+      Ecto.Changeset.change(participation, %{updated_at: updated_at})
+      |> Repo.update!()
+    end
+
     assert [participation_one, participation_two] = Participation.for_user(user)
 
     object2 = Pleroma.Object.normalize(activity_two)

test/fixtures/modules/runtime_module.ex

@@ -0,0 +1,9 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule RuntimeModule do
+  @moduledoc """
+  This is a dummy module to test custom runtime modules.
+  """
+end

test/fixtures/tesla_mock/mobilizon.org-event.json

@@ -0,0 +1 @@
+{"@context":["https://www.w3.org/ns/activitystreams","https://litepub.social/litepub/context.jsonld",{"GeoCoordinates":"sc:GeoCoordinates","Hashtag":"as:Hashtag","Place":"sc:Place","PostalAddress":"sc:PostalAddress","address":{"@id":"sc:address","@type":"sc:PostalAddress"},"addressCountry":"sc:addressCountry","addressLocality":"sc:addressLocality","addressRegion":"sc:addressRegion","category":"sc:category","commentsEnabled":{"@id":"pt:commentsEnabled","@type":"sc:Boolean"},"geo":{"@id":"sc:geo","@type":"sc:GeoCoordinates"},"ical":"http://www.w3.org/2002/12/cal/ical#","joinMode":{"@id":"mz:joinMode","@type":"mz:joinModeType"},"joinModeType":{"@id":"mz:joinModeType","@type":"rdfs:Class"},"location":{"@id":"sc:location","@type":"sc:Place"},"maximumAttendeeCapacity":"sc:maximumAttendeeCapacity","mz":"https://joinmobilizon.org/ns#","postalCode":"sc:postalCode","pt":"https://joinpeertube.org/ns#","repliesModerationOption":{"@id":"mz:repliesModerationOption","@type":"mz:repliesModerationOptionType"},"repliesModerationOptionType":{"@id":"mz:repliesModerationOptionType","@type":"rdfs:Class"},"sc":"http://schema.org#","streetAddress":"sc:streetAddress","uuid":"sc:identifier"}],"actor":"https://mobilizon.org/@tcit","attributedTo":"https://mobilizon.org/@tcit","category":"meeting","cc":[],"commentsEnabled":true,"content":"<p>Mobilizon is now federated! 🎉</p><p></p><p>You can view this event from other instances if they are subscribed to mobilizon.org, and soon directly from Mastodon and Pleroma. It is possible that you may see some comments from other instances, including Mastodon ones, just below.</p><p></p><p>With a Mobilizon account on an instance, you may <strong>participate</strong> at events from other instances and <strong>add comments</strong> on events.</p><p></p><p>Of course, it's still <u>a work in progress</u>: if reports made from an instance on events and comments can be federated, you can't block people right now, and moderators actions are rather limited, but this <strong>will definitely get fixed over time</strong> until first stable version next year.</p><p></p><p>Anyway, if you want to come up with some feedback, head over to our forum or - if you feel you have technical skills and are familiar with it - on our Gitlab repository.</p><p></p><p>Also, to people that want to set Mobilizon themselves even though we really don't advise to do that for now, we have a little documentation but it's quite the early days and you'll probably need some help. No worries, you can chat with us on our Forum or though our Matrix channel.</p><p></p><p>Check our website for more informations and follow us on Twitter or Mastodon.</p>","endTime":"2019-12-18T14:00:00Z","ical:status":"CONFIRMED","id":"https://mobilizon.org/events/252d5816-00a3-4a89-a66f-15bf65c33e39","joinMode":"free","location":{"address":{"addressCountry":"France","addressLocality":"Nantes","addressRegion":"Pays de la Loire","postalCode":null,"streetAddress":" ","type":"PostalAddress"},"geo":{"latitude":-1.54939699141711,"longitude":47.21617415,"type":"GeoCoordinates"},"id":"https://mobilizon.org/address/1368fdab-1e2c-4de6-bcff-a90c84abdee1","name":"Cour du Château des Ducs de Bretagne","type":"Place"},"maximumAttendeeCapacity":0,"mediaType":"text/html","name":"Mobilizon Launching Party","published":"2019-12-17T11:33:56Z","repliesModerationOption":"allow_all","startTime":"2019-12-18T13:00:00Z","tag":[{"href":"https://mobilizon.org/tags/mobilizon","name":"#Mobilizon","type":"Hashtag"},{"href":"https://mobilizon.org/tags/federation","name":"#Federation","type":"Hashtag"},{"href":"https://mobilizon.org/tags/activitypub","name":"#ActivityPub","type":"Hashtag"},{"href":"https://mobilizon.org/tags/party","name":"#Party","type":"Hashtag"}],"to":["https://www.w3.org/ns/activitystreams#Public"],"type":"Event","updated":"2019-12-17T12:25:01Z","url":"https://mobilizon.org/events/252d5816-00a3-4a89-a66f-15bf65c33e39","uuid":"252d5816-00a3-4a89-a66f-15bf65c33e39"}

test/fixtures/tesla_mock/mobilizon.org-user.json

@@ -0,0 +1 @@
+{"@context":["https://www.w3.org/ns/activitystreams","https://litepub.social/litepub/context.jsonld",{"GeoCoordinates":"sc:GeoCoordinates","Hashtag":"as:Hashtag","Place":"sc:Place","PostalAddress":"sc:PostalAddress","address":{"@id":"sc:address","@type":"sc:PostalAddress"},"addressCountry":"sc:addressCountry","addressLocality":"sc:addressLocality","addressRegion":"sc:addressRegion","category":"sc:category","commentsEnabled":{"@id":"pt:commentsEnabled","@type":"sc:Boolean"},"geo":{"@id":"sc:geo","@type":"sc:GeoCoordinates"},"ical":"http://www.w3.org/2002/12/cal/ical#","joinMode":{"@id":"mz:joinMode","@type":"mz:joinModeType"},"joinModeType":{"@id":"mz:joinModeType","@type":"rdfs:Class"},"location":{"@id":"sc:location","@type":"sc:Place"},"maximumAttendeeCapacity":"sc:maximumAttendeeCapacity","mz":"https://joinmobilizon.org/ns#","postalCode":"sc:postalCode","pt":"https://joinpeertube.org/ns#","repliesModerationOption":{"@id":"mz:repliesModerationOption","@type":"mz:repliesModerationOptionType"},"repliesModerationOptionType":{"@id":"mz:repliesModerationOptionType","@type":"rdfs:Class"},"sc":"http://schema.org#","streetAddress":"sc:streetAddress","uuid":"sc:identifier"}],"endpoints":{"sharedInbox":"https://mobilizon.org/inbox"},"followers":"https://mobilizon.org/@tcit/followers","following":"https://mobilizon.org/@tcit/following","icon":{"mediaType":null,"type":"Image","url":"https://mobilizon.org/media/3a5f18c058a8193b1febfaf561f94ae8b91f85ac64c01ddf5ad7b251fb43baf5.jpg?name=profil.jpg"},"id":"https://mobilizon.org/@tcit","inbox":"https://mobilizon.org/@tcit/inbox","manuallyApprovesFollowers":false,"name":"Thomas Citharel","outbox":"https://mobilizon.org/@tcit/outbox","preferredUsername":"tcit","publicKey":{"id":"https://mobilizon.org/@tcit#main-key","owner":"https://mobilizon.org/@tcit","publicKeyPem":"-----BEGIN RSA PUBLIC KEY-----\nMIIBCgKCAQEAtzuZFviv5f12SuA0wZFMuwKS8RIlT3IjPCMLRDhiorZeV3UJ1lik\nDYO6mEh22KDXYgJtNVSYGF0Q5LJivgcvuvU+VQ048iTB1B2x0rHMr47KPByPjfVb\nKDeHt6fkHcLY0JK8UkIxW542wXAg4jX5w3gJi3pgTQrCT8VNyPbH1CaA0uW//9jc\nqzZQVFzpfdJoVOM9E3Urc/u58HC4xOptlM7+B/594ZI9drYwy5m+ZxHwlQUYCva4\n34dvwsfOGxkQyIrzXoep80EnWnFpYCLMcCiz+sEhPYxqLgNE+Cmn/6pv7SIscz6p\neVlQXIchdw+J4yl07paJDkFc6CNTCmaIHQIDAQAB\n-----END RSA PUBLIC KEY-----\n\n"},"summary":"Main profile","type":"Person","url":"https://mobilizon.org/@tcit"}

test/object/fetcher_test.exs

@@ -77,6 +77,15 @@ test "it can fetch peertube videos" do
       assert object
     end
 
+    test "it can fetch Mobilizon events" do
+      {:ok, object} =
+        Fetcher.fetch_object_from_id(
+          "https://mobilizon.org/events/252d5816-00a3-4a89-a66f-15bf65c33e39"
+        )
+
+      assert object
+    end
+
     test "it can fetch wedistribute articles" do
       {:ok, object} =
         Fetcher.fetch_object_from_id("https://wedistribute.org/wp-json/pterotype/v1/object/85810")

test/plugs/rate_limiter_test.exs

@@ -145,9 +145,9 @@ test "are restricted based on remote IP" do
     test "can have limits seperate from unauthenticated connections" do
       limiter_name = :test_authenticated
 
-      scale = 1000
+      scale = 50
       limit = 5
-      Pleroma.Config.put([:rate_limit, limiter_name], [{1, 10}, {scale, limit}])
+      Pleroma.Config.put([:rate_limit, limiter_name], [{1000, 1}, {scale, limit}])
 
       opts = RateLimiter.init(name: limiter_name)
 
@@ -164,16 +164,6 @@ test "can have limits seperate from unauthenticated connections" do
 
       assert %{"error" => "Throttled"} = Phoenix.ConnTest.json_response(conn, :too_many_requests)
       assert conn.halted
-
-      Process.sleep(1550)
-
-      conn = conn(:get, "/") |> assign(:user, user)
-      conn = RateLimiter.call(conn, opts)
-      assert {1, 4} = RateLimiter.inspect_bucket(conn, limiter_name, opts)
-
-      refute conn.status == Plug.Conn.Status.code(:too_many_requests)
-      refute conn.resp_body
-      refute conn.halted
     end
 
     test "diffrerent users are counted independently" do

test/runtime_test.exs

@@ -0,0 +1,11 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.RuntimeTest do
+  use ExUnit.Case, async: true
+
+  test "it loads custom runtime modules" do
+    assert Code.ensure_compiled?(RuntimeModule)
+  end
+end

test/support/http_request_mock.ex

@@ -308,6 +308,24 @@ def get("https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3"
      }}
   end
 
+  def get("https://mobilizon.org/events/252d5816-00a3-4a89-a66f-15bf65c33e39", _, _,
+        Accept: "application/activity+json"
+      ) do
+    {:ok,
+     %Tesla.Env{
+       status: 200,
+       body: File.read!("test/fixtures/tesla_mock/mobilizon.org-event.json")
+     }}
+  end
+
+  def get("https://mobilizon.org/@tcit", _, _, Accept: "application/activity+json") do
+    {:ok,
+     %Tesla.Env{
+       status: 200,
+       body: File.read!("test/fixtures/tesla_mock/mobilizon.org-user.json")
+     }}
+  end
+
   def get("https://baptiste.gelez.xyz/@/BaptisteGelez", _, _, _) do
     {:ok,
      %Tesla.Env{

test/web/activity_pub/activity_pub_test.exs

@@ -1623,6 +1623,44 @@ test "detects hidden follows/followers for friendica" do
       assert follow_info.following_count == 32
       assert follow_info.hide_follows == true
     end
+
+    test "doesn't crash when follower and following counters are hidden" do
+      mock(fn env ->
+        case env.url do
+          "http://localhost:4001/users/masto_hidden_counters/following" ->
+            json(%{
+              "@context" => "https://www.w3.org/ns/activitystreams",
+              "id" => "http://localhost:4001/users/masto_hidden_counters/followers"
+            })
+
+          "http://localhost:4001/users/masto_hidden_counters/following?page=1" ->
+            %Tesla.Env{status: 403, body: ""}
+
+          "http://localhost:4001/users/masto_hidden_counters/followers" ->
+            json(%{
+              "@context" => "https://www.w3.org/ns/activitystreams",
+              "id" => "http://localhost:4001/users/masto_hidden_counters/following"
+            })
+
+          "http://localhost:4001/users/masto_hidden_counters/followers?page=1" ->
+            %Tesla.Env{status: 403, body: ""}
+        end
+      end)
+
+      user =
+        insert(:user,
+          local: false,
+          follower_address: "http://localhost:4001/users/masto_hidden_counters/followers",
+          following_address: "http://localhost:4001/users/masto_hidden_counters/following"
+        )
+
+      {:ok, follow_info} = ActivityPub.fetch_follow_information_for_user(user)
+
+      assert follow_info.hide_followers == true
+      assert follow_info.follower_count == 0
+      assert follow_info.hide_follows == true
+      assert follow_info.following_count == 0
+    end
   end
 
   describe "fetch_favourites/3" do
@@ -1639,13 +1677,10 @@ test "returns a favourite activities sorted by adds to favorite" do
 
       {:ok, _, _} = CommonAPI.favorite(a4.id, user)
       {:ok, _, _} = CommonAPI.favorite(a3.id, other_user)
-      Process.sleep(1000)
       {:ok, _, _} = CommonAPI.favorite(a3.id, user)
       {:ok, _, _} = CommonAPI.favorite(a5.id, other_user)
-      Process.sleep(1000)
       {:ok, _, _} = CommonAPI.favorite(a5.id, user)
       {:ok, _, _} = CommonAPI.favorite(a4.id, other_user)
-      Process.sleep(1000)
       {:ok, _, _} = CommonAPI.favorite(a1.id, user)
       {:ok, _, _} = CommonAPI.favorite(a1.id, other_user)
       result = ActivityPub.fetch_favourites(user)

test/web/activity_pub/views/user_view_test.exs

@@ -126,7 +126,7 @@ test "sets totalItems to zero when followers are hidden" do
       {:ok, _other_user, user, _activity} = CommonAPI.follow(other_user, user)
       assert %{"totalItems" => 1} = UserView.render("followers.json", %{user: user})
       user = Map.merge(user, %{hide_followers_count: true, hide_followers: true})
-      assert %{"totalItems" => 0} = UserView.render("followers.json", %{user: user})
+      refute UserView.render("followers.json", %{user: user}) |> Map.has_key?("totalItems")
     end
 
     test "sets correct totalItems when followers are hidden but the follower counter is not" do

test/web/mastodon_api/views/status_view_test.exs

@@ -394,6 +394,21 @@ test "a peertube video" do
     assert length(represented[:media_attachments]) == 1
   end
 
+  test "a Mobilizon event" do
+    user = insert(:user)
+
+    {:ok, object} =
+      Pleroma.Object.Fetcher.fetch_object_from_id(
+        "https://mobilizon.org/events/252d5816-00a3-4a89-a66f-15bf65c33e39"
+      )
+
+    %Activity{} = activity = Activity.get_create_by_object_ap_id(object.data["id"])
+
+    represented = StatusView.render("show.json", %{for: user, activity: activity})
+
+    assert represented[:id] == to_string(activity.id)
+  end
+
   describe "build_tags/1" do
     test "it returns a a dictionary tags" do
       object_tags = [

test/web/metadata/twitter_card_test.exs

@@ -26,7 +26,32 @@ test "it renders twitter card for user info" do
            ]
   end
 
-  test "it does not render attachments if post is nsfw" do
+  test "it uses summary twittercard if post has no attachment" do
+    user = insert(:user, name: "Jimmy Hendriks", bio: "born 19 March 1994")
+    {:ok, activity} = CommonAPI.post(user, %{"status" => "HI"})
+
+    note =
+      insert(:note, %{
+        data: %{
+          "actor" => user.ap_id,
+          "tag" => [],
+          "id" => "https://pleroma.gov/objects/whatever",
+          "content" => "pleroma in a nutshell"
+        }
+      })
+
+    result = TwitterCard.build_tags(%{object: note, user: user, activity_id: activity.id})
+
+    assert [
+             {:meta, [property: "twitter:title", content: Utils.user_name_string(user)], []},
+             {:meta, [property: "twitter:description", content: "“pleroma in a nutshell”"], []},
+             {:meta, [property: "twitter:image", content: "http://localhost:4001/images/avi.png"],
+              []},
+             {:meta, [property: "twitter:card", content: "summary"], []}
+           ] == result
+  end
+
+  test "it renders avatar not attachment if post is nsfw and unfurl_nsfw is disabled" do
     Pleroma.Config.put([Pleroma.Web.Metadata, :unfurl_nsfw], false)
     user = insert(:user, name: "Jimmy Hendriks", bio: "born 19 March 1994")
     {:ok, activity} = CommonAPI.post(user, %{"status" => "HI"})
@@ -67,7 +92,7 @@ test "it does not render attachments if post is nsfw" do
              {:meta, [property: "twitter:description", content: "“pleroma in a nutshell”"], []},
              {:meta, [property: "twitter:image", content: "http://localhost:4001/images/avi.png"],
               []},
-             {:meta, [property: "twitter:card", content: "summary_large_image"], []}
+             {:meta, [property: "twitter:card", content: "summary"], []}
            ] == result
   end
 

test/web/streamer/streamer_test.exs

@@ -16,6 +16,10 @@ defmodule Pleroma.Web.StreamerTest do
   alias Pleroma.Web.Streamer.Worker
 
   @moduletag needs_streamer: true, capture_log: true
+
+  @streamer_timeout 150
+  @streamer_start_wait 10
+
   clear_config_all([:instance, :skip_thread_containment])
 
   describe "user streams" do
@@ -28,7 +32,7 @@ defmodule Pleroma.Web.StreamerTest do
     test "it sends notify to in the 'user' stream", %{user: user, notify: notify} do
       task =
         Task.async(fn ->
-          assert_receive {:text, _}, 4_000
+          assert_receive {:text, _}, @streamer_timeout
         end)
 
       Streamer.add_socket(
@@ -43,7 +47,7 @@ test "it sends notify to in the 'user' stream", %{user: user, notify: notify} do
     test "it sends notify to in the 'user:notification' stream", %{user: user, notify: notify} do
       task =
         Task.async(fn ->
-          assert_receive {:text, _}, 4_000
+          assert_receive {:text, _}, @streamer_timeout
         end)
 
       Streamer.add_socket(
@@ -61,7 +65,7 @@ test "it doesn't send notify to the 'user:notification' stream when a user is bl
       blocked = insert(:user)
       {:ok, _user_relationship} = User.block(user, blocked)
 
-      task = Task.async(fn -> refute_receive {:text, _}, 4_000 end)
+      task = Task.async(fn -> refute_receive {:text, _}, @streamer_timeout end)
 
       Streamer.add_socket(
         "user:notification",
@@ -79,7 +83,7 @@ test "it doesn't send notify to the 'user:notification' stream when a thread is
       user: user
     } do
       user2 = insert(:user)
-      task = Task.async(fn -> refute_receive {:text, _}, 4_000 end)
+      task = Task.async(fn -> refute_receive {:text, _}, @streamer_timeout end)
 
       Streamer.add_socket(
         "user:notification",
@@ -97,7 +101,7 @@ test "it doesn't send notify to the 'user:notification' stream' when a domain is
       user: user
     } do
       user2 = insert(:user, %{ap_id: "https://hecking-lewd-place.com/user/meanie"})
-      task = Task.async(fn -> refute_receive {:text, _}, 4_000 end)
+      task = Task.async(fn -> refute_receive {:text, _}, @streamer_timeout end)
 
       Streamer.add_socket(
         "user:notification",
@@ -116,7 +120,9 @@ test "it sends follow activities to the 'user:notification' stream", %{
       user: user
     } do
       user2 = insert(:user)
-      task = Task.async(fn -> assert_receive {:text, _}, 4_000 end)
+      task = Task.async(fn -> assert_receive {:text, _}, @streamer_timeout end)
+
+      Process.sleep(@streamer_start_wait)
 
       Streamer.add_socket(
         "user:notification",
@@ -137,7 +143,7 @@ test "it sends to public" do
 
     task =
       Task.async(fn ->
-        assert_receive {:text, _}, 4_000
+        assert_receive {:text, _}, @streamer_timeout
       end)
 
     fake_socket = %StreamerSocket{
@@ -164,7 +170,7 @@ test "it sends to public" do
           }
           |> Jason.encode!()
 
-        assert_receive {:text, received_event}, 4_000
+        assert_receive {:text, received_event}, @streamer_timeout
         assert received_event == expected_event
       end)
 
@@ -458,9 +464,7 @@ test "it doesn't send posts from muted threads" do
 
     {:ok, activity} = CommonAPI.add_mute(user2, activity)
 
-    task = Task.async(fn -> refute_receive {:text, _}, 4_000 end)
+    task = Task.async(fn -> refute_receive {:text, _}, @streamer_timeout end)
-
-    Process.sleep(4000)
 
     Streamer.add_socket(
       "user",
@@ -482,7 +486,7 @@ test "it sends conversation update to the 'direct' stream", %{} do
 
       task =
         Task.async(fn ->
-          assert_receive {:text, received_event}, 4_000
+          assert_receive {:text, received_event}, @streamer_timeout
 
           assert %{"event" => "conversation", "payload" => received_payload} =
                    Jason.decode!(received_event)
@@ -518,13 +522,13 @@ test "it doesn't send conversation update to the 'direct' stream when the last m
 
       task =
         Task.async(fn ->
-          assert_receive {:text, received_event}, 4_000
+          assert_receive {:text, received_event}, @streamer_timeout
           assert %{"event" => "delete", "payload" => _} = Jason.decode!(received_event)
 
-          refute_receive {:text, _}, 4_000
+          refute_receive {:text, _}, @streamer_timeout
         end)
 
-      Process.sleep(1000)
+      Process.sleep(@streamer_start_wait)
 
       Streamer.add_socket(
         "direct",
@@ -555,10 +559,10 @@ test "it sends conversation update to the 'direct' stream when a message is dele
 
       task =
         Task.async(fn ->
-          assert_receive {:text, received_event}, 4_000
+          assert_receive {:text, received_event}, @streamer_timeout
           assert %{"event" => "delete", "payload" => _} = Jason.decode!(received_event)
 
-          assert_receive {:text, received_event}, 4_000
+          assert_receive {:text, received_event}, @streamer_timeout
 
           assert %{"event" => "conversation", "payload" => received_payload} =
                    Jason.decode!(received_event)
@@ -567,7 +571,7 @@ test "it sends conversation update to the 'direct' stream when a message is dele
           assert last_status["id"] == to_string(create_activity.id)
         end)
 
-      Process.sleep(1000)
+      Process.sleep(@streamer_start_wait)
 
       Streamer.add_socket(
         "direct",

test/web/twitter_api/util_controller_test.exs

@@ -898,8 +898,6 @@ test "with credentials and valid password", %{conn: conn, user: current_user} do
         |> post("/api/pleroma/delete_account", %{"password" => "test"})
 
       assert json_response(conn, 200) == %{"status" => "success"}
-      # Wait a second for the started task to end
-      :timer.sleep(1000)
     end
   end
 end