Merge branch 'develop' into 'fix/hide-follows-counters'

# Conflicts:
#   CHANGELOG.md

CHANGELOG.md

@@ -27,6 +27,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Mastodon API: Handling of search timeouts (`/api/v1/search` and `/api/v2/search`)
 - Mastodon API: Embedded relationships not being properly rendered in the Account entity of Status entity
 - Mastodon API: follower/following counters not being nullified, when `hide_follows`/`hide_followers` is set
+- Mastodon API: `muted` in the Status entity, using author's account to determine if the tread was muted
 - Mastodon API: Add `account_id`, `type`, `offset`, and `limit` to search API (`/api/v1/search` and `/api/v2/search`)
 - Mastodon API, streaming: Fix filtering of notifications based on blocks/mutes/thread mutes
 - ActivityPub C2S: follower/following collection pages being inaccessible even when authentifucated if `hide_followers`/ `hide_follows` was set
@@ -73,6 +74,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Admin API: Endpoint for fetching latest user's statuses
 - Pleroma API: Add `/api/v1/pleroma/accounts/confirmation_resend?email=<email>` for resending account confirmation.
 - Relays: Added a task to list relay subscriptions.
+- Mix Tasks: `mix pleroma.database fix_likes_collections`
+- Federation: Remove `likes` from objects.
 
 ### Changed
 - Configuration: Filter.AnonymizeFilename added ability to retain file extension with custom text

docs/config.md

@@ -18,6 +18,7 @@ Note: `strip_exif` has been replaced by `Pleroma.Upload.Filter.Mogrify`.
 
 ## Pleroma.Uploaders.S3
 * `bucket`: S3 bucket name
+* `bucket_namespace`: S3 bucket namespace
 * `public_endpoint`: S3 endpoint that the user finally accesses(ex. "https://s3.dualstack.ap-northeast-1.amazonaws.com")
 * `truncated_namespace`: If you use S3 compatible service such as Digital Ocean Spaces or CDN, set folder name or "" etc.
 For example, when using CDN to S3 virtual host format, set "".

lib/mix/tasks/pleroma/database.ex

@@ -36,6 +36,10 @@ defmodule Mix.Tasks.Pleroma.Database do
   ## Remove duplicated items from following and update followers count for all users
 
       mix pleroma.database update_users_following_followers_counts
+
+  ## Fix the pre-existing "likes" collections for all objects
+
+      mix pleroma.database fix_likes_collections
   """
   def run(["remove_embedded_objects" | args]) do
     {options, [], []} =
@@ -125,4 +129,36 @@ def run(["prune_objects" | args]) do
       )
     end
   end
+
+  def run(["fix_likes_collections"]) do
+    import Ecto.Query
+
+    start_pleroma()
+
+    from(object in Object,
+      where: fragment("(?)->>'likes' is not null", object.data),
+      select: %{id: object.id, likes: fragment("(?)->>'likes'", object.data)}
+    )
+    |> Pleroma.RepoStreamer.chunk_stream(100)
+    |> Stream.each(fn objects ->
+      ids =
+        objects
+        |> Enum.filter(fn object -> object.likes |> Jason.decode!() |> is_map() end)
+        |> Enum.map(& &1.id)
+
+      Object
+      |> where([object], object.id in ^ids)
+      |> update([object],
+        set: [
+          data:
+            fragment(
+              "jsonb_set(?, '{likes}', '[]'::jsonb, true)",
+              object.data
+            )
+        ]
+      )
+      |> Repo.update_all([], timeout: :infinity)
+    end)
+    |> Stream.run()
+  end
 end

lib/pleroma/uploaders/local.ex

@@ -11,7 +11,7 @@ def get_file(_) do
 
   def put_file(upload) do
     {local_path, file} =
-      case Enum.reverse(String.split(upload.path, "/", trim: true)) do
+      case Enum.reverse(Path.split(upload.path)) do
         [file] ->
           {upload_path(), file}
 
@@ -23,7 +23,7 @@ def put_file(upload) do
 
     result_file = Path.join(local_path, file)
 
-    unless File.exists?(result_file) do
+    if not File.exists?(result_file) do
       File.cp!(upload.tempfile, result_file)
     end
 

lib/pleroma/uploaders/mdii.ex

@@ -3,6 +3,8 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Uploaders.MDII do
+  @moduledoc "Represents uploader for https://github.com/hakaba-hitoyo/minimal-digital-image-infrastructure"
+
   alias Pleroma.Config
   alias Pleroma.HTTP
 

lib/pleroma/uploaders/s3.ex

@@ -6,10 +6,12 @@ defmodule Pleroma.Uploaders.S3 do
   @behaviour Pleroma.Uploaders.Uploader
   require Logger
 
+  alias Pleroma.Config
+
   # The file name is re-encoded with S3's constraints here to comply with previous
   # links with less strict filenames
   def get_file(file) do
-    config = Pleroma.Config.get([__MODULE__])
+    config = Config.get([__MODULE__])
     bucket = Keyword.fetch!(config, :bucket)
 
     bucket_with_namespace =
@@ -34,15 +36,15 @@ def get_file(file) do
   end
 
   def put_file(%Pleroma.Upload{} = upload) do
-    config = Pleroma.Config.get([__MODULE__])
+    config = Config.get([__MODULE__])
     bucket = Keyword.get(config, :bucket)
 
-    {:ok, file_data} = File.read(upload.tempfile)
-
     s3_name = strict_encode(upload.path)
 
     op =
-      ExAws.S3.put_object(bucket, s3_name, file_data, [
+      upload.tempfile
+      |> ExAws.S3.Upload.stream_file()
+      |> ExAws.S3.upload(bucket, s3_name, [
         {:acl, :public_read},
         {:content_type, upload.content_type}
       ])

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -26,6 +26,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   """
   def fix_object(object, options \\ []) do
     object
+    |> strip_internal_fields
     |> fix_actor
     |> fix_url
     |> fix_attachments
@@ -34,7 +35,6 @@ def fix_object(object, options \\ []) do
     |> fix_emoji
     |> fix_tag
     |> fix_content_map
-    |> fix_likes
     |> fix_addressing
     |> fix_summary
     |> fix_type(options)
@@ -151,20 +151,6 @@ def fix_actor(%{"attributedTo" => actor} = object) do
     |> Map.put("actor", Containment.get_actor(%{"actor" => actor}))
   end
 
-  # Check for standardisation
-  # This is what Peertube does
-  # curl -H 'Accept: application/activity+json' $likes | jq .totalItems
-  # Prismo returns only an integer (count) as "likes"
-  def fix_likes(%{"likes" => likes} = object) when not is_map(likes) do
-    object
-    |> Map.put("likes", [])
-    |> Map.put("like_count", 0)
-  end
-
-  def fix_likes(object) do
-    object
-  end
-
   def fix_in_reply_to(object, options \\ [])
 
   def fix_in_reply_to(%{"inReplyTo" => in_reply_to} = object, options)
@@ -784,7 +770,6 @@ def prepare_object(object) do
     |> add_mention_tags
     |> add_emoji_tags
     |> add_attributed_to
-    |> add_likes
     |> prepare_attachments
     |> set_conversation
     |> set_reply_to_uri
@@ -971,22 +956,6 @@ def add_attributed_to(object) do
     |> Map.put("attributedTo", attributed_to)
   end
 
-  def add_likes(%{"id" => id, "like_count" => likes} = object) do
-    likes = %{
-      "id" => "#{id}/likes",
-      "first" => "#{id}/likes?page=1",
-      "type" => "OrderedCollection",
-      "totalItems" => likes
-    }
-
-    object
-    |> Map.put("likes", likes)
-  end
-
-  def add_likes(object) do
-    object
-  end
-
   def prepare_attachments(object) do
     attachments =
       (object["attachment"] || [])
@@ -1002,6 +971,7 @@ def prepare_attachments(object) do
   defp strip_internal_fields(object) do
     object
     |> Map.drop([
+      "likes",
       "like_count",
       "announcements",
       "announcement_count",

lib/pleroma/web/mastodon_api/views/status_view.ex

@@ -168,7 +168,7 @@ def render("status.json", %{activity: %{data: %{"object" => _object}} = activity
     thread_muted? =
       case activity.thread_muted? do
         thread_muted? when is_boolean(thread_muted?) -> thread_muted?
-        nil -> CommonAPI.thread_muted?(user, activity)
+        nil -> (opts[:for] && CommonAPI.thread_muted?(opts[:for], activity)) || false
       end
 
     attachment_data = object.data["attachment"] || []

mix.exs

@@ -114,8 +114,9 @@ defp deps do
       {:tesla, "~> 1.2"},
       {:jason, "~> 1.0"},
       {:mogrify, "~> 0.6.1"},
-      {:ex_aws, "~> 2.0"},
+      {:ex_aws, "~> 2.1"},
       {:ex_aws_s3, "~> 2.0"},
+      {:sweet_xml, "~> 0.6.6"},
       {:earmark, "~> 1.3"},
       {:bbcode, "~> 0.1.1"},
       {:ex_machina, "~> 2.3", only: :test},

mix.lock

@@ -80,6 +80,7 @@
   "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm"},
   "recon": {:git, "https://github.com/ferd/recon.git", "75d70c7c08926d2f24f1ee6de14ee50fe8a52763", [tag: "2.4.0"]},
   "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"},
+  "sweet_xml": {:hex, :sweet_xml, "0.6.6", "fc3e91ec5dd7c787b6195757fbcf0abc670cee1e4172687b45183032221b66b8", [:mix], [], "hexpm"},
   "swoosh": {:hex, :swoosh, "0.23.2", "7dda95ff0bf54a2298328d6899c74dae1223777b43563ccebebb4b5d2b61df38", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"},
   "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]},
   "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"},

test/tasks/database_test.exs

@@ -3,8 +3,11 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Mix.Tasks.Pleroma.DatabaseTest do
+  alias Pleroma.Object
   alias Pleroma.Repo
   alias Pleroma.User
+  alias Pleroma.Web.CommonAPI
+
   use Pleroma.DataCase
 
   import Pleroma.Factory
@@ -46,4 +49,37 @@ test "following and followers count are updated" do
       assert user.info.follower_count == 0
     end
   end
+
+  describe "running fix_likes_collections" do
+    test "it turns OrderedCollection likes into empty arrays" do
+      [user, user2] = insert_pair(:user)
+
+      {:ok, %{id: id, object: object}} = CommonAPI.post(user, %{"status" => "test"})
+      {:ok, %{object: object2}} = CommonAPI.post(user, %{"status" => "test test"})
+
+      CommonAPI.favorite(id, user2)
+
+      likes = %{
+        "first" =>
+          "http://mastodon.example.org/objects/dbdbc507-52c8-490d-9b7c-1e1d52e5c132/likes?page=1",
+        "id" => "http://mastodon.example.org/objects/dbdbc507-52c8-490d-9b7c-1e1d52e5c132/likes",
+        "totalItems" => 3,
+        "type" => "OrderedCollection"
+      }
+
+      new_data = Map.put(object2.data, "likes", likes)
+
+      object2
+      |> Ecto.Changeset.change(%{data: new_data})
+      |> Repo.update()
+
+      assert length(Object.get_by_id(object.id).data["likes"]) == 1
+      assert is_map(Object.get_by_id(object2.id).data["likes"])
+
+      assert :ok == Mix.Tasks.Pleroma.Database.run(["fix_likes_collections"])
+
+      assert length(Object.get_by_id(object.id).data["likes"]) == 1
+      assert Enum.empty?(Object.get_by_id(object2.id).data["likes"])
+    end
+  end
 end

test/uploaders/local_test.exs

@@ -0,0 +1,32 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Uploaders.LocalTest do
+  use Pleroma.DataCase
+  alias Pleroma.Uploaders.Local
+
+  describe "get_file/1" do
+    test "it returns path to local folder for files" do
+      assert Local.get_file("") == {:ok, {:static_dir, "test/uploads"}}
+    end
+  end
+
+  describe "put_file/1" do
+    test "put file to local folder" do
+      file_path = "local_upload/files/image.jpg"
+
+      file = %Pleroma.Upload{
+        name: "image.jpg",
+        content_type: "image/jpg",
+        path: file_path,
+        tempfile: Path.absname("test/fixtures/image_tmp.jpg")
+      }
+
+      assert Local.put_file(file) == :ok
+
+      assert Path.join([Local.upload_path(), file_path])
+             |> File.exists?()
+    end
+  end
+end

test/uploaders/mdii_test.exs

@@ -0,0 +1,50 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Uploaders.MDIITest do
+  use Pleroma.DataCase
+  alias Pleroma.Uploaders.MDII
+  import Tesla.Mock
+
+  describe "get_file/1" do
+    test "it returns path to local folder for files" do
+      assert MDII.get_file("") == {:ok, {:static_dir, "test/uploads"}}
+    end
+  end
+
+  describe "put_file/1" do
+    setup do
+      file_upload = %Pleroma.Upload{
+        name: "mdii-image.jpg",
+        content_type: "image/jpg",
+        path: "test_folder/mdii-image.jpg",
+        tempfile: Path.absname("test/fixtures/image_tmp.jpg")
+      }
+
+      [file_upload: file_upload]
+    end
+
+    test "save file", %{file_upload: file_upload} do
+      mock(fn
+        %{method: :post, url: "https://mdii.sakura.ne.jp/mdii-post.cgi?jpg"} ->
+          %Tesla.Env{status: 200, body: "mdii-image"}
+      end)
+
+      assert MDII.put_file(file_upload) ==
+               {:ok, {:url, "https://mdii.sakura.ne.jp/mdii-image.jpg"}}
+    end
+
+    test "save file to local if MDII  isn`t available", %{file_upload: file_upload} do
+      mock(fn
+        %{method: :post, url: "https://mdii.sakura.ne.jp/mdii-post.cgi?jpg"} ->
+          %Tesla.Env{status: 500}
+      end)
+
+      assert MDII.put_file(file_upload) == :ok
+
+      assert Path.join([Pleroma.Uploaders.Local.upload_path(), file_upload.path])
+             |> File.exists?()
+    end
+  end
+end

test/uploaders/s3_test.exs

@@ -0,0 +1,90 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Uploaders.S3Test do
+  use Pleroma.DataCase
+
+  alias Pleroma.Config
+  alias Pleroma.Uploaders.S3
+
+  import Mock
+  import ExUnit.CaptureLog
+
+  setup do
+    config = Config.get([Pleroma.Uploaders.S3])
+
+    Config.put([Pleroma.Uploaders.S3],
+      bucket: "test_bucket",
+      public_endpoint: "https://s3.amazonaws.com"
+    )
+
+    on_exit(fn ->
+      Config.put([Pleroma.Uploaders.S3], config)
+    end)
+
+    :ok
+  end
+
+  describe "get_file/1" do
+    test "it returns path to local folder for files" do
+      assert S3.get_file("test_image.jpg") == {
+               :ok,
+               {:url, "https://s3.amazonaws.com/test_bucket/test_image.jpg"}
+             }
+    end
+
+    test "it returns path without bucket when truncated_namespace set to ''" do
+      Config.put([Pleroma.Uploaders.S3],
+        bucket: "test_bucket",
+        public_endpoint: "https://s3.amazonaws.com",
+        truncated_namespace: ""
+      )
+
+      assert S3.get_file("test_image.jpg") == {
+               :ok,
+               {:url, "https://s3.amazonaws.com/test_image.jpg"}
+             }
+    end
+
+    test "it returns path with bucket namespace when namespace is set" do
+      Config.put([Pleroma.Uploaders.S3],
+        bucket: "test_bucket",
+        public_endpoint: "https://s3.amazonaws.com",
+        bucket_namespace: "family"
+      )
+
+      assert S3.get_file("test_image.jpg") == {
+               :ok,
+               {:url, "https://s3.amazonaws.com/family:test_bucket/test_image.jpg"}
+             }
+    end
+  end
+
+  describe "put_file/1" do
+    setup do
+      file_upload = %Pleroma.Upload{
+        name: "image-tet.jpg",
+        content_type: "image/jpg",
+        path: "test_folder/image-tet.jpg",
+        tempfile: Path.absname("test/fixtures/image_tmp.jpg")
+      }
+
+      [file_upload: file_upload]
+    end
+
+    test "save file", %{file_upload: file_upload} do
+      with_mock ExAws, request: fn _ -> {:ok, :ok} end do
+        assert S3.put_file(file_upload) == {:ok, {:file, "test_folder/image-tet.jpg"}}
+      end
+    end
+
+    test "returns error", %{file_upload: file_upload} do
+      with_mock ExAws, request: fn _ -> {:error, "S3 Upload failed"} end do
+        assert capture_log(fn ->
+                 assert S3.put_file(file_upload) == {:error, "S3 Upload failed"}
+               end) =~ "Elixir.Pleroma.Uploaders.S3: {:error, \"S3 Upload failed\"}"
+      end
+    end
+  end
+end

test/web/activity_pub/transmogrifier_test.exs

@@ -450,6 +450,27 @@ test "it ensures that address fields become lists" do
       assert !is_nil(data["cc"])
     end
 
+    test "it strips internal likes" do
+      data =
+        File.read!("test/fixtures/mastodon-post-activity.json")
+        |> Poison.decode!()
+
+      likes = %{
+        "first" =>
+          "http://mastodon.example.org/objects/dbdbc507-52c8-490d-9b7c-1e1d52e5c132/likes?page=1",
+        "id" => "http://mastodon.example.org/objects/dbdbc507-52c8-490d-9b7c-1e1d52e5c132/likes",
+        "totalItems" => 3,
+        "type" => "OrderedCollection"
+      }
+
+      object = Map.put(data["object"], "likes", likes)
+      data = Map.put(data, "object", object)
+
+      {:ok, %Activity{object: object}} = Transmogrifier.handle_incoming(data)
+
+      refute Map.has_key?(object.data, "likes")
+    end
+
     test "it works for incoming update activities" do
       data = File.read!("test/fixtures/mastodon-post-activity.json") |> Poison.decode!()
 
@@ -1061,14 +1082,7 @@ test "it strips internal fields of article" do
       assert is_nil(modified["object"]["announcements"])
       assert is_nil(modified["object"]["announcement_count"])
       assert is_nil(modified["object"]["context_id"])
-    end
+      assert is_nil(modified["object"]["likes"])
-
-    test "it adds like collection to object" do
-      activity = insert(:note_activity)
-      {:ok, modified} = Transmogrifier.prepare_outgoing(activity.data)
-
-      assert modified["object"]["likes"]["type"] == "OrderedCollection"
-      assert modified["object"]["likes"]["totalItems"] == 0
     end
 
     test "the directMessage flag is present" do

test/web/mastodon_api/mastodon_api_controller_test.exs

@@ -2901,8 +2901,10 @@ test "bookmarks" do
 
   describe "conversation muting" do
     setup do
+      post_user = insert(:user)
       user = insert(:user)
-      {:ok, activity} = CommonAPI.post(user, %{"status" => "HIE"})
+
+      {:ok, activity} = CommonAPI.post(post_user, %{"status" => "HIE"})
 
       [user: user, activity: activity]
     end