Merge branch 'develop' into admin-be

This commit is contained in:
Alexander Strizhakov 2020-01-22 11:22:31 +03:00
commit 32a643a159
No known key found for this signature in database
GPG key ID: 022896A53AEF1381
69 changed files with 226 additions and 98 deletions

View file

@ -94,6 +94,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Captcha: Support native provider
- Captcha: Enable by default
- Mastodon API: Add support for `account_id` param to filter notifications by the account
- Mastodon API: Add `emoji_reactions` property to Statuses
</details>
### Fixed

View file

@ -500,7 +500,8 @@
mailer: 10,
transmogrifier: 20,
scheduled_activities: 10,
background: 5
background: 5,
attachments_cleanup: 5
]
config :pleroma, :workers,

View file

@ -29,6 +29,7 @@ Has these additional fields under the `pleroma` object:
- `spoiler_text`: a map consisting of alternate representations of the `spoiler_text` property with the key being it's mimetype. Currently the only alternate representation supported is `text/plain`
- `expires_at`: a datetime (iso8601) that states when the post will expire (be deleted automatically), or empty if the post won't expire
- `thread_muted`: true if the thread the post belongs to is muted
- `emoji_reactions`: An object with all the emoji reactions with count. Contains no information about the reacting users, for that use the `emoji_reactions_by` endpoint.
## Attachments

View file

@ -312,9 +312,7 @@ def restrict_deactivated_users(query) do
from(u in User.Query.build(deactivated: true), select: u.ap_id)
|> Repo.all()
from(activity in query,
where: activity.actor not in ^deactivated_users
)
Activity.Queries.exclude_authors(query, deactivated_users)
end
defdelegate search(user, query, options \\ []), to: Pleroma.Activity.Search

View file

@ -12,6 +12,7 @@ defmodule Pleroma.Activity.Queries do
@type query :: Ecto.Queryable.t() | Activity.t()
alias Pleroma.Activity
alias Pleroma.User
@spec by_ap_id(query, String.t()) :: query
def by_ap_id(query \\ Activity, ap_id) do
@ -29,6 +30,11 @@ def by_actor(query \\ Activity, actor) do
)
end
@spec by_author(query, String.t()) :: query
def by_author(query \\ Activity, %User{ap_id: ap_id}) do
from(a in query, where: a.actor == ^ap_id)
end
@spec by_object_id(query, String.t() | [String.t()]) :: query
def by_object_id(query \\ Activity, object_id)
@ -72,4 +78,8 @@ def exclude_type(query \\ Activity, activity_type) do
where: fragment("(?)->>'type' != ?", activity.data, ^activity_type)
)
end
def exclude_authors(query \\ Activity, actors) do
from(activity in query, where: activity.actor not in ^actors)
end
end

View file

@ -26,18 +26,23 @@ def search(user, search_query, options \\ []) do
|> query_with(index_type, search_query)
|> maybe_restrict_local(user)
|> maybe_restrict_author(author)
|> maybe_restrict_blocked(user)
|> Pagination.fetch_paginated(%{"offset" => offset, "limit" => limit}, :offset)
|> maybe_fetch(user, search_query)
end
def maybe_restrict_author(query, %User{} = author) do
from([a, o] in query,
where: a.actor == ^author.ap_id
)
Activity.Queries.by_author(query, author)
end
def maybe_restrict_author(query, _), do: query
def maybe_restrict_blocked(query, %User{} = user) do
Activity.Queries.exclude_authors(query, User.blocked_users_ap_ids(user))
end
def maybe_restrict_blocked(query, _), do: query
defp restrict_public(q) do
from([a, o] in q,
where: fragment("?->>'type' = 'Create'", a.data),

View file

@ -19,6 +19,8 @@ defmodule Pleroma.Object do
@type t() :: %__MODULE__{}
@derive {Jason.Encoder, only: [:data]}
schema "objects" do
field(:data, :map)
@ -180,85 +182,17 @@ def swap_object_with_tombstone(object) do
def delete(%Object{data: %{"id" => id}} = object) do
with {:ok, _obj} = swap_object_with_tombstone(object),
:ok <- delete_attachments(object),
deleted_activity = Activity.delete_all_by_object_ap_id(id),
{:ok, true} <- Cachex.del(:object_cache, "object:#{id}"),
{:ok, _} <- Cachex.del(:web_resp_cache, URI.parse(id).path) do
{:ok, _} <- Cachex.del(:web_resp_cache, URI.parse(id).path),
{:ok, _} <-
Pleroma.Workers.AttachmentsCleanupWorker.enqueue("cleanup_attachments", %{
"object" => object
}) do
{:ok, object, deleted_activity}
end
end
defp delete_attachments(%{data: %{"attachment" => [_ | _] = attachments, "actor" => actor}}) do
hrefs =
Enum.flat_map(attachments, fn attachment ->
Enum.map(attachment["url"], & &1["href"])
end)
names = Enum.map(attachments, & &1["name"])
uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
# find all objects for copies of the attachments, name and actor doesn't matter here
delete_ids =
from(o in Object,
where:
fragment(
"to_jsonb(array(select jsonb_array_elements((?)#>'{url}') ->> 'href'))::jsonb \\?| (?)",
o.data,
^hrefs
)
)
|> Repo.all()
# we should delete 1 object for any given attachment, but don't delete files if
# there are more than 1 object for it
|> Enum.reduce(%{}, fn %{
id: id,
data: %{
"url" => [%{"href" => href}],
"actor" => obj_actor,
"name" => name
}
},
acc ->
Map.update(acc, href, %{id: id, count: 1}, fn val ->
case obj_actor == actor and name in names do
true ->
# set id of the actor's object that will be deleted
%{val | id: id, count: val.count + 1}
false ->
# another actor's object, just increase count to not delete file
%{val | count: val.count + 1}
end
end)
end)
|> Enum.map(fn {href, %{id: id, count: count}} ->
# only delete files that have single instance
with 1 <- count do
prefix =
case Pleroma.Config.get([Pleroma.Upload, :base_url]) do
nil -> "media"
_ -> ""
end
base_url = Pleroma.Config.get([__MODULE__, :base_url], Pleroma.Web.base_url())
file_path = String.trim_leading(href, "#{base_url}/#{prefix}")
uploader.delete_file(file_path)
end
id
end)
from(o in Object, where: o.id in ^delete_ids)
|> Repo.delete_all()
:ok
end
defp delete_attachments(%{data: _data}), do: :ok
def prune(%Object{data: %{"id" => id}} = object) do
with {:ok, object} <- Repo.delete(object),
{:ok, true} <- Cachex.del(:object_cache, "object:#{id}"),

View file

@ -43,7 +43,7 @@ defp do_search(version, %{assigns: %{user: user}} = conn, %{"q" => query} = para
result =
default_values
|> Enum.map(fn {resource, default_value} ->
if params["type"] == nil or params["type"] == resource do
if params["type"] in [nil, resource] do
{resource, fn -> resource_search(version, resource, query, options) end}
else
{resource, fn -> default_value end}

View file

@ -253,6 +253,16 @@ def render("show.json", %{activity: %{data: %{"object" => _object}} = activity}
nil
end
emoji_reactions =
with %{data: %{"reactions" => emoji_reactions}} <- object do
Enum.map(emoji_reactions, fn {emoji, users} ->
{emoji, length(users)}
end)
|> Enum.into(%{})
else
_ -> %{}
end
%{
id: to_string(activity.id),
uri: object.data["id"],
@ -293,7 +303,8 @@ def render("show.json", %{activity: %{data: %{"object" => _object}} = activity}
spoiler_text: %{"text/plain" => summary_plaintext},
expires_at: expires_at,
direct_conversation_id: direct_conversation_id,
thread_muted: thread_muted?
thread_muted: thread_muted?,
emoji_reactions: emoji_reactions
}
}
end

View file

@ -0,0 +1,88 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Workers.AttachmentsCleanupWorker do
import Ecto.Query
alias Pleroma.Object
alias Pleroma.Repo
use Pleroma.Workers.WorkerHelper, queue: "attachments_cleanup"
@impl Oban.Worker
def perform(
%{"object" => %{"data" => %{"attachment" => [_ | _] = attachments, "actor" => actor}}},
_job
) do
hrefs =
Enum.flat_map(attachments, fn attachment ->
Enum.map(attachment["url"], & &1["href"])
end)
names = Enum.map(attachments, & &1["name"])
uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
# find all objects for copies of the attachments, name and actor doesn't matter here
delete_ids =
from(o in Object,
where:
fragment(
"to_jsonb(array(select jsonb_array_elements((?)#>'{url}') ->> 'href' where jsonb_typeof((?)#>'{url}') = 'array'))::jsonb \\?| (?)",
o.data,
o.data,
^hrefs
)
)
# The query above can be time consumptive on large instances until we
# refactor how uploads are stored
|> Repo.all(timout: :infinity)
# we should delete 1 object for any given attachment, but don't delete
# files if there are more than 1 object for it
|> Enum.reduce(%{}, fn %{
id: id,
data: %{
"url" => [%{"href" => href}],
"actor" => obj_actor,
"name" => name
}
},
acc ->
Map.update(acc, href, %{id: id, count: 1}, fn val ->
case obj_actor == actor and name in names do
true ->
# set id of the actor's object that will be deleted
%{val | id: id, count: val.count + 1}
false ->
# another actor's object, just increase count to not delete file
%{val | count: val.count + 1}
end
end)
end)
|> Enum.map(fn {href, %{id: id, count: count}} ->
# only delete files that have single instance
with 1 <- count do
prefix =
case Pleroma.Config.get([Pleroma.Upload, :base_url]) do
nil -> "media"
_ -> ""
end
base_url = Pleroma.Config.get([__MODULE__, :base_url], Pleroma.Web.base_url())
file_path = String.trim_leading(href, "#{base_url}/#{prefix}")
uploader.delete_file(file_path)
end
id
end)
from(o in Object, where: o.id in ^delete_ids)
|> Repo.delete_all()
end
def perform(%{"object" => _object}, _job), do: :ok
end

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

View file

@ -1 +1 @@
<!DOCTYPE html><html><head><meta charset=utf-8><meta http-equiv=X-UA-Compatible content="IE=edge,chrome=1"><meta name=renderer content=webkit><meta name=viewport content="width=device-width,initial-scale=1,maximum-scale=1,user-scalable=no"><title>Admin FE</title><link rel="shortcut icon" href=favicon.ico><link href=chunk-elementUI.a842fb0a.css rel=stylesheet><link href=chunk-libs.57fe98a3.css rel=stylesheet><link href=app.fdd73ce4.css rel=stylesheet></head><body><div id=app></div><script type=text/javascript src=static/js/runtime.d6d1aaab.js></script><script type=text/javascript src=static/js/chunk-elementUI.fa319e7b.js></script><script type=text/javascript src=static/js/chunk-libs.35c18287.js></script><script type=text/javascript src=static/js/app.19b7049e.js></script></body></html>
<!DOCTYPE html><html><head><meta charset=utf-8><meta http-equiv=X-UA-Compatible content="IE=edge,chrome=1"><meta name=renderer content=webkit><meta name=viewport content="width=device-width,initial-scale=1,maximum-scale=1,user-scalable=no"><title>Admin FE</title><link rel="shortcut icon" href=favicon.ico><link href=chunk-elementUI.1abbc9b8.css rel=stylesheet><link href=chunk-libs.57fe98a3.css rel=stylesheet><link href=app.fdd73ce4.css rel=stylesheet></head><body><div id=app></div><script type=text/javascript src=static/js/runtime.cab03b3e.js></script><script type=text/javascript src=static/js/chunk-elementUI.2de79b84.js></script><script type=text/javascript src=static/js/chunk-libs.680db3fc.js></script><script type=text/javascript src=static/js/app.3da0f475.js></script></body></html>

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

View file

@ -4,12 +4,14 @@
defmodule Pleroma.ObjectTest do
use Pleroma.DataCase
use Oban.Testing, repo: Pleroma.Repo
import ExUnit.CaptureLog
import Pleroma.Factory
import Tesla.Mock
alias Pleroma.Activity
alias Pleroma.Object
alias Pleroma.Repo
alias Pleroma.Tests.ObanHelpers
alias Pleroma.Web.CommonAPI
setup do
@ -99,6 +101,8 @@ test "in subdirectories" do
Object.delete(note)
ObanHelpers.perform(all_enqueued(worker: Pleroma.Workers.AttachmentsCleanupWorker))
assert Object.get_by_id(attachment.id) == nil
assert {:ok, []} == File.ls("#{uploads_dir}/#{path}")
@ -133,10 +137,46 @@ test "with dedupe enabled" do
Object.delete(note)
ObanHelpers.perform(all_enqueued(worker: Pleroma.Workers.AttachmentsCleanupWorker))
assert Object.get_by_id(attachment.id) == nil
assert {:ok, files} = File.ls(uploads_dir)
refute filename in files
end
test "with objects that have legacy data.url attribute" do
Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
file = %Plug.Upload{
content_type: "image/jpg",
path: Path.absname("test/fixtures/image.jpg"),
filename: "an_image.jpg"
}
user = insert(:user)
{:ok, %Object{} = attachment} =
Pleroma.Web.ActivityPub.ActivityPub.upload(file, actor: user.ap_id)
{:ok, %Object{}} = Object.create(%{url: "https://google.com", actor: user.ap_id})
%{data: %{"attachment" => [%{"url" => [%{"href" => href}]}]}} =
note = insert(:note, %{user: user, data: %{"attachment" => [attachment.data]}})
uploads_dir = Pleroma.Config.get!([Pleroma.Uploaders.Local, :uploads])
path = href |> Path.dirname() |> Path.basename()
assert {:ok, ["an_image.jpg"]} == File.ls("#{uploads_dir}/#{path}")
Object.delete(note)
ObanHelpers.perform(all_enqueued(worker: Pleroma.Workers.AttachmentsCleanupWorker))
assert Object.get_by_id(attachment.id) == nil
assert {:ok, []} == File.ls("#{uploads_dir}/#{path}")
end
end
describe "normalizer" do

View file

@ -53,7 +53,8 @@ test "search", %{conn: conn} do
{:ok, _} = CommonAPI.post(user_two, %{"status" => "This isn't"})
results =
get(conn, "/api/v2/search", %{"q" => "2hu #private"})
conn
|> get("/api/v2/search", %{"q" => "2hu #private"})
|> json_response(200)
[account | _] = results["accounts"]
@ -73,6 +74,30 @@ test "search", %{conn: conn} do
[status] = results["statuses"]
assert status["id"] == to_string(activity.id)
end
test "excludes a blocked users from search results", %{conn: conn} do
user = insert(:user)
user_smith = insert(:user, %{nickname: "Agent", name: "I love 2hu"})
user_neo = insert(:user, %{nickname: "Agent Neo", name: "Agent"})
{:ok, act1} = CommonAPI.post(user, %{"status" => "This is about 2hu private 天子"})
{:ok, act2} = CommonAPI.post(user_smith, %{"status" => "Agent Smith"})
{:ok, act3} = CommonAPI.post(user_neo, %{"status" => "Agent Smith"})
Pleroma.User.block(user, user_smith)
results =
conn
|> assign(:user, user)
|> assign(:token, insert(:oauth_token, user: user, scopes: ["read"]))
|> get("/api/v2/search", %{"q" => "Agent"})
|> json_response(200)
status_ids = Enum.map(results["statuses"], fn g -> g["id"] end)
assert act3.id in status_ids
refute act2.id in status_ids
refute act1.id in status_ids
end
end
describe ".account_search" do
@ -146,11 +171,10 @@ test "search", %{conn: conn} do
{:ok, _} = CommonAPI.post(user_two, %{"status" => "This isn't"})
conn =
results =
conn
|> get("/api/v1/search", %{"q" => "2hu"})
assert results = json_response(conn, 200)
|> json_response(200)
[account | _] = results["accounts"]
assert account["id"] == to_string(user_three.id)
@ -168,11 +192,10 @@ test "search fetches remote statuses and prefers them over other results", %{con
"status" => "check out https://shitposter.club/notice/2827873"
})
conn =
results =
conn
|> get("/api/v1/search", %{"q" => "https://shitposter.club/notice/2827873"})
assert results = json_response(conn, 200)
|> json_response(200)
[status, %{"id" => ^activity_id}] = results["statuses"]
@ -189,11 +212,10 @@ test "search doesn't show statuses that it shouldn't", %{conn: conn} do
})
capture_log(fn ->
conn =
results =
conn
|> get("/api/v1/search", %{"q" => Object.normalize(activity).data["id"]})
assert results = json_response(conn, 200)
|> json_response(200)
[] = results["statuses"]
end)
@ -202,23 +224,23 @@ test "search doesn't show statuses that it shouldn't", %{conn: conn} do
test "search fetches remote accounts", %{conn: conn} do
user = insert(:user)
conn =
results =
conn
|> assign(:user, user)
|> assign(:token, insert(:oauth_token, user: user, scopes: ["read"]))
|> get("/api/v1/search", %{"q" => "mike@osada.macgirvin.com", "resolve" => "true"})
|> json_response(200)
assert results = json_response(conn, 200)
[account] = results["accounts"]
assert account["acct"] == "mike@osada.macgirvin.com"
end
test "search doesn't fetch remote accounts if resolve is false", %{conn: conn} do
conn =
results =
conn
|> get("/api/v1/search", %{"q" => "mike@osada.macgirvin.com", "resolve" => "false"})
|> json_response(200)
assert results = json_response(conn, 200)
assert [] == results["accounts"]
end

View file

@ -24,6 +24,22 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do
:ok
end
test "has an emoji reaction list" do
user = insert(:user)
other_user = insert(:user)
third_user = insert(:user)
{:ok, activity} = CommonAPI.post(user, %{"status" => "dae cofe??"})
{:ok, _, _} = CommonAPI.react_with_emoji(activity.id, user, "")
{:ok, _, _} = CommonAPI.react_with_emoji(activity.id, other_user, "")
{:ok, _, _} = CommonAPI.react_with_emoji(activity.id, third_user, "🍵")
activity = Repo.get(Activity, activity.id)
status = StatusView.render("show.json", activity: activity)
assert status[:pleroma][:emoji_reactions]["🍵"] == 1
assert status[:pleroma][:emoji_reactions][""] == 2
end
test "loads and returns the direct conversation id when given the `with_direct_conversation_id` option" do
user = insert(:user)
@ -172,7 +188,8 @@ test "a note activity" do
spoiler_text: %{"text/plain" => HTML.strip_tags(object_data["summary"])},
expires_at: nil,
direct_conversation_id: nil,
thread_muted: false
thread_muted: false,
emoji_reactions: %{}
}
}