Merge branch 'develop' into feature/reports-groups-and-multiple-state-update

This commit is contained in:
Maxim Filippov 2019-11-08 18:30:49 +09:00
commit b14bf9044f
18 changed files with 148 additions and 53 deletions

View file

@ -29,22 +29,6 @@ build:
- mix deps.get
- mix compile --force
docs-build:
stage: build
only:
- master@pleroma/pleroma
- develop@pleroma/pleroma
variables:
MIX_ENV: dev
PLEROMA_BUILD_ENV: prod
script:
- mix deps.get
- mix compile
- mix docs
artifacts:
paths:
- priv/static/doc
benchmark:
stage: benchmark
variables:

View file

@ -16,9 +16,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Add `rel="ugc"` to all links in statuses, to prevent SEO spam
- Extract RSS functionality from OStatus
- MRF (Simple Policy): Also use `:accept`/`:reject` on the actors rather than only their activities
- OStatus: Extract RSS functionality
- Deprecated `User.Info` embedded schema (fields moved to `User`)
- Store status data inside Flag activity
<details>
<summary>API Changes</summary>
- **Breaking** Admin API: `PATCH /api/pleroma/admin/users/:nickname/force_password_reset` is now `PATCH /api/pleroma/admin/users/force_password_reset` (accepts `nicknames` array in the request body)
- **Breaking:** Admin API: Return link alongside with token on password reset
- **Breaking:** `/api/pleroma/admin/users/invite_token` now uses `POST`, changed accepted params and returns full invite in json instead of only token string.
- Admin API: Return `total` when querying for reports
@ -35,6 +39,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Authentication: Added rate limit for password-authorized actions / login existence checks
- Mix task to re-count statuses for all users (`mix pleroma.count_statuses`)
- Support for `X-Forwarded-For` and similar HTTP headers which used by reverse proxies to pass a real user IP address to the backend. Must not be enabled unless your instance is behind at least one reverse proxy (such as Nginx, Apache HTTPD or Varnish Cache).
- Admin API: Add ability to fetch reports, grouped by status `GET /api/pleroma/admin/grouped_reports`
<details>
<summary>API Changes</summary>
@ -53,23 +58,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Admin API: Add `GET /api/pleroma/admin/relay` endpoint - lists all followed relays
- Pleroma API: `POST /api/v1/pleroma/conversations/read` to mark all conversations as read
- Mastodon API: Add `/api/v1/markers` for managing timeline read markers
- Admin API: Add ability to fetch reports, grouped by status `GET /api/pleroma/admin/grouped_reports`
### Changed
- **Breaking:** Elixir >=1.8 is now required (was >= 1.7)
- **Breaking:** Admin API: Return link alongside with token on password reset
- **Breaking:** Admin API: Changing report state now uses `PATCH` (it was `PUT` before) and allows updating multiple reports at once (API changed)
- Replaced [pleroma_job_queue](https://git.pleroma.social/pleroma/pleroma_job_queue) and `Pleroma.Web.Federator.RetryQueue` with [Oban](https://github.com/sorentwo/oban) (see [`docs/config.md`](docs/config.md) on migrating customized worker / retry settings)
- Introduced [quantum](https://github.com/quantum-elixir/quantum-core) job scheduler
- Admin API: Return `total` when querying for reports
- Mastodon API: Return `pleroma.direct_conversation_id` when creating a direct message (`POST /api/v1/statuses`)
- Admin API: Return link alongside with token on password reset
- MRF (Simple Policy): Also use `:accept`/`:reject` on the actors rather than only their activities
- OStatus: Extract RSS functionality
- Mastodon API: Add `pleroma.direct_conversation_id` to the status endpoint (`GET /api/v1/statuses/:id`)
- Mastodon API: Mark the direct conversation as read for the author when they send a new direct message
- Deprecated `User.Info` embedded schema (fields moved to `User`)
- Store status data inside Flag activity
- Mastodon API: Add the `recipients` parameter to `GET /api/v1/conversations`
</details>
### Fixed

View file

@ -603,7 +603,7 @@
activity_pub: nil,
activity_pub_question: 30_000
config :swarm, node_blacklist: [~r/myhtmlex_.*$/]
config :swarm, node_blacklist: [~r/myhtml_.*$/]
# Import environment specific config. This must remain at the bottom
# of this file so it overrides the configuration defined above.
import_config "#{Mix.env()}.exs"

View file

@ -381,11 +381,12 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
}
```
## `PATCH /api/pleroma/admin/users/:nickname/force_password_reset`
## `PATCH /api/pleroma/admin/users/force_password_reset`
### Force passord reset for a user with a given nickname
- Params: none
- Params:
- `nicknames`
- Response: none (code `204`)
## `GET /api/pleroma/admin/reports`

View file

@ -72,6 +72,12 @@ Has an additional field under the `pleroma` object:
- `recipients`: The list of the recipients of this Conversation. These will be addressed when replying to this conversation.
## GET `/api/v1/conversations`
Accepts additional parameters:
- `recipients`: Only return conversations with the given recipients (a list of user ids). Usage example: `GET /api/v1/conversations?recipients[]=1&recipients[]=2`
## Account Search
Behavior has changed:

View file

@ -122,9 +122,37 @@ def for_user(user, params \\ %{}) do
order_by: [desc: p.updated_at],
preload: [conversation: [:users]]
)
|> restrict_recipients(user, params)
|> Pleroma.Pagination.fetch_paginated(params)
end
def restrict_recipients(query, user, %{"recipients" => user_ids}) do
user_ids =
[user.id | user_ids]
|> Enum.uniq()
|> Enum.reduce([], fn user_id, acc ->
case FlakeId.Ecto.CompatType.dump(user_id) do
{:ok, user_id} -> [user_id | acc]
_ -> acc
end
end)
conversation_subquery =
__MODULE__
|> group_by([p], p.conversation_id)
|> having(
[p],
count(p.user_id) == ^length(user_ids) and
fragment("array_agg(?) @> ?", p.user_id, ^user_ids)
)
|> select([p], %{id: p.conversation_id})
query
|> join(:inner, [p], c in subquery(conversation_subquery), on: p.conversation_id == c.id)
end
def restrict_recipients(query, _, _), do: query
def for_user_and_conversation(user, conversation) do
from(p in __MODULE__,
where: p.user_id == ^user.id,

View file

@ -616,6 +616,17 @@ def get_log_entry_message(%ModerationLog{
"@#{actor_nickname} deleted status ##{subject_id}"
end
@spec get_log_entry_message(ModerationLog) :: String.t()
def get_log_entry_message(%ModerationLog{
data: %{
"actor" => %{"nickname" => actor_nickname},
"action" => "force_password_reset",
"subject" => subjects
}
}) do
"@#{actor_nickname} force password reset for users: #{users_to_nicknames_string(subjects)}"
end
defp nicknames_to_string(nicknames) do
nicknames
|> Enum.map(&"@#{&1}")

View file

@ -38,7 +38,8 @@ defp reinject_object(struct, data) do
data <- maybe_reinject_internal_fields(data, struct),
changeset <- Object.change(struct, %{data: data}),
changeset <- touch_changeset(changeset),
{:ok, object} <- Repo.insert_or_update(changeset) do
{:ok, object} <- Repo.insert_or_update(changeset),
{:ok, object} <- Object.set_cache(object) do
{:ok, object}
else
e ->
@ -53,7 +54,7 @@ def refetch_object(%Object{data: %{"id" => id}} = object) do
{:ok, object} <- reinject_object(object, data) do
{:ok, object}
else
{:local, true} -> object
{:local, true} -> {:ok, object}
e -> {:error, e}
end
end

View file

@ -24,7 +24,8 @@ defmodule Pleroma.Plugs.TrailingFormatPlug do
"/api/help",
"/api/externalprofile",
"/notice",
"/api/pleroma/emoji"
"/api/pleroma/emoji",
"/api/oauth_tokens"
]
def init(opts) do

View file

@ -608,10 +608,16 @@ def get_password_reset(conn, %{"nickname" => nickname}) do
end
@doc "Force password reset for a given user"
def force_password_reset(conn, %{"nickname" => nickname}) do
(%User{local: true} = user) = User.get_cached_by_nickname(nickname)
def force_password_reset(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do
users = nicknames |> Enum.map(&User.get_cached_by_nickname/1)
User.force_password_reset_async(user)
Enum.map(users, &User.force_password_reset_async/1)
ModerationLog.insert_log(%{
actor: admin,
subject: users,
action: "force_password_reset"
})
json_response(conn, :no_content, "")
end

View file

@ -12,7 +12,7 @@ defmodule Pleroma.Web.Endpoint do
plug(Pleroma.Plugs.HTTPSecurityPlug)
plug(Pleroma.Plugs.UploadedMedia)
@static_cache_control "public, no-cache"
@static_cache_control "public max-age=86400 must-revalidate"
# InstanceStatic needs to be before Plug.Static to be able to override shipped-static files
# If you're adding new paths to `only:` you'll need to configure them in InstanceStatic as well

View file

@ -171,7 +171,7 @@ defmodule Pleroma.Web.Router do
post("/users/email_invite", AdminAPIController, :email_invite)
get("/users/:nickname/password_reset", AdminAPIController, :get_password_reset)
patch("/users/:nickname/force_password_reset", AdminAPIController, :force_password_reset)
patch("/users/force_password_reset", AdminAPIController, :force_password_reset)
get("/users", AdminAPIController, :list_users)
get("/users/:nickname", AdminAPIController, :user_show)

View file

@ -63,7 +63,7 @@ def copy_nginx_config(%{path: target_path} = release) do
def application do
[
mod: {Pleroma.Application, []},
extra_applications: [:logger, :runtime_tools, :comeonin, :quack, :myhtmlex, :swarm],
extra_applications: [:logger, :runtime_tools, :comeonin, :quack, :fast_sanitize, :swarm],
included_applications: [:ex_syslogger]
]
end
@ -108,10 +108,8 @@ defp deps do
{:comeonin, "~> 4.1.1"},
{:pbkdf2_elixir, "~> 0.12.3"},
{:trailing_format_plug, "~> 0.0.7"},
{:fast_sanitize,
git: "https://git.pleroma.social/pleroma/fast_sanitize.git",
ref: "1af67547a02a104e26c99d03012383e8643bc4c2"},
{:html_entities, "~> 0.4"},
{:fast_sanitize, "~> 0.1"},
{:html_entities, "~> 0.5", override: true},
{:phoenix_html, "~> 2.10"},
{:calendar, "~> 0.17.4"},
{:cachex, "~> 3.0.2"},

View file

@ -36,7 +36,8 @@
"ex_rated": {:hex, :ex_rated, "1.3.3", "30ecbdabe91f7eaa9d37fa4e81c85ba420f371babeb9d1910adbcd79ec798d27", [:mix], [{:ex2ms, "~> 1.5", [hex: :ex2ms, repo: "hexpm", optional: false]}], "hexpm"},
"ex_syslogger": {:git, "https://github.com/slashmili/ex_syslogger.git", "f3963399047af17e038897c69e20d552e6899e1d", [tag: "1.4.0"]},
"excoveralls": {:hex, :excoveralls, "0.11.2", "0c6f2c8db7683b0caa9d490fb8125709c54580b4255ffa7ad35f3264b075a643", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"},
"fast_sanitize": {:git, "https://git.pleroma.social/pleroma/fast_sanitize.git", "1af67547a02a104e26c99d03012383e8643bc4c2", [ref: "1af67547a02a104e26c99d03012383e8643bc4c2"]},
"fast_html": {:hex, :fast_html, "0.99.0", "ea740358b15c7da6085b421b775f22d4f2c6928a28a15ebb5ad4e8a2ce00350b", [:make, :mix], [], "hexpm"},
"fast_sanitize": {:hex, :fast_sanitize, "0.1.1", "a403c3c09369e23423d3e6beb14068ad07be82741d10b293c71abac445dcc636", [:mix], [{:fast_html, "~> 0.99", [hex: :fast_html, repo: "hexpm", optional: false]}, {:plug, "~> 1.8", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"},
"flake_id": {:hex, :flake_id, "0.1.0", "7716b086d2e405d09b647121a166498a0d93d1a623bead243e1f74216079ccb3", [:mix], [{:base62, "~> 1.2", [hex: :base62, repo: "hexpm", optional: false]}, {:ecto, ">= 2.0.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"},
"floki": {:hex, :floki, "0.23.0", "956ab6dba828c96e732454809fb0bd8d43ce0979b75f34de6322e73d4c917829", [:mix], [{:html_entities, "~> 0.4.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm"},
"gen_smtp": {:hex, :gen_smtp, "0.15.0", "9f51960c17769b26833b50df0b96123605a8024738b62db747fece14eb2fbfcc", [:rebar3], [], "hexpm"},
@ -44,7 +45,7 @@
"gen_state_machine": {:hex, :gen_state_machine, "2.0.5", "9ac15ec6e66acac994cc442dcc2c6f9796cf380ec4b08267223014be1c728a95", [:mix], [], "hexpm"},
"gettext": {:hex, :gettext, "0.17.1", "8baab33482df4907b3eae22f719da492cee3981a26e649b9c2be1c0192616962", [:mix], [], "hexpm"},
"hackney": {:hex, :hackney, "1.15.2", "07e33c794f8f8964ee86cebec1a8ed88db5070e52e904b8f12209773c1036085", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.5", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"},
"html_entities": {:hex, :html_entities, "0.4.0", "f2fee876858cf6aaa9db608820a3209e45a087c5177332799592142b50e89a6b", [:mix], [], "hexpm"},
"html_entities": {:hex, :html_entities, "0.5.0", "40f5c5b9cbe23073b48a4e69c67b6c11974f623a76165e2b92d098c0e88ccb1d", [:mix], [], "hexpm"},
"html_sanitize_ex": {:hex, :html_sanitize_ex, "1.3.0", "f005ad692b717691203f940c686208aa3d8ffd9dd4bb3699240096a51fa9564e", [:mix], [{:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm"},
"http_signatures": {:git, "https://git.pleroma.social/pleroma/http_signatures.git", "293d77bb6f4a67ac8bde1428735c3b42f22cbb30", [ref: "293d77bb6f4a67ac8bde1428735c3b42f22cbb30"]},
"httpoison": {:hex, :httpoison, "1.6.1", "2ce5bf6e535cd0ab02e905ba8c276580bab80052c5c549f53ddea52d72e81f33", [:mix], [{:hackney, "~> 1.15 and >= 1.15.2", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"},

View file

@ -124,6 +124,8 @@ test "refetches if the time since the last refetch is greater than the interval"
%Object{} =
object = Object.normalize("https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d")
Object.set_cache(object)
assert Enum.at(object.data["oneOf"], 0)["replies"]["totalItems"] == 4
assert Enum.at(object.data["oneOf"], 1)["replies"]["totalItems"] == 0
@ -133,6 +135,8 @@ test "refetches if the time since the last refetch is greater than the interval"
})
updated_object = Object.get_by_id_and_maybe_refetch(object.id, interval: -1)
object_in_cache = Object.get_cached_by_ap_id(object.data["id"])
assert updated_object == object_in_cache
assert Enum.at(updated_object.data["oneOf"], 0)["replies"]["totalItems"] == 8
assert Enum.at(updated_object.data["oneOf"], 1)["replies"]["totalItems"] == 3
end
@ -141,6 +145,8 @@ test "returns the old object if refetch fails", %{mock_modified: mock_modified}
%Object{} =
object = Object.normalize("https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d")
Object.set_cache(object)
assert Enum.at(object.data["oneOf"], 0)["replies"]["totalItems"] == 4
assert Enum.at(object.data["oneOf"], 1)["replies"]["totalItems"] == 0
@ -148,6 +154,8 @@ test "returns the old object if refetch fails", %{mock_modified: mock_modified}
mock_modified.(%Tesla.Env{status: 404, body: ""})
updated_object = Object.get_by_id_and_maybe_refetch(object.id, interval: -1)
object_in_cache = Object.get_cached_by_ap_id(object.data["id"])
assert updated_object == object_in_cache
assert Enum.at(updated_object.data["oneOf"], 0)["replies"]["totalItems"] == 4
assert Enum.at(updated_object.data["oneOf"], 1)["replies"]["totalItems"] == 0
end) =~
@ -160,6 +168,8 @@ test "does not refetch if the time since the last refetch is greater than the in
%Object{} =
object = Object.normalize("https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d")
Object.set_cache(object)
assert Enum.at(object.data["oneOf"], 0)["replies"]["totalItems"] == 4
assert Enum.at(object.data["oneOf"], 1)["replies"]["totalItems"] == 0
@ -169,6 +179,8 @@ test "does not refetch if the time since the last refetch is greater than the in
})
updated_object = Object.get_by_id_and_maybe_refetch(object.id, interval: 100)
object_in_cache = Object.get_cached_by_ap_id(object.data["id"])
assert updated_object == object_in_cache
assert Enum.at(updated_object.data["oneOf"], 0)["replies"]["totalItems"] == 4
assert Enum.at(updated_object.data["oneOf"], 1)["replies"]["totalItems"] == 0
end
@ -177,6 +189,8 @@ test "preserves internal fields on refetch", %{mock_modified: mock_modified} do
%Object{} =
object = Object.normalize("https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d")
Object.set_cache(object)
assert Enum.at(object.data["oneOf"], 0)["replies"]["totalItems"] == 4
assert Enum.at(object.data["oneOf"], 1)["replies"]["totalItems"] == 0
@ -192,6 +206,8 @@ test "preserves internal fields on refetch", %{mock_modified: mock_modified} do
})
updated_object = Object.get_by_id_and_maybe_refetch(object.id, interval: -1)
object_in_cache = Object.get_cached_by_ap_id(object.data["id"])
assert updated_object == object_in_cache
assert Enum.at(updated_object.data["oneOf"], 0)["replies"]["totalItems"] == 8
assert Enum.at(updated_object.data["oneOf"], 1)["replies"]["totalItems"] == 3

View file

@ -9,7 +9,7 @@ defmodule Pleroma.Web.CacheControlTest do
test "Verify Cache-Control header on static assets", %{conn: conn} do
conn = get(conn, "/index.html")
assert Conn.get_resp_header(conn, "cache-control") == ["public, no-cache"]
assert Conn.get_resp_header(conn, "cache-control") == ["public max-age=86400 must-revalidate"]
end
test "Verify Cache-Control header on the API", %{conn: conn} do

View file

@ -2760,7 +2760,7 @@ test "sets password_reset_pending to true", %{admin: admin, user: user} do
conn =
build_conn()
|> assign(:user, admin)
|> patch("/api/pleroma/admin/users/#{user.nickname}/force_password_reset")
|> patch("/api/pleroma/admin/users/force_password_reset", %{nicknames: [user.nickname]})
assert json_response(conn, 204) == ""

View file

@ -59,6 +59,59 @@ test "returns a list of conversations", %{conn: conn} do
assert User.get_cached_by_id(user_one.id).unread_conversation_count == 0
end
test "filters conversations by recipients", %{conn: conn} do
user_one = insert(:user)
user_two = insert(:user)
user_three = insert(:user)
{:ok, direct1} =
CommonAPI.post(user_one, %{
"status" => "Hi @#{user_two.nickname}!",
"visibility" => "direct"
})
{:ok, _direct2} =
CommonAPI.post(user_one, %{
"status" => "Hi @#{user_three.nickname}!",
"visibility" => "direct"
})
{:ok, direct3} =
CommonAPI.post(user_one, %{
"status" => "Hi @#{user_two.nickname}, @#{user_three.nickname}!",
"visibility" => "direct"
})
{:ok, _direct4} =
CommonAPI.post(user_two, %{
"status" => "Hi @#{user_three.nickname}!",
"visibility" => "direct"
})
{:ok, direct5} =
CommonAPI.post(user_two, %{
"status" => "Hi @#{user_one.nickname}!",
"visibility" => "direct"
})
[conversation1, conversation2] =
conn
|> assign(:user, user_one)
|> get("/api/v1/conversations", %{"recipients" => [user_two.id]})
|> json_response(200)
assert conversation1["last_status"]["id"] == direct5.id
assert conversation2["last_status"]["id"] == direct1.id
[conversation1] =
conn
|> assign(:user, user_one)
|> get("/api/v1/conversations", %{"recipients" => [user_two.id, user_three.id]})
|> json_response(200)
assert conversation1["last_status"]["id"] == direct3.id
end
test "updates the last_status on reply", %{conn: conn} do
user_one = insert(:user)
user_two = insert(:user)