Merge branch 'feature/restart-pleroma-from-outside-application' into 'develop'

Restarting pleroma from outside application

See merge request pleroma/pleroma!2144
This commit is contained in:
rinpatch 2020-02-05 16:59:21 +00:00
commit 49e80a1537
13 changed files with 318 additions and 42 deletions

View file

@ -665,6 +665,19 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
- 404 Not Found `"Not found"` - 404 Not Found `"Not found"`
- On success: 200 OK `{}` - On success: 200 OK `{}`
## `GET /api/pleroma/admin/restart`
### Restarts pleroma application
- Params: none
- Response:
- On failure:
- 400 Bad Request `"To use this endpoint you need to enable configuration from database."`
```json
{}
```
## `GET /api/pleroma/admin/config/migrate_from_db` ## `GET /api/pleroma/admin/config/migrate_from_db`
### Run mix task pleroma.config migrate_from_db ### Run mix task pleroma.config migrate_from_db

View file

@ -6,11 +6,7 @@ config :pleroma, configurable_from_database: true
``` ```
## How it works ## How it works
Settings are stored in database and are applied in `runtime` after each change. Most of the settings take effect immediately, except some, which need instance reboot. These settings are needed in `compile time`, that's why settings are duplicated to the file. Settings are stored in database and are applied in `runtime` after each change. Most of the settings take effect immediately, except some, which need instance reboot.
File with duplicated settings is located in `config/{env}.exported_from_db.exs` if pleroma is runned from source. For prod env it will be `config/prod.exported_from_db.exs`.
For releases: `/etc/pleroma/prod.exported_from_db.secret.exs` or `PLEROMA_CONFIG_PATH/prod.exported_from_db.exs`.
## How to set it up ## How to set it up
You need to migrate your existing settings to the database. This task will migrate only added by user settings. You need to migrate your existing settings to the database. This task will migrate only added by user settings.
@ -25,7 +21,7 @@ You can do this with mix task (all config files will remain untouched):
mix pleroma.config migrate_to_db mix pleroma.config migrate_to_db
``` ```
Now you can change settings in admin interface. After each save, settings from database are duplicated to the `config/{env}.exported_from_db.exs` file. Now you can change settings in admin interface. If `reboot time` settings were changed, pleroma must be rebooted.
<span style="color:red">**ATTENTION**</span> <span style="color:red">**ATTENTION**</span>
@ -35,10 +31,19 @@ Now you can change settings in admin interface. After each save, settings from d
- all settings inside these keys: - all settings inside these keys:
- `:hackney_pools` - `:hackney_pools`
- `:chat` - `:chat`
- `Oban`
- `:rate_limit`
- `:markup`
- `:streamer`
- partially settings inside these keys: - partially settings inside these keys:
- `:seconds_valid` in `Pleroma.Captcha` - `:seconds_valid` in `Pleroma.Captcha`
- `:proxy_remote` in `Pleroma.Upload` - `:proxy_remote` in `Pleroma.Upload`
- `:upload_limit` in `:instance` - `:upload_limit` in `:instance`
- `:digest` in `:email_notifications`
- `:clean_expired_tokens` in `:oauth2`
- `:enabled` in `Pleroma.ActivityExpiration`
- `:enabled` in `Pleroma.ScheduledActivity`
- `:enabled` in `:gopher`
## How to dump settings from database to file ## How to dump settings from database to file
@ -59,7 +64,7 @@ mix pleroma.config migrate_from_db [-d]
```sql ```sql
TRUNCATE TABLE config; TRUNCATE TABLE config;
``` ```
2. Delete `config/{env}.exported_from_db.exs`. 2. If migrate_from_db task was runned, backup and delete `config/{env}.exported_from_db.exs`.
For `prod` env: For `prod` env:
```bash ```bash

View file

@ -3,8 +3,6 @@
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Config.Loader do defmodule Pleroma.Config.Loader do
@paths ["config/config.exs", "config/#{Mix.env()}.exs"]
@reject_keys [ @reject_keys [
Pleroma.Repo, Pleroma.Repo,
Pleroma.Web.Endpoint, Pleroma.Web.Endpoint,
@ -35,8 +33,8 @@ defmodule Pleroma.Config.Loader do
def load_and_merge do def load_and_merge do
all_paths = all_paths =
if Pleroma.Config.get(:release), if Pleroma.Config.get(:release),
do: @paths ++ ["config/releases.exs"], do: ["config/config.exs", "config/releases.exs"],
else: @paths else: ["config/config.exs"]
all_paths all_paths
|> Enum.map(&load(&1)) |> Enum.map(&load(&1))

View file

@ -10,6 +10,30 @@ defmodule Pleroma.Config.TransferTask do
require Logger require Logger
@type env() :: :test | :benchmark | :dev | :prod
@reboot_time_keys [
{:pleroma, :hackney_pools},
{:pleroma, :chat},
{:pleroma, Oban},
{:pleroma, :rate_limit},
{:pleroma, :markup},
{:plerome, :streamer}
]
@reboot_time_subkeys [
{:pleroma, Pleroma.Captcha, [:seconds_valid]},
{:pleroma, Pleroma.Upload, [:proxy_remote]},
{:pleroma, :instance, [:upload_limit]},
{:pleroma, :email_notifications, [:digest]},
{:pleroma, :oauth2, [:clean_expired_tokens]},
{:pleroma, Pleroma.ActivityExpiration, [:enabled]},
{:pleroma, Pleroma.ScheduledActivity, [:enabled]},
{:pleroma, :gopher, [:enabled]}
]
@reject [nil, :prometheus]
def start_link(_) do def start_link(_) do
load_and_update_env() load_and_update_env()
if Pleroma.Config.get(:env) == :test, do: Ecto.Adapters.SQL.Sandbox.checkin(Repo) if Pleroma.Config.get(:env) == :test, do: Ecto.Adapters.SQL.Sandbox.checkin(Repo)
@ -17,21 +41,34 @@ defmodule Pleroma.Config.TransferTask do
end end
@spec load_and_update_env([ConfigDB.t()]) :: :ok | false @spec load_and_update_env([ConfigDB.t()]) :: :ok | false
def load_and_update_env(deleted \\ []) do def load_and_update_env(deleted \\ [], restart_pleroma? \\ true) do
with true <- Pleroma.Config.get(:configurable_from_database), with true <- Pleroma.Config.get(:configurable_from_database),
true <- Ecto.Adapters.SQL.table_exists?(Repo, "config"), true <- Ecto.Adapters.SQL.table_exists?(Repo, "config"),
started_applications <- Application.started_applications() do started_applications <- Application.started_applications() do
# We need to restart applications for loaded settings take effect # We need to restart applications for loaded settings take effect
in_db = Repo.all(ConfigDB) in_db = Repo.all(ConfigDB)
with_deleted = in_db ++ deleted with_deleted = in_db ++ deleted
with_deleted reject_for_restart = if restart_pleroma?, do: @reject, else: [:pleroma | @reject]
|> Enum.map(&merge_and_update(&1))
|> Enum.uniq() applications =
# TODO: some problem with prometheus after restart! with_deleted
|> Enum.reject(&(&1 in [:pleroma, nil, :prometheus])) |> Enum.map(&merge_and_update(&1))
|> Enum.each(&restart(started_applications, &1)) |> Enum.uniq()
# TODO: some problem with prometheus after restart!
|> Enum.reject(&(&1 in reject_for_restart))
# to be ensured that pleroma will be restarted last
applications =
if :pleroma in applications do
List.delete(applications, :pleroma) ++ [:pleroma]
else
applications
end
Enum.each(applications, &restart(started_applications, &1, Pleroma.Config.get(:env)))
:ok :ok
end end
@ -43,12 +80,25 @@ defmodule Pleroma.Config.TransferTask do
group = ConfigDB.from_string(setting.group) group = ConfigDB.from_string(setting.group)
default = Pleroma.Config.Holder.config(group, key) default = Pleroma.Config.Holder.config(group, key)
merged_value = merge_value(setting, default, group, key) value = ConfigDB.from_binary(setting.value)
merged_value =
if Ecto.get_meta(setting, :state) == :deleted do
default
else
if can_be_merged?(default, value) do
ConfigDB.merge_group(group, key, default, value)
else
value
end
end
:ok = update_env(group, key, merged_value) :ok = update_env(group, key, merged_value)
if group != :logger do if group != :logger do
group if group != :pleroma or pleroma_need_restart?(group, key, value) do
group
end
else else
# change logger configuration in runtime, without restart # change logger configuration in runtime, without restart
if Keyword.keyword?(merged_value) and if Keyword.keyword?(merged_value) and
@ -76,22 +126,31 @@ defmodule Pleroma.Config.TransferTask do
end end
end end
defp merge_value(%{__meta__: %{state: :deleted}}, default, _group, _key), do: default @spec pleroma_need_restart?(atom(), atom(), any()) :: boolean()
def pleroma_need_restart?(group, key, value) do
group_and_key_need_reboot?(group, key) or group_and_subkey_need_reboot?(group, key, value)
end
defp merge_value(setting, default, group, key) do defp group_and_key_need_reboot?(group, key) do
value = ConfigDB.from_binary(setting.value) Enum.any?(@reboot_time_keys, fn {g, k} -> g == group and k == key end)
end
if can_be_merged?(default, value) do defp group_and_subkey_need_reboot?(group, key, value) do
ConfigDB.merge_group(group, key, default, value) Keyword.keyword?(value) and
else Enum.any?(@reboot_time_subkeys, fn {g, k, subkeys} ->
value g == group and k == key and
end Enum.any?(Keyword.keys(value), &(&1 in subkeys))
end)
end end
defp update_env(group, key, nil), do: Application.delete_env(group, key) defp update_env(group, key, nil), do: Application.delete_env(group, key)
defp update_env(group, key, value), do: Application.put_env(group, key, value) defp update_env(group, key, value), do: Application.put_env(group, key, value)
defp restart(started_applications, app) do defp restart(_, :pleroma, :test), do: Logger.warn("pleroma restarted")
defp restart(_, :pleroma, _), do: send(Restarter.Pleroma, :after_boot)
defp restart(started_applications, app, _) do
with {^app, _, _} <- List.keyfind(started_applications, app, 0), with {^app, _, _} <- List.keyfind(started_applications, app, 0),
:ok <- Application.stop(app) do :ok <- Application.stop(app) do
:ok = Application.start(app) :ok = Application.start(app)

View file

@ -890,17 +890,36 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
Ecto.get_meta(config, :state) == :deleted Ecto.get_meta(config, :state) == :deleted
end) end)
Pleroma.Config.TransferTask.load_and_update_env(deleted) Pleroma.Config.TransferTask.load_and_update_env(deleted, false)
Mix.Tasks.Pleroma.Config.run([ need_reboot? =
"migrate_from_db", Enum.any?(updated, fn config ->
"--env", group = ConfigDB.from_string(config.group)
to_string(Pleroma.Config.get(:env)) key = ConfigDB.from_string(config.key)
]) value = ConfigDB.from_binary(config.value)
Pleroma.Config.TransferTask.pleroma_need_restart?(group, key, value)
end)
response = %{configs: updated}
response =
if need_reboot?, do: Map.put(response, :need_reboot, need_reboot?), else: response
conn conn
|> put_view(ConfigView) |> put_view(ConfigView)
|> render("index.json", %{configs: updated}) |> render("index.json", response)
end
end
def restart(conn, _params) do
with :ok <- configurable_from_database(conn) do
if Pleroma.Config.get(:env) == :test do
Logger.warn("pleroma restarted")
else
send(Restarter.Pleroma, {:restart, 50})
end
json(conn, %{})
end end
end end

View file

@ -5,10 +5,16 @@
defmodule Pleroma.Web.AdminAPI.ConfigView do defmodule Pleroma.Web.AdminAPI.ConfigView do
use Pleroma.Web, :view use Pleroma.Web, :view
def render("index.json", %{configs: configs}) do def render("index.json", %{configs: configs} = params) do
%{ map = %{
configs: render_many(configs, __MODULE__, "show.json", as: :config) configs: render_many(configs, __MODULE__, "show.json", as: :config)
} }
if params[:need_reboot] do
Map.put(map, :need_reboot, true)
else
map
end
end end
def render("show.json", %{config: config}) do def render("show.json", %{config: config}) do

View file

@ -197,6 +197,7 @@ defmodule Pleroma.Web.Router do
post("/config", AdminAPIController, :config_update) post("/config", AdminAPIController, :config_update)
get("/config/descriptions", AdminAPIController, :config_descriptions) get("/config/descriptions", AdminAPIController, :config_descriptions)
get("/config/migrate_from_db", AdminAPIController, :migrate_from_db) get("/config/migrate_from_db", AdminAPIController, :migrate_from_db)
get("/restart", AdminAPIController, :restart)
get("/moderation_log", AdminAPIController, :list_log) get("/moderation_log", AdminAPIController, :list_log)

10
mix.exs
View file

@ -8,7 +8,7 @@ defmodule Pleroma.Mixfile do
elixir: "~> 1.8", elixir: "~> 1.8",
elixirc_paths: elixirc_paths(Mix.env()), elixirc_paths: elixirc_paths(Mix.env()),
compilers: [:phoenix, :gettext] ++ Mix.compilers(), compilers: [:phoenix, :gettext] ++ Mix.compilers(),
elixirc_options: [warnings_as_errors: true], elixirc_options: [warnings_as_errors: warnings_as_errors(Mix.env())],
xref: [exclude: [:eldap]], xref: [exclude: [:eldap]],
start_permanent: Mix.env() == :prod, start_permanent: Mix.env() == :prod,
aliases: aliases(), aliases: aliases(),
@ -73,6 +73,11 @@ defmodule Pleroma.Mixfile do
defp elixirc_paths(:test), do: ["lib", "test/support"] defp elixirc_paths(:test), do: ["lib", "test/support"]
defp elixirc_paths(_), do: ["lib"] defp elixirc_paths(_), do: ["lib"]
defp warnings_as_errors(:prod), do: false
# Uncomment this if you need testing configurable_from_database logic
# defp warnings_as_errors(:dev), do: false
defp warnings_as_errors(_), do: true
# Specifies OAuth dependencies. # Specifies OAuth dependencies.
defp oauth_deps do defp oauth_deps do
oauth_strategy_packages = oauth_strategy_packages =
@ -166,7 +171,8 @@ defmodule Pleroma.Mixfile do
{:captcha, {:captcha,
git: "https://git.pleroma.social/pleroma/elixir-libraries/elixir-captcha.git", git: "https://git.pleroma.social/pleroma/elixir-libraries/elixir-captcha.git",
ref: "e0f16822d578866e186a0974d65ad58cddc1e2ab"}, ref: "e0f16822d578866e186a0974d65ad58cddc1e2ab"},
{:mox, "~> 0.5", only: :test} {:mox, "~> 0.5", only: :test},
{:restarter, path: "./restarter"}
] ++ oauth_deps() ] ++ oauth_deps()
end end

28
restarter/lib/pleroma.ex Normal file
View file

@ -0,0 +1,28 @@
defmodule Restarter.Pleroma do
use GenServer
def start_link(_) do
GenServer.start_link(__MODULE__, [], name: __MODULE__)
end
def init(_), do: {:ok, %{}}
def handle_info(:after_boot, %{after_boot: true} = state), do: {:noreply, state}
def handle_info(:after_boot, state) do
restart(:pleroma)
{:noreply, Map.put(state, :after_boot, true)}
end
def handle_info({:restart, delay}, state) do
Process.sleep(delay)
restart(:pleroma)
{:noreply, state}
end
defp restart(app) do
:ok = Application.ensure_started(app)
:ok = Application.stop(app)
:ok = Application.start(app)
end
end

View file

@ -0,0 +1,8 @@
defmodule Restarter do
use Application
def start(_, _) do
opts = [strategy: :one_for_one, name: Restarter.Supervisor]
Supervisor.start_link([Restarter.Pleroma], opts)
end
end

21
restarter/mix.exs Normal file
View file

@ -0,0 +1,21 @@
defmodule Restarter.MixProject do
use Mix.Project
def project do
[
app: :restarter,
version: "0.1.0",
elixir: "~> 1.8",
start_permanent: Mix.env() == :prod,
deps: deps()
]
end
def application do
[
mod: {Restarter, []}
]
end
defp deps, do: []
end

View file

@ -5,6 +5,8 @@
defmodule Pleroma.Config.TransferTaskTest do defmodule Pleroma.Config.TransferTaskTest do
use Pleroma.DataCase use Pleroma.DataCase
import ExUnit.CaptureLog
alias Pleroma.Config.TransferTask alias Pleroma.Config.TransferTask
alias Pleroma.ConfigDB alias Pleroma.ConfigDB
@ -105,4 +107,75 @@ defmodule Pleroma.Config.TransferTaskTest do
Application.put_env(:pleroma, :assets, assets) Application.put_env(:pleroma, :assets, assets)
end) end)
end end
describe "pleroma restart" do
test "don't restart if no reboot time settings were changed" do
emoji = Application.get_env(:pleroma, :emoji)
on_exit(fn -> Application.put_env(:pleroma, :emoji, emoji) end)
ConfigDB.create(%{
group: ":pleroma",
key: ":emoji",
value: [groups: [a: 1, b: 2]]
})
refute String.contains?(
capture_log(fn -> TransferTask.start_link([]) end),
"pleroma restarted"
)
end
test "restart pleroma on reboot time key" do
chat = Application.get_env(:pleroma, :chat)
on_exit(fn -> Application.put_env(:pleroma, :chat, chat) end)
ConfigDB.create(%{
group: ":pleroma",
key: ":chat",
value: [enabled: false]
})
assert capture_log(fn -> TransferTask.start_link([]) end) =~ "pleroma restarted"
end
test "restart pleroma on reboot time subkey" do
captcha = Application.get_env(:pleroma, Pleroma.Captcha)
on_exit(fn -> Application.put_env(:pleroma, Pleroma.Captcha, captcha) end)
ConfigDB.create(%{
group: ":pleroma",
key: "Pleroma.Captcha",
value: [seconds_valid: 60]
})
assert capture_log(fn -> TransferTask.start_link([]) end) =~ "pleroma restarted"
end
test "don't restart pleroma on reboot time key and subkey if there is false flag" do
chat = Application.get_env(:pleroma, :chat)
captcha = Application.get_env(:pleroma, Pleroma.Captcha)
on_exit(fn ->
Application.put_env(:pleroma, :chat, chat)
Application.put_env(:pleroma, Pleroma.Captcha, captcha)
end)
ConfigDB.create(%{
group: ":pleroma",
key: ":chat",
value: [enabled: false]
})
ConfigDB.create(%{
group: ":pleroma",
key: "Pleroma.Captcha",
value: [seconds_valid: 60]
})
refute String.contains?(
capture_log(fn -> TransferTask.load_and_update_env([], false) end),
"pleroma restarted"
)
end
end
end end

View file

@ -2043,7 +2043,6 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
Application.delete_env(:pleroma, Pleroma.Captcha.NotReal) Application.delete_env(:pleroma, Pleroma.Captcha.NotReal)
Application.put_env(:pleroma, :http, http) Application.put_env(:pleroma, :http, http)
Application.put_env(:tesla, :adapter, Tesla.Mock) Application.put_env(:tesla, :adapter, Tesla.Mock)
:ok = File.rm("config/test.exported_from_db.secret.exs")
end) end)
end end
@ -2170,7 +2169,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
assert Application.get_env(:idna, :key5) == {"string", Pleroma.Captcha.NotReal, []} assert Application.get_env(:idna, :key5) == {"string", Pleroma.Captcha.NotReal, []}
end end
test "save config setting without key", %{conn: conn} do test "save configs setting without explicit key", %{conn: conn} do
level = Application.get_env(:quack, :level) level = Application.get_env(:quack, :level)
meta = Application.get_env(:quack, :meta) meta = Application.get_env(:quack, :meta)
webhook_url = Application.get_env(:quack, :webhook_url) webhook_url = Application.get_env(:quack, :webhook_url)
@ -2256,6 +2255,34 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
} }
end end
test "saving config which need pleroma reboot", %{conn: conn} do
chat = Pleroma.Config.get(:chat)
on_exit(fn -> Pleroma.Config.put(:chat, chat) end)
conn =
post(
conn,
"/api/pleroma/admin/config",
%{
configs: [
%{group: ":pleroma", key: ":chat", value: [%{"tuple" => [":enabled", true]}]}
]
}
)
assert json_response(conn, 200) == %{
"configs" => [
%{
"db" => [":enabled"],
"group" => ":pleroma",
"key" => ":chat",
"value" => [%{"tuple" => [":enabled", true]}]
}
],
"need_reboot" => true
}
end
test "saving config with nested merge", %{conn: conn} do test "saving config with nested merge", %{conn: conn} do
config = config =
insert(:config, key: ":key1", value: :erlang.term_to_binary(key1: 1, key2: [k1: 1, k2: 2])) insert(:config, key: ":key1", value: :erlang.term_to_binary(key1: 1, key2: [k1: 1, k2: 2]))
@ -3001,6 +3028,18 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
end end
end end
describe "GET /api/pleroma/admin/restart" do
clear_config(:configurable_from_database) do
Pleroma.Config.put(:configurable_from_database, true)
end
test "pleroma restarts", %{conn: conn} do
ExUnit.CaptureLog.capture_log(fn ->
assert conn |> get("/api/pleroma/admin/restart") |> json_response(200) == %{}
end) =~ "pleroma restarted"
end
end
describe "GET /api/pleroma/admin/users/:nickname/statuses" do describe "GET /api/pleroma/admin/users/:nickname/statuses" do
setup do setup do
user = insert(:user) user = insert(:user)