Merge branch 'feature/905-dynamic-configuration-in-sql' into 'develop'

Feature/905 dynamic configuration in sql

Closes #905

See merge request pleroma/pleroma!1195
This commit is contained in:
kaniini 2019-06-14 15:45:05 +00:00
commit 270f09dbc2
32 changed files with 940 additions and 52 deletions

View file

@ -19,6 +19,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Mix Tasks: `mix pleroma.database remove_embedded_objects` - Mix Tasks: `mix pleroma.database remove_embedded_objects`
- Mix Tasks: `mix pleroma.database update_users_following_followers_counts` - Mix Tasks: `mix pleroma.database update_users_following_followers_counts`
- Mix Tasks: `mix pleroma.user toggle_confirmed` - Mix Tasks: `mix pleroma.user toggle_confirmed`
- Mix Tasks: `mix pleroma.config migrate_to_db`
- Mix Tasks: `mix pleroma.config migrate_from_db`
- Federation: Support for `Question` and `Answer` objects - Federation: Support for `Question` and `Answer` objects
- Federation: Support for reports - Federation: Support for reports
- Configuration: `poll_limits` option - Configuration: `poll_limits` option
@ -37,6 +39,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Admin API: added filters (role, tags, email, name) for users endpoint - Admin API: added filters (role, tags, email, name) for users endpoint
- Admin API: Endpoints for managing reports - Admin API: Endpoints for managing reports
- Admin API: Endpoints for deleting and changing the scope of individual reported statuses - Admin API: Endpoints for deleting and changing the scope of individual reported statuses
- Admin API: Endpoints to view and change config settings.
- AdminFE: initial release with basic user management accessible at /pleroma/admin/ - AdminFE: initial release with basic user management accessible at /pleroma/admin/
- Mastodon API: [Scheduled statuses](https://docs.joinmastodon.org/api/rest/scheduled-statuses/) - Mastodon API: [Scheduled statuses](https://docs.joinmastodon.org/api/rest/scheduled-statuses/)
- Mastodon API: `/api/v1/notifications/destroy_multiple` (glitch-soc extension) - Mastodon API: `/api/v1/notifications/destroy_multiple` (glitch-soc extension)

View file

@ -245,7 +245,8 @@ config :pleroma, :instance,
healthcheck: false, healthcheck: false,
remote_post_retention_days: 90, remote_post_retention_days: 90,
skip_thread_containment: true, skip_thread_containment: true,
limit_to_local_content: :unauthenticated limit_to_local_content: :unauthenticated,
dynamic_configuration: false
config :pleroma, :markup, config :pleroma, :markup,
# XXX - unfortunately, inline images must be enabled by default right now, because # XXX - unfortunately, inline images must be enabled by default right now, because

View file

@ -59,3 +59,6 @@ else
"!!! RUNNING IN LOCALHOST DEV MODE! !!!\nFEDERATION WON'T WORK UNTIL YOU CONFIGURE A dev.secret.exs" "!!! RUNNING IN LOCALHOST DEV MODE! !!!\nFEDERATION WON'T WORK UNTIL YOU CONFIGURE A dev.secret.exs"
) )
end end
if File.exists?("./config/dev.migrated.secret.exs"),
do: import_config("./config/dev.migrated.secret.exs")

View file

@ -63,3 +63,6 @@ config :logger, level: :warn
# Finally import the config/prod.secret.exs # Finally import the config/prod.secret.exs
# which should be versioned separately. # which should be versioned separately.
import_config "prod.secret.exs" import_config "prod.secret.exs"
if File.exists?("./config/prod.migrated.secret.exs"),
do: import_config("./config/prod.migrated.secret.exs")

View file

@ -557,3 +557,83 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
- 403 Forbidden `{"error": "error_msg"}` - 403 Forbidden `{"error": "error_msg"}`
- 404 Not Found `"Not found"` - 404 Not Found `"Not found"`
- On success: 200 OK `{}` - On success: 200 OK `{}`
## `/api/pleroma/admin/config`
### List config settings
- Method `GET`
- Params: none
- Response:
```json
{
configs: [
{
"key": string,
"value": string or {} or []
}
]
}
```
## `/api/pleroma/admin/config`
### Update config settings
Module name can be passed as string, which starts with `Pleroma`, e.g. `"Pleroma.Upload"`.
Atom or boolean value can be passed with `:` in the beginning, e.g. `":true"`, `":upload"`.
Integer with `i:`, e.g. `"i:150"`.
Compile time settings (need instance reboot):
- all settings by this keys:
- `:hackney_pools`
- `:chat`
- `Pleroma.Web.Endpoint`
- `Pleroma.Repo`
- part settings:
- `Pleroma.Captcha` -> `:seconds_valid`
- `Pleroma.Upload` -> `:proxy_remote`
- `:instance` -> `:upload_limit`
- Method `POST`
- Params:
- `configs` => [
- `key` (string)
- `value` (string, [], {})
- `delete` = true (optional, if parameter must be deleted)
]
- Request (example):
```json
{
configs: [
{
"key": "Pleroma.Upload",
"value": {
"uploader": "Pleroma.Uploaders.Local",
"filters": ["Pleroma.Upload.Filter.Dedupe"],
"link_name": ":true",
"proxy_remote": ":false",
"proxy_opts": {
"redirect_on_failure": ":false",
"max_body_length": "i:1048576",
"http": {
"follow_redirect": ":true",
"pool": ":upload"
}
}
}
}
]
}
- Response:
```json
{
configs: [
{
"key": string,
"value": string or {} or []
}
]
}
```

View file

@ -114,6 +114,7 @@ config :pleroma, Pleroma.Emails.Mailer,
* `remote_post_retention_days`: The default amount of days to retain remote posts when pruning the database. * `remote_post_retention_days`: The default amount of days to retain remote posts when pruning the database.
* `skip_thread_containment`: Skip filter out broken threads. The default is `false`. * `skip_thread_containment`: Skip filter out broken threads. The default is `false`.
* `limit_to_local_content`: Limit unauthenticated users to search for local statutes and users only. Possible values: `:unauthenticated`, `:all` and `false`. The default is `:unauthenticated`. * `limit_to_local_content`: Limit unauthenticated users to search for local statutes and users only. Possible values: `:unauthenticated`, `:all` and `false`. The default is `:unauthenticated`.
* `dynamic_configuration`: Allow transferring configuration to DB with the subsequent customization from Admin api.
## :logger ## :logger

View file

@ -0,0 +1,68 @@
defmodule Mix.Tasks.Pleroma.Config do
use Mix.Task
alias Mix.Tasks.Pleroma.Common
alias Pleroma.Repo
alias Pleroma.Web.AdminAPI.Config
@shortdoc "Manages the location of the config"
@moduledoc """
Manages the location of the config.
## Transfers config from file to DB.
mix pleroma.config migrate_to_db
## Transfers config from DB to file.
mix pleroma.config migrate_from_db ENV
"""
def run(["migrate_to_db"]) do
Common.start_pleroma()
if Pleroma.Config.get([:instance, :dynamic_configuration]) do
Application.get_all_env(:pleroma)
|> Enum.reject(fn {k, _v} -> k in [Pleroma.Repo, :env] end)
|> Enum.each(fn {k, v} ->
key = to_string(k) |> String.replace("Elixir.", "")
{:ok, _} = Config.update_or_create(%{key: key, value: v})
Mix.shell().info("#{key} is migrated.")
end)
Mix.shell().info("Settings migrated.")
else
Mix.shell().info(
"Migration is not allowed by config. You can change this behavior in instance settings."
)
end
end
def run(["migrate_from_db", env]) do
Common.start_pleroma()
if Pleroma.Config.get([:instance, :dynamic_configuration]) do
config_path = "config/#{env}.migrated.secret.exs"
{:ok, file} = File.open(config_path, [:write])
Repo.all(Config)
|> Enum.each(fn config ->
mark = if String.starts_with?(config.key, "Pleroma."), do: ",", else: ":"
IO.write(
file,
"config :pleroma, #{config.key}#{mark} #{inspect(Config.from_binary(config.value))}\r\n"
)
{:ok, _} = Repo.delete(config)
Mix.shell().info("#{config.key} deleted from DB.")
end)
File.close(file)
System.cmd("mix", ["format", config_path])
else
Mix.shell().info(
"Migration is not allowed by config. You can change this behavior in instance settings."
)
end
end
end

View file

@ -55,15 +55,13 @@ defmodule Mix.Tasks.Pleroma.Emoji do
are extracted). are extracted).
""" """
@default_manifest Pleroma.Config.get!([:emoji, :default_manifest])
def run(["ls-packs" | args]) do def run(["ls-packs" | args]) do
Application.ensure_all_started(:hackney) Application.ensure_all_started(:hackney)
{options, [], []} = parse_global_opts(args) {options, [], []} = parse_global_opts(args)
manifest = manifest =
fetch_manifest(if options[:manifest], do: options[:manifest], else: @default_manifest) fetch_manifest(if options[:manifest], do: options[:manifest], else: default_manifest())
Enum.each(manifest, fn {name, info} -> Enum.each(manifest, fn {name, info} ->
to_print = [ to_print = [
@ -88,7 +86,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do
{options, pack_names, []} = parse_global_opts(args) {options, pack_names, []} = parse_global_opts(args)
manifest_url = if options[:manifest], do: options[:manifest], else: @default_manifest manifest_url = if options[:manifest], do: options[:manifest], else: default_manifest()
manifest = fetch_manifest(manifest_url) manifest = fetch_manifest(manifest_url)
@ -298,4 +296,6 @@ defmodule Mix.Tasks.Pleroma.Emoji do
Tesla.client(middleware) Tesla.client(middleware)
end end
defp default_manifest, do: Pleroma.Config.get!([:emoji, :default_manifest])
end end

View file

@ -30,6 +30,7 @@ defmodule Mix.Tasks.Pleroma.Instance do
- `--dbuser DBUSER` - the user (aka role) to use for the database connection - `--dbuser DBUSER` - the user (aka role) to use for the database connection
- `--dbpass DBPASS` - the password to use for the database connection - `--dbpass DBPASS` - the password to use for the database connection
- `--indexable Y/N` - Allow/disallow indexing site by search engines - `--indexable Y/N` - Allow/disallow indexing site by search engines
- `--db-configurable Y/N` - Allow/disallow configuring instance from admin part
""" """
def run(["gen" | rest]) do def run(["gen" | rest]) do
@ -48,7 +49,8 @@ defmodule Mix.Tasks.Pleroma.Instance do
dbname: :string, dbname: :string,
dbuser: :string, dbuser: :string,
dbpass: :string, dbpass: :string,
indexable: :string indexable: :string,
db_configurable: :string
], ],
aliases: [ aliases: [
o: :output, o: :output,
@ -101,6 +103,14 @@ defmodule Mix.Tasks.Pleroma.Instance do
"y" "y"
) === "y" ) === "y"
db_configurable? =
Common.get_option(
options,
:db_configurable,
"Do you want to be able to configure instance from admin part? (y/n)",
"y"
) === "y"
dbhost = dbhost =
Common.get_option(options, :dbhost, "What is the hostname of your database?", "localhost") Common.get_option(options, :dbhost, "What is the hostname of your database?", "localhost")
@ -144,7 +154,8 @@ defmodule Mix.Tasks.Pleroma.Instance do
secret: secret, secret: secret,
signing_salt: signing_salt, signing_salt: signing_salt,
web_push_public_key: Base.url_encode64(web_push_public_key, padding: false), web_push_public_key: Base.url_encode64(web_push_public_key, padding: false),
web_push_private_key: Base.url_encode64(web_push_private_key, padding: false) web_push_private_key: Base.url_encode64(web_push_private_key, padding: false),
db_configurable?: db_configurable?
) )
result_psql = result_psql =

View file

@ -16,7 +16,8 @@ config :pleroma, :instance,
notify_email: "<%= notify_email %>", notify_email: "<%= notify_email %>",
limit: 5000, limit: 5000,
registrations_open: true, registrations_open: true,
dedupe_media: false dedupe_media: false,
dynamic_configuration: <%= db_configurable? %>
config :pleroma, :media_proxy, config :pleroma, :media_proxy,
enabled: false, enabled: false,

View file

@ -31,6 +31,7 @@ defmodule Pleroma.Application do
[ [
# Start the Ecto repository # Start the Ecto repository
%{id: Pleroma.Repo, start: {Pleroma.Repo, :start_link, []}, type: :supervisor}, %{id: Pleroma.Repo, start: {Pleroma.Repo, :start_link, []}, type: :supervisor},
%{id: Pleroma.Config.TransferTask, start: {Pleroma.Config.TransferTask, :start_link, []}},
%{id: Pleroma.Emoji, start: {Pleroma.Emoji, :start_link, []}}, %{id: Pleroma.Emoji, start: {Pleroma.Emoji, :start_link, []}},
%{id: Pleroma.Captcha, start: {Pleroma.Captcha, :start_link, []}}, %{id: Pleroma.Captcha, start: {Pleroma.Captcha, :start_link, []}},
%{ %{
@ -186,7 +187,7 @@ defmodule Pleroma.Application do
else else
[] []
end ++ end ++
if Pleroma.Config.get([Pleroma.Uploader, :proxy_remote]) do if Pleroma.Config.get([Pleroma.Upload, :proxy_remote]) do
[:upload] [:upload]
else else
[] []

View file

@ -0,0 +1,41 @@
defmodule Pleroma.Config.TransferTask do
use Task
alias Pleroma.Web.AdminAPI.Config
def start_link do
load_and_update_env()
if Pleroma.Config.get(:env) == :test, do: Ecto.Adapters.SQL.Sandbox.checkin(Pleroma.Repo)
:ignore
end
def load_and_update_env do
if Pleroma.Config.get([:instance, :dynamic_configuration]) do
Pleroma.Repo.all(Config)
|> Enum.each(&update_env(&1))
end
end
defp update_env(setting) do
try do
key =
if String.starts_with?(setting.key, "Pleroma.") do
"Elixir." <> setting.key
else
setting.key
end
Application.put_env(
:pleroma,
String.to_existing_atom(key),
Config.from_binary(setting.value)
)
rescue
e ->
require Logger
Logger.warn(
"updating env causes error, key: #{inspect(setting.key)}, error: #{inspect(e)}"
)
end
end
end

View file

@ -22,7 +22,6 @@ defmodule Pleroma.Emoji do
@ets __MODULE__.Ets @ets __MODULE__.Ets
@ets_options [:ordered_set, :protected, :named_table, {:read_concurrency, true}] @ets_options [:ordered_set, :protected, :named_table, {:read_concurrency, true}]
@groups Pleroma.Config.get([:emoji, :groups])
@doc false @doc false
def start_link do def start_link do
@ -87,6 +86,8 @@ defmodule Pleroma.Emoji do
"emoji" "emoji"
) )
emoji_groups = Pleroma.Config.get([:emoji, :groups])
case File.ls(emoji_dir_path) do case File.ls(emoji_dir_path) do
{:error, :enoent} -> {:error, :enoent} ->
# The custom emoji directory doesn't exist, # The custom emoji directory doesn't exist,
@ -118,7 +119,7 @@ defmodule Pleroma.Emoji do
emojis = emojis =
Enum.flat_map( Enum.flat_map(
packs, packs,
fn pack -> load_pack(Path.join(emoji_dir_path, pack)) end fn pack -> load_pack(Path.join(emoji_dir_path, pack), emoji_groups) end
) )
true = :ets.insert(@ets, emojis) true = :ets.insert(@ets, emojis)
@ -129,9 +130,9 @@ defmodule Pleroma.Emoji do
shortcode_globs = Pleroma.Config.get([:emoji, :shortcode_globs], []) shortcode_globs = Pleroma.Config.get([:emoji, :shortcode_globs], [])
emojis = emojis =
(load_from_file("config/emoji.txt") ++ (load_from_file("config/emoji.txt", emoji_groups) ++
load_from_file("config/custom_emoji.txt") ++ load_from_file("config/custom_emoji.txt", emoji_groups) ++
load_from_globs(shortcode_globs)) load_from_globs(shortcode_globs, emoji_groups))
|> Enum.reject(fn value -> value == nil end) |> Enum.reject(fn value -> value == nil end)
true = :ets.insert(@ets, emojis) true = :ets.insert(@ets, emojis)
@ -139,13 +140,13 @@ defmodule Pleroma.Emoji do
:ok :ok
end end
defp load_pack(pack_dir) do defp load_pack(pack_dir, emoji_groups) do
pack_name = Path.basename(pack_dir) pack_name = Path.basename(pack_dir)
emoji_txt = Path.join(pack_dir, "emoji.txt") emoji_txt = Path.join(pack_dir, "emoji.txt")
if File.exists?(emoji_txt) do if File.exists?(emoji_txt) do
load_from_file(emoji_txt) load_from_file(emoji_txt, emoji_groups)
else else
Logger.info( Logger.info(
"No emoji.txt found for pack \"#{pack_name}\", assuming all .png files are emoji" "No emoji.txt found for pack \"#{pack_name}\", assuming all .png files are emoji"
@ -155,7 +156,7 @@ defmodule Pleroma.Emoji do
|> Enum.map(fn {shortcode, rel_file} -> |> Enum.map(fn {shortcode, rel_file} ->
filename = Path.join("/emoji/#{pack_name}", rel_file) filename = Path.join("/emoji/#{pack_name}", rel_file)
{shortcode, filename, [to_string(match_extra(@groups, filename))]} {shortcode, filename, [to_string(match_extra(emoji_groups, filename))]}
end) end)
end end
end end
@ -184,21 +185,21 @@ defmodule Pleroma.Emoji do
|> Enum.filter(fn f -> Path.extname(f) in exts end) |> Enum.filter(fn f -> Path.extname(f) in exts end)
end end
defp load_from_file(file) do defp load_from_file(file, emoji_groups) do
if File.exists?(file) do if File.exists?(file) do
load_from_file_stream(File.stream!(file)) load_from_file_stream(File.stream!(file), emoji_groups)
else else
[] []
end end
end end
defp load_from_file_stream(stream) do defp load_from_file_stream(stream, emoji_groups) do
stream stream
|> Stream.map(&String.trim/1) |> Stream.map(&String.trim/1)
|> Stream.map(fn line -> |> Stream.map(fn line ->
case String.split(line, ~r/,\s*/) do case String.split(line, ~r/,\s*/) do
[name, file] -> [name, file] ->
{name, file, [to_string(match_extra(@groups, file))]} {name, file, [to_string(match_extra(emoji_groups, file))]}
[name, file | tags] -> [name, file | tags] ->
{name, file, tags} {name, file, tags}
@ -210,7 +211,7 @@ defmodule Pleroma.Emoji do
|> Enum.to_list() |> Enum.to_list()
end end
defp load_from_globs(globs) do defp load_from_globs(globs, emoji_groups) do
static_path = Path.join(:code.priv_dir(:pleroma), "static") static_path = Path.join(:code.priv_dir(:pleroma), "static")
paths = paths =
@ -221,7 +222,7 @@ defmodule Pleroma.Emoji do
|> Enum.concat() |> Enum.concat()
Enum.map(paths, fn path -> Enum.map(paths, fn path ->
tag = match_extra(@groups, Path.join("/", Path.relative_to(path, static_path))) tag = match_extra(emoji_groups, Path.join("/", Path.relative_to(path, static_path)))
shortcode = Path.basename(path, Path.extname(path)) shortcode = Path.basename(path, Path.extname(path))
external_path = Path.join("/", Path.relative_to(path, static_path)) external_path = Path.join("/", Path.relative_to(path, static_path))
{shortcode, external_path, [to_string(tag)]} {shortcode, external_path, [to_string(tag)]}

View file

@ -13,7 +13,7 @@ defmodule Pleroma.Instances do
def reachability_datetime_threshold do def reachability_datetime_threshold do
federation_reachability_timeout_days = federation_reachability_timeout_days =
Pleroma.Config.get(:instance)[:federation_reachability_timeout_days] || 0 Pleroma.Config.get([:instance, :federation_reachability_timeout_days], 0)
if federation_reachability_timeout_days > 0 do if federation_reachability_timeout_days > 0 do
NaiveDateTime.add( NaiveDateTime.add(

View file

@ -36,7 +36,7 @@ defmodule Pleroma.Plugs.UploadedMedia do
conn conn
end end
config = Pleroma.Config.get([Pleroma.Upload]) config = Pleroma.Config.get(Pleroma.Upload)
with uploader <- Keyword.fetch!(config, :uploader), with uploader <- Keyword.fetch!(config, :uploader),
proxy_remote = Keyword.get(config, :proxy_remote, false), proxy_remote = Keyword.get(config, :proxy_remote, false),

View file

@ -146,7 +146,7 @@ defmodule Pleroma.ReverseProxy do
Logger.debug("#{__MODULE__} #{method} #{url} #{inspect(headers)}") Logger.debug("#{__MODULE__} #{method} #{url} #{inspect(headers)}")
method = method |> String.downcase() |> String.to_existing_atom() method = method |> String.downcase() |> String.to_existing_atom()
case :hackney.request(method, url, headers, "", hackney_opts) do case hackney().request(method, url, headers, "", hackney_opts) do
{:ok, code, headers, client} when code in @valid_resp_codes -> {:ok, code, headers, client} when code in @valid_resp_codes ->
{:ok, code, downcase_headers(headers), client} {:ok, code, downcase_headers(headers), client}
@ -196,7 +196,7 @@ defmodule Pleroma.ReverseProxy do
duration, duration,
Keyword.get(opts, :max_read_duration, @max_read_duration) Keyword.get(opts, :max_read_duration, @max_read_duration)
), ),
{:ok, data} <- :hackney.stream_body(client), {:ok, data} <- hackney().stream_body(client),
{:ok, duration} <- increase_read_duration(duration), {:ok, duration} <- increase_read_duration(duration),
sent_so_far = sent_so_far + byte_size(data), sent_so_far = sent_so_far + byte_size(data),
:ok <- body_size_constraint(sent_so_far, Keyword.get(opts, :max_body_size)), :ok <- body_size_constraint(sent_so_far, Keyword.get(opts, :max_body_size)),
@ -377,4 +377,6 @@ defmodule Pleroma.ReverseProxy do
defp increase_read_duration(_) do defp increase_read_duration(_) do
{:ok, :no_duration_limit, :no_duration_limit} {:ok, :no_duration_limit, :no_duration_limit}
end end
defp hackney, do: Pleroma.Config.get(:hackney, :hackney)
end end

View file

@ -1036,9 +1036,7 @@ defmodule Pleroma.User do
Pleroma.HTML.Scrubber.TwitterText Pleroma.HTML.Scrubber.TwitterText
end end
@default_scrubbers Pleroma.Config.get([:markup, :scrub_policy]) def html_filter_policy(_), do: Pleroma.Config.get([:markup, :scrub_policy])
def html_filter_policy(_), do: @default_scrubbers
def fetch_by_ap_id(ap_id) do def fetch_by_ap_id(ap_id) do
ap_try = ActivityPub.make_user_from_ap_id(ap_id) ap_try = ActivityPub.make_user_from_ap_id(ap_id)

View file

@ -88,7 +88,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
true true
else else
inbox_info = URI.parse(inbox) inbox_info = URI.parse(inbox)
!Enum.member?(Pleroma.Config.get([:instance, :quarantined_instances], []), inbox_info.host) !Enum.member?(Config.get([:instance, :quarantined_instances], []), inbox_info.host)
end end
end end

View file

@ -10,6 +10,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.ActivityPub
alias Pleroma.Web.ActivityPub.Relay alias Pleroma.Web.ActivityPub.Relay
alias Pleroma.Web.AdminAPI.AccountView alias Pleroma.Web.AdminAPI.AccountView
alias Pleroma.Web.AdminAPI.Config
alias Pleroma.Web.AdminAPI.ConfigView
alias Pleroma.Web.AdminAPI.ReportView alias Pleroma.Web.AdminAPI.ReportView
alias Pleroma.Web.AdminAPI.Search alias Pleroma.Web.AdminAPI.Search
alias Pleroma.Web.CommonAPI alias Pleroma.Web.CommonAPI
@ -362,6 +364,41 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
end end
end end
def config_show(conn, _params) do
configs = Pleroma.Repo.all(Config)
conn
|> put_view(ConfigView)
|> render("index.json", %{configs: configs})
end
def config_update(conn, %{"configs" => configs}) do
updated =
if Pleroma.Config.get([:instance, :dynamic_configuration]) do
updated =
Enum.map(configs, fn
%{"key" => key, "value" => value} ->
{:ok, config} = Config.update_or_create(%{key: key, value: value})
config
%{"key" => key, "delete" => "true"} ->
{:ok, _} = Config.delete(key)
nil
end)
|> Enum.reject(&is_nil(&1))
Pleroma.Config.TransferTask.load_and_update_env()
Mix.Tasks.Pleroma.Config.run(["migrate_from_db", Pleroma.Config.get(:env)])
updated
else
[]
end
conn
|> put_view(ConfigView)
|> render("index.json", %{configs: updated})
end
def errors(conn, {:error, :not_found}) do def errors(conn, {:error, :not_found}) do
conn conn
|> put_status(404) |> put_status(404)

View file

@ -0,0 +1,144 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.AdminAPI.Config do
use Ecto.Schema
import Ecto.Changeset
alias __MODULE__
alias Pleroma.Repo
@type t :: %__MODULE__{}
schema "config" do
field(:key, :string)
field(:value, :binary)
timestamps()
end
@spec get_by_key(String.t()) :: Config.t() | nil
def get_by_key(key), do: Repo.get_by(Config, key: key)
@spec changeset(Config.t(), map()) :: Changeset.t()
def changeset(config, params \\ %{}) do
config
|> cast(params, [:key, :value])
|> validate_required([:key, :value])
|> unique_constraint(:key)
end
@spec create(map()) :: {:ok, Config.t()} | {:error, Changeset.t()}
def create(%{key: key, value: value}) do
%Config{}
|> changeset(%{key: key, value: transform(value)})
|> Repo.insert()
end
@spec update(Config.t(), map()) :: {:ok, Config} | {:error, Changeset.t()}
def update(%Config{} = config, %{value: value}) do
config
|> change(value: transform(value))
|> Repo.update()
end
@spec update_or_create(map()) :: {:ok, Config.t()} | {:error, Changeset.t()}
def update_or_create(%{key: key} = params) do
with %Config{} = config <- Config.get_by_key(key) do
Config.update(config, params)
else
nil -> Config.create(params)
end
end
@spec delete(String.t()) :: {:ok, Config.t()} | {:error, Changeset.t()}
def delete(key) do
with %Config{} = config <- Config.get_by_key(key) do
Repo.delete(config)
else
nil -> {:error, "Config with key #{key} not found"}
end
end
@spec from_binary(binary()) :: term()
def from_binary(value), do: :erlang.binary_to_term(value)
@spec from_binary_to_map(binary()) :: any()
def from_binary_to_map(binary) do
from_binary(binary)
|> do_convert()
end
defp do_convert([{k, v}] = value) when is_list(value) and length(value) == 1,
do: %{k => do_convert(v)}
defp do_convert(values) when is_list(values), do: for(val <- values, do: do_convert(val))
defp do_convert({k, v} = value) when is_tuple(value),
do: %{k => do_convert(v)}
defp do_convert(value) when is_binary(value) or is_atom(value) or is_map(value),
do: value
@spec transform(any()) :: binary()
def transform(entity) when is_map(entity) do
tuples =
for {k, v} <- entity,
into: [],
do: {if(is_atom(k), do: k, else: String.to_atom(k)), do_transform(v)}
Enum.reject(tuples, fn {_k, v} -> is_nil(v) end)
|> Enum.sort()
|> :erlang.term_to_binary()
end
def transform(entity) when is_list(entity) do
list = Enum.map(entity, &do_transform(&1))
:erlang.term_to_binary(list)
end
def transform(entity), do: :erlang.term_to_binary(entity)
defp do_transform(%Regex{} = value) when is_map(value), do: value
defp do_transform(value) when is_map(value) do
values =
for {key, val} <- value,
into: [],
do: {String.to_atom(key), do_transform(val)}
Enum.sort(values)
end
defp do_transform(value) when is_list(value) do
Enum.map(value, &do_transform(&1))
end
defp do_transform(entity) when is_list(entity) and length(entity) == 1, do: hd(entity)
defp do_transform(value) when is_binary(value) do
value = String.trim(value)
case String.length(value) do
0 ->
nil
_ ->
cond do
String.starts_with?(value, "Pleroma") ->
String.to_existing_atom("Elixir." <> value)
String.starts_with?(value, ":") ->
String.replace(value, ":", "") |> String.to_existing_atom()
String.starts_with?(value, "i:") ->
String.replace(value, "i:", "") |> String.to_integer()
true ->
value
end
end
end
defp do_transform(value), do: value
end

View file

@ -0,0 +1,16 @@
defmodule Pleroma.Web.AdminAPI.ConfigView do
use Pleroma.Web, :view
def render("index.json", %{configs: configs}) do
%{
configs: render_many(configs, __MODULE__, "show.json", as: :config)
}
end
def render("show.json", %{config: config}) do
%{
key: config.key,
value: Pleroma.Web.AdminAPI.Config.from_binary_to_map(config.value)
}
end
end

View file

@ -91,7 +91,7 @@ defmodule Pleroma.Web.Endpoint do
Plug.Session, Plug.Session,
store: :cookie, store: :cookie,
key: cookie_name, key: cookie_name,
signing_salt: {Pleroma.Config, :get, [[__MODULE__, :signing_salt], "CqaoopA2"]}, signing_salt: Pleroma.Config.get([__MODULE__, :signing_salt], "CqaoopA2"),
http_only: true, http_only: true,
secure: secure_cookies, secure: secure_cookies,
extra: extra extra: extra

View file

@ -14,7 +14,6 @@ defmodule Pleroma.Web.OAuth.Token do
alias Pleroma.Web.OAuth.Token alias Pleroma.Web.OAuth.Token
alias Pleroma.Web.OAuth.Token.Query alias Pleroma.Web.OAuth.Token.Query
@expires_in Pleroma.Config.get([:oauth2, :token_expires_in], 600)
@type t :: %__MODULE__{} @type t :: %__MODULE__{}
schema "oauth_tokens" do schema "oauth_tokens" do
@ -78,7 +77,7 @@ defmodule Pleroma.Web.OAuth.Token do
defp put_valid_until(changeset, attrs) do defp put_valid_until(changeset, attrs) do
expires_in = expires_in =
Map.get(attrs, :valid_until, NaiveDateTime.add(NaiveDateTime.utc_now(), @expires_in)) Map.get(attrs, :valid_until, NaiveDateTime.add(NaiveDateTime.utc_now(), expires_in()))
changeset changeset
|> change(%{valid_until: expires_in}) |> change(%{valid_until: expires_in})
@ -123,4 +122,6 @@ defmodule Pleroma.Web.OAuth.Token do
end end
def is_expired?(_), do: false def is_expired?(_), do: false
defp expires_in, do: Pleroma.Config.get([:oauth2, :token_expires_in], 600)
end end

View file

@ -4,15 +4,13 @@ defmodule Pleroma.Web.OAuth.Token.Response do
alias Pleroma.User alias Pleroma.User
alias Pleroma.Web.OAuth.Token.Utils alias Pleroma.Web.OAuth.Token.Utils
@expires_in Pleroma.Config.get([:oauth2, :token_expires_in], 600)
@doc false @doc false
def build(%User{} = user, token, opts \\ %{}) do def build(%User{} = user, token, opts \\ %{}) do
%{ %{
token_type: "Bearer", token_type: "Bearer",
access_token: token.token, access_token: token.token,
refresh_token: token.refresh_token, refresh_token: token.refresh_token,
expires_in: @expires_in, expires_in: expires_in(),
scope: Enum.join(token.scopes, " "), scope: Enum.join(token.scopes, " "),
me: user.ap_id me: user.ap_id
} }
@ -25,8 +23,10 @@ defmodule Pleroma.Web.OAuth.Token.Response do
access_token: token.token, access_token: token.token,
refresh_token: token.refresh_token, refresh_token: token.refresh_token,
created_at: Utils.format_created_at(token), created_at: Utils.format_created_at(token),
expires_in: @expires_in, expires_in: expires_in(),
scope: Enum.join(token.scopes, " ") scope: Enum.join(token.scopes, " ")
} }
end end
defp expires_in, do: Pleroma.Config.get([:oauth2, :token_expires_in], 600)
end end

View file

@ -202,6 +202,9 @@ defmodule Pleroma.Web.Router do
put("/statuses/:id", AdminAPIController, :status_update) put("/statuses/:id", AdminAPIController, :status_update)
delete("/statuses/:id", AdminAPIController, :status_delete) delete("/statuses/:id", AdminAPIController, :status_delete)
get("/config", AdminAPIController, :config_show)
post("/config", AdminAPIController, :config_update)
end end
scope "/", Pleroma.Web.TwitterAPI do scope "/", Pleroma.Web.TwitterAPI do

View file

@ -0,0 +1,13 @@
defmodule Pleroma.Repo.Migrations.CreateConfig do
use Ecto.Migration
def change do
create table(:config) do
add(:key, :string)
add(:value, :binary)
timestamps()
end
create(unique_index(:config, :key))
end
end

View file

@ -0,0 +1,35 @@
defmodule Pleroma.Config.TransferTaskTest do
use Pleroma.DataCase
setup do
dynamic = Pleroma.Config.get([:instance, :dynamic_configuration])
Pleroma.Config.put([:instance, :dynamic_configuration], true)
on_exit(fn ->
Pleroma.Config.put([:instance, :dynamic_configuration], dynamic)
end)
end
test "transfer config values from db to env" do
refute Application.get_env(:pleroma, :test_key)
Pleroma.Web.AdminAPI.Config.create(%{key: "test_key", value: [live: 2, com: 3]})
Pleroma.Config.TransferTask.start_link()
assert Application.get_env(:pleroma, :test_key) == [live: 2, com: 3]
on_exit(fn ->
Application.delete_env(:pleroma, :test_key)
end)
end
test "non existing atom" do
Pleroma.Web.AdminAPI.Config.create(%{key: "undefined_atom_key", value: [live: 2, com: 3]})
assert ExUnit.CaptureLog.capture_log(fn ->
Pleroma.Config.TransferTask.start_link()
end) =~
"updating env causes error, key: \"undefined_atom_key\", error: %ArgumentError{message: \"argument error\"}"
end
end

View file

@ -310,4 +310,17 @@ defmodule Pleroma.Factory do
} }
} }
end end
def config_factory do
%Pleroma.Web.AdminAPI.Config{
key: sequence(:key, &"some_key_#{&1}"),
value:
sequence(
:value,
fn key ->
:erlang.term_to_binary(%{another_key: "#{key}somevalue", another: "#{key}somevalue"})
end
)
}
end
end end

View file

@ -0,0 +1,54 @@
defmodule Mix.Tasks.Pleroma.ConfigTest do
use Pleroma.DataCase
alias Pleroma.Repo
alias Pleroma.Web.AdminAPI.Config
setup_all do
Mix.shell(Mix.Shell.Process)
temp_file = "config/temp.migrated.secret.exs"
dynamic = Pleroma.Config.get([:instance, :dynamic_configuration])
Pleroma.Config.put([:instance, :dynamic_configuration], true)
on_exit(fn ->
Mix.shell(Mix.Shell.IO)
Application.delete_env(:pleroma, :first_setting)
Application.delete_env(:pleroma, :second_setting)
Pleroma.Config.put([:instance, :dynamic_configuration], dynamic)
:ok = File.rm(temp_file)
end)
{:ok, temp_file: temp_file}
end
test "settings are migrated to db" do
assert Repo.all(Config) == []
Application.put_env(:pleroma, :first_setting, key: "value", key2: [Pleroma.Repo])
Application.put_env(:pleroma, :second_setting, key: "value2", key2: [Pleroma.Activity])
Mix.Tasks.Pleroma.Config.run(["migrate_to_db"])
first_db = Config.get_by_key("first_setting")
second_db = Config.get_by_key("second_setting")
refute Config.get_by_key("Pleroma.Repo")
assert Config.from_binary(first_db.value) == [key: "value", key2: [Pleroma.Repo]]
assert Config.from_binary(second_db.value) == [key: "value2", key2: [Pleroma.Activity]]
end
test "settings are migrated to file and deleted from db", %{temp_file: temp_file} do
Config.create(%{key: "setting_first", value: [key: "value", key2: [Pleroma.Activity]]})
Config.create(%{key: "setting_second", value: [key: "valu2", key2: [Pleroma.Repo]]})
Mix.Tasks.Pleroma.Config.run(["migrate_from_db", "temp"])
assert Repo.all(Config) == []
assert File.exists?(temp_file)
{:ok, file} = File.read(temp_file)
assert file =~ "config :pleroma, setting_first:"
assert file =~ "config :pleroma, setting_second:"
end
end

View file

@ -36,6 +36,8 @@ defmodule Pleroma.InstanceTest do
"--dbpass", "--dbpass",
"dbpass", "dbpass",
"--indexable", "--indexable",
"y",
"--db-configurable",
"y" "y"
]) ])
end end
@ -53,6 +55,7 @@ defmodule Pleroma.InstanceTest do
assert generated_config =~ "database: \"dbname\"" assert generated_config =~ "database: \"dbname\""
assert generated_config =~ "username: \"dbuser\"" assert generated_config =~ "username: \"dbuser\""
assert generated_config =~ "password: \"dbpass\"" assert generated_config =~ "password: \"dbpass\""
assert generated_config =~ "dynamic_configuration: true"
assert File.read!(tmp_path() <> "setup.psql") == generated_setup_psql() assert File.read!(tmp_path() <> "setup.psql") == generated_setup_psql()
end end

View file

@ -1292,4 +1292,176 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
assert json_response(conn, :bad_request) == "Could not delete" assert json_response(conn, :bad_request) == "Could not delete"
end end
end end
describe "GET /api/pleroma/admin/config" do
setup %{conn: conn} do
admin = insert(:user, info: %{is_admin: true})
%{conn: assign(conn, :user, admin)}
end
test "without any settings in db", %{conn: conn} do
conn = get(conn, "/api/pleroma/admin/config")
assert json_response(conn, 200) == %{"configs" => []}
end
test "with settings in db", %{conn: conn} do
config1 = insert(:config)
config2 = insert(:config)
conn = get(conn, "/api/pleroma/admin/config")
%{
"configs" => [
%{
"key" => key1,
"value" => _
},
%{
"key" => key2,
"value" => _
}
]
} = json_response(conn, 200)
assert key1 == config1.key
assert key2 == config2.key
end
end
describe "POST /api/pleroma/admin/config" do
setup %{conn: conn} do
admin = insert(:user, info: %{is_admin: true})
temp_file = "config/test.migrated.secret.exs"
on_exit(fn ->
Application.delete_env(:pleroma, :key1)
Application.delete_env(:pleroma, :key2)
Application.delete_env(:pleroma, :key3)
Application.delete_env(:pleroma, :key4)
Application.delete_env(:pleroma, :keyaa1)
Application.delete_env(:pleroma, :keyaa2)
:ok = File.rm(temp_file)
end)
dynamic = Pleroma.Config.get([:instance, :dynamic_configuration])
Pleroma.Config.put([:instance, :dynamic_configuration], true)
on_exit(fn ->
Pleroma.Config.put([:instance, :dynamic_configuration], dynamic)
end)
%{conn: assign(conn, :user, admin)}
end
test "create new config setting in db", %{conn: conn} do
conn =
post(conn, "/api/pleroma/admin/config", %{
configs: [
%{key: "key1", value: "value1"},
%{
key: "key2",
value: %{
"nested_1" => "nested_value1",
"nested_2" => [
%{"nested_22" => "nested_value222"},
%{"nested_33" => %{"nested_44" => "nested_444"}}
]
}
},
%{
key: "key3",
value: [
%{"nested_3" => ":nested_3", "nested_33" => "nested_33"},
%{"nested_4" => ":true"}
]
},
%{
key: "key4",
value: %{"nested_5" => ":upload", "endpoint" => "https://example.com"}
}
]
})
assert json_response(conn, 200) == %{
"configs" => [
%{
"key" => "key1",
"value" => "value1"
},
%{
"key" => "key2",
"value" => [
%{"nested_1" => "nested_value1"},
%{
"nested_2" => [
%{"nested_22" => "nested_value222"},
%{"nested_33" => %{"nested_44" => "nested_444"}}
]
}
]
},
%{
"key" => "key3",
"value" => [
[%{"nested_3" => "nested_3"}, %{"nested_33" => "nested_33"}],
%{"nested_4" => true}
]
},
%{
"key" => "key4",
"value" => [%{"endpoint" => "https://example.com"}, %{"nested_5" => "upload"}]
}
]
}
assert Application.get_env(:pleroma, :key1) == "value1"
assert Application.get_env(:pleroma, :key2) == [
nested_1: "nested_value1",
nested_2: [
[nested_22: "nested_value222"],
[nested_33: [nested_44: "nested_444"]]
]
]
assert Application.get_env(:pleroma, :key3) == [
[nested_3: :nested_3, nested_33: "nested_33"],
[nested_4: true]
]
assert Application.get_env(:pleroma, :key4) == [
endpoint: "https://example.com",
nested_5: :upload
]
end
test "update config setting & delete", %{conn: conn} do
config1 = insert(:config, key: "keyaa1")
config2 = insert(:config, key: "keyaa2")
conn =
post(conn, "/api/pleroma/admin/config", %{
configs: [
%{key: config1.key, value: "another_value"},
%{key: config2.key, delete: "true"}
]
})
assert json_response(conn, 200) == %{
"configs" => [
%{
"key" => config1.key,
"value" => "another_value"
}
]
}
assert Application.get_env(:pleroma, :keyaa1) == "another_value"
refute Application.get_env(:pleroma, :keyaa2)
end
end
end end

View file

@ -0,0 +1,183 @@
defmodule Pleroma.Web.AdminAPI.ConfigTest do
use Pleroma.DataCase, async: true
import Pleroma.Factory
alias Pleroma.Web.AdminAPI.Config
test "get_by_key/1" do
config = insert(:config)
insert(:config)
assert config == Config.get_by_key(config.key)
end
test "create/1" do
{:ok, config} = Config.create(%{key: "some_key", value: "some_value"})
assert config == Config.get_by_key("some_key")
end
test "update/1" do
config = insert(:config)
{:ok, updated} = Config.update(config, %{value: "some_value"})
loaded = Config.get_by_key(config.key)
assert loaded == updated
end
test "update_or_create/1" do
config = insert(:config)
key2 = "another_key"
params = [
%{key: key2, value: "another_value"},
%{key: config.key, value: "new_value"}
]
assert Repo.all(Config) |> length() == 1
Enum.each(params, &Config.update_or_create(&1))
assert Repo.all(Config) |> length() == 2
config1 = Config.get_by_key(config.key)
config2 = Config.get_by_key(key2)
assert config1.value == Config.transform("new_value")
assert config2.value == Config.transform("another_value")
end
test "delete/1" do
config = insert(:config)
{:ok, _} = Config.delete(config.key)
refute Config.get_by_key(config.key)
end
describe "transform/1" do
test "string" do
binary = Config.transform("value as string")
assert binary == :erlang.term_to_binary("value as string")
assert Config.from_binary(binary) == "value as string"
end
test "list of modules" do
binary = Config.transform(["Pleroma.Repo", "Pleroma.Activity"])
assert binary == :erlang.term_to_binary([Pleroma.Repo, Pleroma.Activity])
assert Config.from_binary(binary) == [Pleroma.Repo, Pleroma.Activity]
end
test "list of strings" do
binary = Config.transform(["string1", "string2"])
assert binary == :erlang.term_to_binary(["string1", "string2"])
assert Config.from_binary(binary) == ["string1", "string2"]
end
test "map" do
binary =
Config.transform(%{
"types" => "Pleroma.PostgresTypes",
"telemetry_event" => ["Pleroma.Repo.Instrumenter"],
"migration_lock" => ""
})
assert binary ==
:erlang.term_to_binary(
telemetry_event: [Pleroma.Repo.Instrumenter],
types: Pleroma.PostgresTypes
)
assert Config.from_binary(binary) == [
telemetry_event: [Pleroma.Repo.Instrumenter],
types: Pleroma.PostgresTypes
]
end
test "complex map with nested integers, lists and atoms" do
binary =
Config.transform(%{
"uploader" => "Pleroma.Uploaders.Local",
"filters" => ["Pleroma.Upload.Filter.Dedupe"],
"link_name" => ":true",
"proxy_remote" => ":false",
"proxy_opts" => %{
"redirect_on_failure" => ":false",
"max_body_length" => "i:1048576",
"http" => %{
"follow_redirect" => ":true",
"pool" => ":upload"
}
}
})
assert binary ==
:erlang.term_to_binary(
filters: [Pleroma.Upload.Filter.Dedupe],
link_name: true,
proxy_opts: [
http: [
follow_redirect: true,
pool: :upload
],
max_body_length: 1_048_576,
redirect_on_failure: false
],
proxy_remote: false,
uploader: Pleroma.Uploaders.Local
)
assert Config.from_binary(binary) ==
[
filters: [Pleroma.Upload.Filter.Dedupe],
link_name: true,
proxy_opts: [
http: [
follow_redirect: true,
pool: :upload
],
max_body_length: 1_048_576,
redirect_on_failure: false
],
proxy_remote: false,
uploader: Pleroma.Uploaders.Local
]
end
test "keyword" do
binary =
Config.transform(%{
"level" => ":warn",
"meta" => [":all"],
"webhook_url" => "https://hooks.slack.com/services/YOUR-KEY-HERE"
})
assert binary ==
:erlang.term_to_binary(
level: :warn,
meta: [:all],
webhook_url: "https://hooks.slack.com/services/YOUR-KEY-HERE"
)
assert Config.from_binary(binary) == [
level: :warn,
meta: [:all],
webhook_url: "https://hooks.slack.com/services/YOUR-KEY-HERE"
]
end
test "complex map with sigil" do
binary =
Config.transform(%{
federated_timeline_removal: [],
reject: [~r/comp[lL][aA][iI][nN]er/],
replace: []
})
assert binary ==
:erlang.term_to_binary(
federated_timeline_removal: [],
reject: [~r/comp[lL][aA][iI][nN]er/],
replace: []
)
assert Config.from_binary(binary) ==
[federated_timeline_removal: [], reject: [~r/comp[lL][aA][iI][nN]er/], replace: []]
end
end
end