From 60ba2339a244290f7353e8026032b1a5d185227c Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 17 Jan 2020 11:45:44 +0300 Subject: [PATCH] saving to DB only added by user settings --- lib/mix/tasks/pleroma/config.ex | 65 +++++++++++-------- lib/pleroma/config/config_db.ex | 18 +++++ lib/pleroma/config/holder.ex | 51 +++++++++++++++ lib/pleroma/config/transfer_task.ex | 22 ++++++- .../web/admin_api/admin_api_controller.ex | 42 +++++++++++- lib/pleroma/web/router.ex | 1 - test/config/config_db_test.exs | 25 ++++++- test/config/transfer_task_test.exs | 29 +++++++++ test/fixtures/config/temp.secret.exs | 9 +++ test/tasks/config_test.exs | 14 ++-- .../admin_api/admin_api_controller_test.exs | 48 ++++++++++++-- 11 files changed, 280 insertions(+), 44 deletions(-) create mode 100644 lib/pleroma/config/holder.ex create mode 100644 test/fixtures/config/temp.secret.exs diff --git a/lib/mix/tasks/pleroma/config.ex b/lib/mix/tasks/pleroma/config.ex index 57952aeba..3157e7b20 100644 --- a/lib/mix/tasks/pleroma/config.ex +++ b/lib/mix/tasks/pleroma/config.ex @@ -4,42 +4,25 @@ defmodule Mix.Tasks.Pleroma.Config do use Mix.Task + import Mix.Pleroma + alias Pleroma.ConfigDB alias Pleroma.Repo + + require Logger + @shortdoc "Manages the location of the config" @moduledoc File.read!("docs/administration/CLI_tasks/config.md") - @groups [ - :pleroma, - :logger, - :quack, - :mime, - :tesla, - :phoenix, - :cors_plug, - :auto_linker, - :esshd, - :ueberauth, - :http_signatures, - :web_push_encryption, - :joken - ] - def run(["migrate_to_db"]) do # we want to save original logger level start_pleroma(false) - - if Pleroma.Config.get([:configurable_from_database]) do - Enum.each(@groups, &load_and_create(&1)) - else - Mix.shell().info( - "Migration is not allowed by config. You can change this behavior in instance settings." - ) - end + migrate_to_db() end def run(["migrate_from_db" | options]) do + # TODO: add support for releases start_pleroma() {opts, _} = @@ -72,10 +55,36 @@ def run(["migrate_from_db" | options]) do end end - defp load_and_create(group) do - group - |> Application.get_all_env() - |> Enum.reject(fn {k, _v} -> + @spec migrate_to_db(Path.t() | nil) :: any() + def migrate_to_db(file_path \\ nil) do + if Pleroma.Config.get([:configurable_from_database]) do + # TODO: add support for releases + config_file = file_path || "config/#{Pleroma.Config.get(:env)}.secret.exs" + do_migrate_to_db(config_file) + else + Mix.shell().info( + "migration is not allowed by config. You can change this behavior in instance settings." + ) + end + end + + defp do_migrate_to_db(config_file) do + if File.exists?(config_file) do + {custom_config, _paths} = + if Code.ensure_loaded?(Config.Reader), + do: Config.Reader.read_imports!(config_file), + else: Mix.Config.eval!(config_file) + + custom_config + |> Keyword.keys() + |> Enum.each(&create(&1, custom_config[&1])) + else + Logger.warn("to migrate settings, you must define custom settings in #{config_file}") + end + end + + defp create(group, settings) do + Enum.reject(settings, fn {k, _v} -> k in [Pleroma.Repo, Pleroma.Web.Endpoint, :env, :configurable_from_database] or (group == :phoenix and k == :serve_endpoints) end) diff --git a/lib/pleroma/config/config_db.ex b/lib/pleroma/config/config_db.ex index e433ce442..294fbc3f0 100644 --- a/lib/pleroma/config/config_db.ex +++ b/lib/pleroma/config/config_db.ex @@ -5,6 +5,7 @@ defmodule Pleroma.ConfigDB do use Ecto.Schema import Ecto.Changeset + import Ecto.Query import Pleroma.Web.Gettext alias __MODULE__ alias Pleroma.Repo @@ -19,6 +20,20 @@ defmodule Pleroma.ConfigDB do timestamps() end + @spec get_all_as_keyword() :: keyword() + def get_all_as_keyword do + ConfigDB + |> select([c], {c.group, c.key, c.value}) + |> Repo.all() + |> Enum.reduce([], fn {group, key, value}, acc -> + group = ConfigDB.from_string(group) + key = ConfigDB.from_string(key) + value = from_binary(value) + + Keyword.update(acc, group, [{key, value}], &Keyword.merge(&1, [{key, value}])) + end) + end + @spec get_by_params(map()) :: ConfigDB.t() | nil def get_by_params(params), do: Repo.get_by(ConfigDB, params) @@ -137,6 +152,9 @@ def from_string(entity) when is_binary(entity) do end end + @spec convert(any()) :: any() + def convert(entity), do: do_convert(entity) + defp do_convert(entity) when is_list(entity) do for v <- entity, into: [], do: do_convert(v) end diff --git a/lib/pleroma/config/holder.ex b/lib/pleroma/config/holder.ex new file mode 100644 index 000000000..aa35382b2 --- /dev/null +++ b/lib/pleroma/config/holder.ex @@ -0,0 +1,51 @@ +defmodule Pleroma.Config.Loader do + # TODO: add support for releases + if Code.ensure_loaded?(Config.Reader) do + @spec load() :: map() + def load do + config = Config.Reader.read!("config/config.exs") + env_config = Config.Reader.read!("config/#{Mix.env()}.exs") + + Config.Reader.merge(config, env_config) + end + else + # support for Elixir less than 1.9 + @spec load() :: map() + def load do + config = Mix.Config.eval!("config/config.exs") + env_config = Mix.Config.eval!("config/#{Mix.env()}.exs") + + Mix.Config.merge(config, env_config) + end + end +end + +defmodule Pleroma.Config.Holder do + @config Pleroma.Config.Loader.load() + + @spec config() :: keyword() + def config do + @config + |> Keyword.keys() + |> Enum.map(&filter(&1, config(&1))) + |> List.flatten() + end + + @spec config(atom()) :: any() + def config(group), do: @config[group] + + @spec config(atom(), atom()) :: any() + def config(group, key), do: @config[group][key] + + defp filter(group, settings) when group not in [:swarm] do + filtered = + Enum.reject(settings, fn {k, _v} -> + k in [Pleroma.Repo, Pleroma.Web.Endpoint, :env, :configurable_from_database] or + (group == :phoenix and k == :serve_endpoints) + end) + + {group, filtered} + end + + defp filter(_, _), do: [] +end diff --git a/lib/pleroma/config/transfer_task.ex b/lib/pleroma/config/transfer_task.ex index b39f9594e..828a297de 100644 --- a/lib/pleroma/config/transfer_task.ex +++ b/lib/pleroma/config/transfer_task.ex @@ -38,10 +38,18 @@ defp update_env(setting) do value = ConfigDB.from_binary(setting.value) if group != :phoenix and key != :serve_endpoints do - :ok = Application.put_env(group, key, value) - end + default = Pleroma.Config.Holder.config(group, key) - group + merged_value = + if can_be_merged?(default, value) do + DeepMerge.deep_merge(default, value) + else + value + end + + :ok = Application.put_env(group, key, merged_value) + group + end rescue e -> Logger.warn( @@ -61,4 +69,12 @@ defp restart(started_applications, app) do error -> Logger.warn(inspect(error)) end end + + defp can_be_merged?(val1, val2) when is_map(val1) and is_map(val2), do: true + + defp can_be_merged?(val1, val2) when is_list(val1) and is_list(val2) do + Keyword.keyword?(val1) and Keyword.keyword?(val2) + end + + defp can_be_merged?(_val1, _val2), do: false end diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index e1d16c30b..cc93c2309 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -797,7 +797,7 @@ def migrate_from_db(conn, _params) do end end - def config_show(conn, _params) do + def config_show(conn, %{"only_db" => true}) do with :ok <- configurable_from_database(conn) do configs = Pleroma.Repo.all(ConfigDB) @@ -814,6 +814,46 @@ def config_show(conn, _params) do end end + def config_show(conn, _params) do + with :ok <- configurable_from_database(conn) do + configs = ConfigDB.get_all_as_keyword() + + if configs == [] do + errors( + conn, + {:error, "To use configuration from database migrate your settings to database."} + ) + else + merged = + Pleroma.Config.Holder.config() + |> DeepMerge.deep_merge(configs) + |> Enum.map(fn {group, value} -> + Enum.map(value, fn {key, value} -> + db = + if configs[group][key] do + if Keyword.keyword?(value) do + Keyword.keys(value) |> Enum.map(fn key -> ConfigDB.convert(key) end) + else + ConfigDB.convert(key) + end + end + + setting = %{ + group: ConfigDB.convert(group), + key: ConfigDB.convert(key), + value: ConfigDB.convert(value) + } + + if db, do: Map.put(setting, :db, db), else: setting + end) + end) + |> List.flatten() + + json(conn, %{configs: merged}) + end + end + end + def config_update(conn, %{"configs" => configs}) do with :ok <- configurable_from_database(conn) do updated = diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 8cd7e021a..ef6e5a565 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -196,7 +196,6 @@ defmodule Pleroma.Web.Router do get("/config", AdminAPIController, :config_show) post("/config", AdminAPIController, :config_update) get("/config/descriptions", AdminAPIController, :config_descriptions) - get("/config/migrate_to_db", AdminAPIController, :migrate_to_db) get("/config/migrate_from_db", AdminAPIController, :migrate_from_db) get("/moderation_log", AdminAPIController, :list_log) diff --git a/test/config/config_db_test.exs b/test/config/config_db_test.exs index 096df9203..6f76008e6 100644 --- a/test/config/config_db_test.exs +++ b/test/config/config_db_test.exs @@ -15,8 +15,8 @@ test "get_by_key/1" do end test "create/1" do - {:ok, config} = ConfigDB.create(%{group: "pleroma", key: "some_key", value: "some_value"}) - assert config == ConfigDB.get_by_params(%{group: "pleroma", key: "some_key"}) + {:ok, config} = ConfigDB.create(%{group: ":pleroma", key: ":some_key", value: "some_value"}) + assert config == ConfigDB.get_by_params(%{group: ":pleroma", key: ":some_key"}) end test "update/1" do @@ -26,6 +26,27 @@ test "update/1" do assert loaded == updated end + test "get_all_as_keyword/0" do + insert(:config) + insert(:config, group: ":quack", key: ":level", value: ConfigDB.to_binary(:info)) + insert(:config, group: ":quack", key: ":meta", value: ConfigDB.to_binary([:none])) + + insert(:config, + group: ":quack", + key: ":webhook_url", + value: ConfigDB.to_binary("https://hooks.slack.com/services/KEY/some_val") + ) + + assert [ + pleroma: [{_, %{another: _, another_key: _}}], + quack: [ + level: :info, + meta: [:none], + webhook_url: "https://hooks.slack.com/services/KEY/some_val" + ] + ] = ConfigDB.get_all_as_keyword() + end + describe "update_or_create/1" do test "common" do config = insert(:config) diff --git a/test/config/transfer_task_test.exs b/test/config/transfer_task_test.exs index 89de93ca3..c3c4ef674 100644 --- a/test/config/transfer_task_test.exs +++ b/test/config/transfer_task_test.exs @@ -47,6 +47,35 @@ test "transfer config values from db to env" do end) end + test "transfer config values for 1 group and some keys" do + level = Application.get_env(:quack, :level) + meta = Application.get_env(:quack, :meta) + + ConfigDB.create(%{ + group: ":quack", + key: ":level", + value: :info + }) + + ConfigDB.create(%{ + group: ":quack", + key: ":meta", + value: [:none] + }) + + Pleroma.Config.TransferTask.start_link([]) + + assert Application.get_env(:quack, :level) == :info + assert Application.get_env(:quack, :meta) == [:none] + default = Pleroma.Config.Holder.config(:quack, :webhook_url) + assert Application.get_env(:quack, :webhook_url) == default + + on_exit(fn -> + Application.put_env(:quack, :level, level) + Application.put_env(:quack, :meta, meta) + end) + end + test "non existing atom" do ConfigDB.create(%{ group: ":pleroma", diff --git a/test/fixtures/config/temp.secret.exs b/test/fixtures/config/temp.secret.exs new file mode 100644 index 000000000..f4686c101 --- /dev/null +++ b/test/fixtures/config/temp.secret.exs @@ -0,0 +1,9 @@ +use Mix.Config + +config :pleroma, :first_setting, key: "value", key2: [Pleroma.Repo] + +config :pleroma, :second_setting, key: "value2", key2: ["Activity"] + +config :quack, level: :info + +config :pleroma, Pleroma.Repo, pool: Ecto.Adapters.SQL.Sandbox diff --git a/test/tasks/config_test.exs b/test/tasks/config_test.exs index 7759f0586..ff921ecfa 100644 --- a/test/tasks/config_test.exs +++ b/test/tasks/config_test.exs @@ -4,6 +4,9 @@ defmodule Mix.Tasks.Pleroma.ConfigTest do use Pleroma.DataCase + + import ExUnit.CaptureLog + alias Pleroma.ConfigDB alias Pleroma.Repo @@ -23,16 +26,17 @@ defmodule Mix.Tasks.Pleroma.ConfigTest do Pleroma.Config.put(:configurable_from_database, true) end + test "warning if file with custom settings doesn't exist" do + assert capture_log(fn -> Mix.Tasks.Pleroma.Config.run(["migrate_to_db"]) end) =~ + "to migrate settings, you must define custom settings in config/test.secret.exs" + end + test "settings are migrated to db" do initial = Application.get_env(:quack, :level) on_exit(fn -> Application.put_env(:quack, :level, initial) end) assert Repo.all(ConfigDB) == [] - Application.put_env(:pleroma, :first_setting, key: "value", key2: [Repo]) - Application.put_env(:pleroma, :second_setting, key: "value2", key2: ["Activity"]) - Application.put_env(:quack, :level, :info) - - Mix.Tasks.Pleroma.Config.run(["migrate_to_db"]) + Mix.Tasks.Pleroma.Config.migrate_to_db("test/fixtures/config/temp.secret.exs") config1 = ConfigDB.get_by_params(%{group: ":pleroma", key: ":first_setting"}) config2 = ConfigDB.get_by_params(%{group: ":pleroma", key: ":second_setting"}) diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index 8e80f9b47..35cef4df3 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -7,6 +7,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do use Oban.Testing, repo: Pleroma.Repo alias Pleroma.Activity + alias Pleroma.ConfigDB alias Pleroma.HTML alias Pleroma.ModerationLog alias Pleroma.Repo @@ -1881,11 +1882,11 @@ test "without any settings in db", %{conn: conn} do "To use configuration from database migrate your settings to database." end - test "with settings in db", %{conn: conn} do + test "with settings only in db", %{conn: conn} do config1 = insert(:config) config2 = insert(:config) - conn = get(conn, "/api/pleroma/admin/config") + conn = get(conn, "/api/pleroma/admin/config", %{"only_db" => true}) %{ "configs" => [ @@ -1895,6 +1896,7 @@ test "with settings in db", %{conn: conn} do "value" => _ }, %{ + "group" => ":pleroma", "key" => key2, "value" => _ } @@ -1904,6 +1906,45 @@ test "with settings in db", %{conn: conn} do assert key1 == config1.key assert key2 == config2.key end + + test "merged default setting with db settings", %{conn: conn} do + config1 = insert(:config) + config2 = insert(:config) + + config3 = + insert(:config, + value: ConfigDB.to_binary(k1: :v1, k2: :v2) + ) + + conn = get(conn, "/api/pleroma/admin/config") + + %{"configs" => configs} = json_response(conn, 200) + + assert length(configs) > 3 + + received_configs = + Enum.filter(configs, fn %{"group" => group, "key" => key} -> + group == ":pleroma" and key in [config1.key, config2.key, config3.key] + end) + + assert length(received_configs) == 3 + + db_keys = + config3.value + |> ConfigDB.from_binary() + |> Keyword.keys() + |> ConfigDB.convert() + + Enum.each(received_configs, fn %{"value" => value, "db" => db} -> + assert db in [config1.key, config2.key, db_keys] + + assert value in [ + ConfigDB.from_binary_with_convert(config1.value), + ConfigDB.from_binary_with_convert(config2.value), + ConfigDB.from_binary_with_convert(config3.value) + ] + end) + end end test "POST /api/pleroma/admin/config error", %{conn: conn} do @@ -2831,9 +2872,8 @@ test "proxy tuple ip", %{conn: conn} do end test "transfer settings to DB and to file", %{conn: conn} do - on_exit(fn -> :ok = File.rm("config/test.exported_from_db.secret.exs") end) assert Repo.all(Pleroma.ConfigDB) == [] - Mix.Tasks.Pleroma.Config.run(["migrate_to_db"]) + Mix.Tasks.Pleroma.Config.migrate_to_db("test/fixtures/config/temp.secret.exs") assert Repo.aggregate(Pleroma.ConfigDB, :count, :id) > 0 conn = get(conn, "/api/pleroma/admin/config/migrate_from_db")