saving to DB only added by user settings

lib/mix/tasks/pleroma/config.ex

@@ -4,42 +4,25 @@
 
 defmodule Mix.Tasks.Pleroma.Config do
   use Mix.Task
+
   import Mix.Pleroma
+
   alias Pleroma.ConfigDB
   alias Pleroma.Repo
+
+  require Logger
+
   @shortdoc "Manages the location of the config"
   @moduledoc File.read!("docs/administration/CLI_tasks/config.md")
 
-  @groups [
-    :pleroma,
-    :logger,
-    :quack,
-    :mime,
-    :tesla,
-    :phoenix,
-    :cors_plug,
-    :auto_linker,
-    :esshd,
-    :ueberauth,
-    :http_signatures,
-    :web_push_encryption,
-    :joken
-  ]
-
   def run(["migrate_to_db"]) do
     # we want to save original logger level
     start_pleroma(false)
-
+    migrate_to_db()
-    if Pleroma.Config.get([:configurable_from_database]) do
-      Enum.each(@groups, &load_and_create(&1))
-    else
-      Mix.shell().info(
-        "Migration is not allowed by config. You can change this behavior in instance settings."
-      )
-    end
   end
 
   def run(["migrate_from_db" | options]) do
+    # TODO: add support for releases
     start_pleroma()
 
     {opts, _} =
@@ -72,10 +55,36 @@ def run(["migrate_from_db" | options]) do
     end
   end
 
-  defp load_and_create(group) do
+  @spec migrate_to_db(Path.t() | nil) :: any()
-    group
+  def migrate_to_db(file_path \\ nil) do
-    |> Application.get_all_env()
+    if Pleroma.Config.get([:configurable_from_database]) do
-    |> Enum.reject(fn {k, _v} ->
+      # TODO: add support for releases
+      config_file = file_path || "config/#{Pleroma.Config.get(:env)}.secret.exs"
+      do_migrate_to_db(config_file)
+    else
+      Mix.shell().info(
+        "migration is not allowed by config. You can change this behavior in instance settings."
+      )
+    end
+  end
+
+  defp do_migrate_to_db(config_file) do
+    if File.exists?(config_file) do
+      {custom_config, _paths} =
+        if Code.ensure_loaded?(Config.Reader),
+          do: Config.Reader.read_imports!(config_file),
+          else: Mix.Config.eval!(config_file)
+
+      custom_config
+      |> Keyword.keys()
+      |> Enum.each(&create(&1, custom_config[&1]))
+    else
+      Logger.warn("to migrate settings, you must define custom settings in #{config_file}")
+    end
+  end
+
+  defp create(group, settings) do
+    Enum.reject(settings, fn {k, _v} ->
       k in [Pleroma.Repo, Pleroma.Web.Endpoint, :env, :configurable_from_database] or
         (group == :phoenix and k == :serve_endpoints)
     end)

lib/pleroma/config/config_db.ex

@@ -5,6 +5,7 @@
 defmodule Pleroma.ConfigDB do
   use Ecto.Schema
   import Ecto.Changeset
+  import Ecto.Query
   import Pleroma.Web.Gettext
   alias __MODULE__
   alias Pleroma.Repo
@@ -19,6 +20,20 @@ defmodule Pleroma.ConfigDB do
     timestamps()
   end
 
+  @spec get_all_as_keyword() :: keyword()
+  def get_all_as_keyword do
+    ConfigDB
+    |> select([c], {c.group, c.key, c.value})
+    |> Repo.all()
+    |> Enum.reduce([], fn {group, key, value}, acc ->
+      group = ConfigDB.from_string(group)
+      key = ConfigDB.from_string(key)
+      value = from_binary(value)
+
+      Keyword.update(acc, group, [{key, value}], &Keyword.merge(&1, [{key, value}]))
+    end)
+  end
+
   @spec get_by_params(map()) :: ConfigDB.t() | nil
   def get_by_params(params), do: Repo.get_by(ConfigDB, params)
 
@@ -137,6 +152,9 @@ def from_string(entity) when is_binary(entity) do
     end
   end
 
+  @spec convert(any()) :: any()
+  def convert(entity), do: do_convert(entity)
+
   defp do_convert(entity) when is_list(entity) do
     for v <- entity, into: [], do: do_convert(v)
   end

lib/pleroma/config/holder.ex

@@ -0,0 +1,51 @@
+defmodule Pleroma.Config.Loader do
+  # TODO: add support for releases
+  if Code.ensure_loaded?(Config.Reader) do
+    @spec load() :: map()
+    def load do
+      config = Config.Reader.read!("config/config.exs")
+      env_config = Config.Reader.read!("config/#{Mix.env()}.exs")
+
+      Config.Reader.merge(config, env_config)
+    end
+  else
+    # support for Elixir less than 1.9
+    @spec load() :: map()
+    def load do
+      config = Mix.Config.eval!("config/config.exs")
+      env_config = Mix.Config.eval!("config/#{Mix.env()}.exs")
+
+      Mix.Config.merge(config, env_config)
+    end
+  end
+end
+
+defmodule Pleroma.Config.Holder do
+  @config Pleroma.Config.Loader.load()
+
+  @spec config() :: keyword()
+  def config do
+    @config
+    |> Keyword.keys()
+    |> Enum.map(&filter(&1, config(&1)))
+    |> List.flatten()
+  end
+
+  @spec config(atom()) :: any()
+  def config(group), do: @config[group]
+
+  @spec config(atom(), atom()) :: any()
+  def config(group, key), do: @config[group][key]
+
+  defp filter(group, settings) when group not in [:swarm] do
+    filtered =
+      Enum.reject(settings, fn {k, _v} ->
+        k in [Pleroma.Repo, Pleroma.Web.Endpoint, :env, :configurable_from_database] or
+          (group == :phoenix and k == :serve_endpoints)
+      end)
+
+    {group, filtered}
+  end
+
+  defp filter(_, _), do: []
+end

lib/pleroma/config/transfer_task.ex

@@ -38,10 +38,18 @@ defp update_env(setting) do
       value = ConfigDB.from_binary(setting.value)
 
       if group != :phoenix and key != :serve_endpoints do
-        :ok = Application.put_env(group, key, value)
+        default = Pleroma.Config.Holder.config(group, key)
-      end
 
-      group
+        merged_value =
+          if can_be_merged?(default, value) do
+            DeepMerge.deep_merge(default, value)
+          else
+            value
+          end
+
+        :ok = Application.put_env(group, key, merged_value)
+        group
+      end
     rescue
       e ->
         Logger.warn(
@@ -61,4 +69,12 @@ defp restart(started_applications, app) do
       error -> Logger.warn(inspect(error))
     end
   end
+
+  defp can_be_merged?(val1, val2) when is_map(val1) and is_map(val2), do: true
+
+  defp can_be_merged?(val1, val2) when is_list(val1) and is_list(val2) do
+    Keyword.keyword?(val1) and Keyword.keyword?(val2)
+  end
+
+  defp can_be_merged?(_val1, _val2), do: false
 end

lib/pleroma/web/admin_api/admin_api_controller.ex

@@ -797,7 +797,7 @@ def migrate_from_db(conn, _params) do
     end
   end
 
-  def config_show(conn, _params) do
+  def config_show(conn, %{"only_db" => true}) do
     with :ok <- configurable_from_database(conn) do
       configs = Pleroma.Repo.all(ConfigDB)
 
@@ -814,6 +814,46 @@ def config_show(conn, _params) do
     end
   end
 
+  def config_show(conn, _params) do
+    with :ok <- configurable_from_database(conn) do
+      configs = ConfigDB.get_all_as_keyword()
+
+      if configs == [] do
+        errors(
+          conn,
+          {:error, "To use configuration from database migrate your settings to database."}
+        )
+      else
+        merged =
+          Pleroma.Config.Holder.config()
+          |> DeepMerge.deep_merge(configs)
+          |> Enum.map(fn {group, value} ->
+            Enum.map(value, fn {key, value} ->
+              db =
+                if configs[group][key] do
+                  if Keyword.keyword?(value) do
+                    Keyword.keys(value) |> Enum.map(fn key -> ConfigDB.convert(key) end)
+                  else
+                    ConfigDB.convert(key)
+                  end
+                end
+
+              setting = %{
+                group: ConfigDB.convert(group),
+                key: ConfigDB.convert(key),
+                value: ConfigDB.convert(value)
+              }
+
+              if db, do: Map.put(setting, :db, db), else: setting
+            end)
+          end)
+          |> List.flatten()
+
+        json(conn, %{configs: merged})
+      end
+    end
+  end
+
   def config_update(conn, %{"configs" => configs}) do
     with :ok <- configurable_from_database(conn) do
       updated =

lib/pleroma/web/router.ex

@@ -196,7 +196,6 @@ defmodule Pleroma.Web.Router do
     get("/config", AdminAPIController, :config_show)
     post("/config", AdminAPIController, :config_update)
     get("/config/descriptions", AdminAPIController, :config_descriptions)
-    get("/config/migrate_to_db", AdminAPIController, :migrate_to_db)
     get("/config/migrate_from_db", AdminAPIController, :migrate_from_db)
 
     get("/moderation_log", AdminAPIController, :list_log)

test/config/config_db_test.exs

@@ -15,8 +15,8 @@ test "get_by_key/1" do
   end
 
   test "create/1" do
-    {:ok, config} = ConfigDB.create(%{group: "pleroma", key: "some_key", value: "some_value"})
+    {:ok, config} = ConfigDB.create(%{group: ":pleroma", key: ":some_key", value: "some_value"})
-    assert config == ConfigDB.get_by_params(%{group: "pleroma", key: "some_key"})
+    assert config == ConfigDB.get_by_params(%{group: ":pleroma", key: ":some_key"})
   end
 
   test "update/1" do
@@ -26,6 +26,27 @@ test "update/1" do
     assert loaded == updated
   end
 
+  test "get_all_as_keyword/0" do
+    insert(:config)
+    insert(:config, group: ":quack", key: ":level", value: ConfigDB.to_binary(:info))
+    insert(:config, group: ":quack", key: ":meta", value: ConfigDB.to_binary([:none]))
+
+    insert(:config,
+      group: ":quack",
+      key: ":webhook_url",
+      value: ConfigDB.to_binary("https://hooks.slack.com/services/KEY/some_val")
+    )
+
+    assert [
+             pleroma: [{_, %{another: _, another_key: _}}],
+             quack: [
+               level: :info,
+               meta: [:none],
+               webhook_url: "https://hooks.slack.com/services/KEY/some_val"
+             ]
+           ] = ConfigDB.get_all_as_keyword()
+  end
+
   describe "update_or_create/1" do
     test "common" do
       config = insert(:config)

test/config/transfer_task_test.exs

@@ -47,6 +47,35 @@ test "transfer config values from db to env" do
     end)
   end
 
+  test "transfer config values for 1 group and some keys" do
+    level = Application.get_env(:quack, :level)
+    meta = Application.get_env(:quack, :meta)
+
+    ConfigDB.create(%{
+      group: ":quack",
+      key: ":level",
+      value: :info
+    })
+
+    ConfigDB.create(%{
+      group: ":quack",
+      key: ":meta",
+      value: [:none]
+    })
+
+    Pleroma.Config.TransferTask.start_link([])
+
+    assert Application.get_env(:quack, :level) == :info
+    assert Application.get_env(:quack, :meta) == [:none]
+    default = Pleroma.Config.Holder.config(:quack, :webhook_url)
+    assert Application.get_env(:quack, :webhook_url) == default
+
+    on_exit(fn ->
+      Application.put_env(:quack, :level, level)
+      Application.put_env(:quack, :meta, meta)
+    end)
+  end
+
   test "non existing atom" do
     ConfigDB.create(%{
       group: ":pleroma",

test/fixtures/config/temp.secret.exs

@@ -0,0 +1,9 @@
+use Mix.Config
+
+config :pleroma, :first_setting, key: "value", key2: [Pleroma.Repo]
+
+config :pleroma, :second_setting, key: "value2", key2: ["Activity"]
+
+config :quack, level: :info
+
+config :pleroma, Pleroma.Repo, pool: Ecto.Adapters.SQL.Sandbox

test/tasks/config_test.exs

@@ -4,6 +4,9 @@
 
 defmodule Mix.Tasks.Pleroma.ConfigTest do
   use Pleroma.DataCase
+
+  import ExUnit.CaptureLog
+
   alias Pleroma.ConfigDB
   alias Pleroma.Repo
 
@@ -23,16 +26,17 @@ defmodule Mix.Tasks.Pleroma.ConfigTest do
     Pleroma.Config.put(:configurable_from_database, true)
   end
 
+  test "warning if file with custom settings doesn't exist" do
+    assert capture_log(fn -> Mix.Tasks.Pleroma.Config.run(["migrate_to_db"]) end) =~
+             "to migrate settings, you must define custom settings in config/test.secret.exs"
+  end
+
   test "settings are migrated to db" do
     initial = Application.get_env(:quack, :level)
     on_exit(fn -> Application.put_env(:quack, :level, initial) end)
     assert Repo.all(ConfigDB) == []
 
-    Application.put_env(:pleroma, :first_setting, key: "value", key2: [Repo])
+    Mix.Tasks.Pleroma.Config.migrate_to_db("test/fixtures/config/temp.secret.exs")
-    Application.put_env(:pleroma, :second_setting, key: "value2", key2: ["Activity"])
-    Application.put_env(:quack, :level, :info)
-
-    Mix.Tasks.Pleroma.Config.run(["migrate_to_db"])
 
     config1 = ConfigDB.get_by_params(%{group: ":pleroma", key: ":first_setting"})
     config2 = ConfigDB.get_by_params(%{group: ":pleroma", key: ":second_setting"})

test/web/admin_api/admin_api_controller_test.exs

@@ -7,6 +7,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
   use Oban.Testing, repo: Pleroma.Repo
 
   alias Pleroma.Activity
+  alias Pleroma.ConfigDB
   alias Pleroma.HTML
   alias Pleroma.ModerationLog
   alias Pleroma.Repo
@@ -1881,11 +1882,11 @@ test "without any settings in db", %{conn: conn} do
                "To use configuration from database migrate your settings to database."
     end
 
-    test "with settings in db", %{conn: conn} do
+    test "with settings only in db", %{conn: conn} do
       config1 = insert(:config)
       config2 = insert(:config)
 
-      conn = get(conn, "/api/pleroma/admin/config")
+      conn = get(conn, "/api/pleroma/admin/config", %{"only_db" => true})
 
       %{
         "configs" => [
@@ -1895,6 +1896,7 @@ test "with settings in db", %{conn: conn} do
             "value" => _
           },
           %{
+            "group" => ":pleroma",
             "key" => key2,
             "value" => _
           }
@@ -1904,6 +1906,45 @@ test "with settings in db", %{conn: conn} do
       assert key1 == config1.key
       assert key2 == config2.key
     end
+
+    test "merged default setting with db settings", %{conn: conn} do
+      config1 = insert(:config)
+      config2 = insert(:config)
+
+      config3 =
+        insert(:config,
+          value: ConfigDB.to_binary(k1: :v1, k2: :v2)
+        )
+
+      conn = get(conn, "/api/pleroma/admin/config")
+
+      %{"configs" => configs} = json_response(conn, 200)
+
+      assert length(configs) > 3
+
+      received_configs =
+        Enum.filter(configs, fn %{"group" => group, "key" => key} ->
+          group == ":pleroma" and key in [config1.key, config2.key, config3.key]
+        end)
+
+      assert length(received_configs) == 3
+
+      db_keys =
+        config3.value
+        |> ConfigDB.from_binary()
+        |> Keyword.keys()
+        |> ConfigDB.convert()
+
+      Enum.each(received_configs, fn %{"value" => value, "db" => db} ->
+        assert db in [config1.key, config2.key, db_keys]
+
+        assert value in [
+                 ConfigDB.from_binary_with_convert(config1.value),
+                 ConfigDB.from_binary_with_convert(config2.value),
+                 ConfigDB.from_binary_with_convert(config3.value)
+               ]
+      end)
+    end
   end
 
   test "POST /api/pleroma/admin/config error", %{conn: conn} do
@@ -2831,9 +2872,8 @@ test "proxy tuple ip", %{conn: conn} do
     end
 
     test "transfer settings to DB and to file", %{conn: conn} do
-      on_exit(fn -> :ok = File.rm("config/test.exported_from_db.secret.exs") end)
       assert Repo.all(Pleroma.ConfigDB) == []
-      Mix.Tasks.Pleroma.Config.run(["migrate_to_db"])
+      Mix.Tasks.Pleroma.Config.migrate_to_db("test/fixtures/config/temp.secret.exs")
       assert Repo.aggregate(Pleroma.ConfigDB, :count, :id) > 0
 
       conn = get(conn, "/api/pleroma/admin/config/migrate_from_db")