From efb8ef5abee1a8defa2bfba40ad1065db4c09ddf Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Sat, 18 Jan 2020 16:55:33 +0300 Subject: [PATCH] releases support --- config/releases.exs | 8 +++ lib/mix/tasks/pleroma/config.ex | 49 +++++++++++++------ lib/pleroma/config/holder.ex | 33 +------------ lib/pleroma/config/loader.ex | 39 +++++++++++++++ test/tasks/config_test.exs | 22 ++++++--- .../admin_api/admin_api_controller_test.exs | 9 ++-- 6 files changed, 103 insertions(+), 57 deletions(-) create mode 100644 lib/pleroma/config/loader.ex diff --git a/config/releases.exs b/config/releases.exs index b224960db..79651a342 100644 --- a/config/releases.exs +++ b/config/releases.exs @@ -6,6 +6,8 @@ config_path = System.get_env("PLEROMA_CONFIG_PATH") || "/etc/pleroma/config.exs" +config :pleroma, release: true, config_path: config_path + if File.exists?(config_path) do import_config config_path else @@ -18,3 +20,9 @@ IO.puts(warning) end + +exported_config = Path.join([Path.dirname(config_path), "prod.exported_from_db.secret.exs"]) + +if File.exists?(exported_config) do + import_config exported_config +end diff --git a/lib/mix/tasks/pleroma/config.ex b/lib/mix/tasks/pleroma/config.ex index 148d18141..715b72dbe 100644 --- a/lib/mix/tasks/pleroma/config.ex +++ b/lib/mix/tasks/pleroma/config.ex @@ -10,8 +10,6 @@ defmodule Mix.Tasks.Pleroma.Config do alias Pleroma.ConfigDB alias Pleroma.Repo - require Logger - @shortdoc "Manages the location of the config" @moduledoc File.read!("docs/administration/CLI_tasks/config.md") @@ -32,10 +30,10 @@ def run(["migrate_from_db" | options]) do with {:active?, true} <- {:active?, Pleroma.Config.get([:configurable_from_database])}, - env_path when is_binary(env_path) <- opts[:env], - config_path <- "config/#{env_path}.exported_from_db.secret.exs", + env when is_binary(env) <- opts[:env] || "prod", + config_path <- config_path(env), {:ok, file} <- File.open(config_path, [:write, :utf8]) do - IO.write(file, "use Mix.Config\r\n") + IO.write(file, config_header()) ConfigDB |> Repo.all() @@ -45,31 +43,50 @@ def run(["migrate_from_db" | options]) do System.cmd("mix", ["format", config_path]) else {:active?, false} -> - Mix.shell().info( - "migration is not allowed by config. You can change this behavior in instance settings." + shell_info( + "Migration is not allowed by config. You can change this behavior in instance settings." ) error -> - Mix.shell().info("error occuried while opening file. #{inspect(error)}") + shell_info("Error occuried while opening file. #{inspect(error)}") end end + defp config_path(env) do + path = + if Pleroma.Config.get(:release) do + :config_path + |> Pleroma.Config.get() + |> Path.dirname() + else + "config" + end + + Path.join(path, "#{env}.exported_from_db.secret.exs") + end + @spec migrate_to_db(Path.t() | nil) :: any() def migrate_to_db(file_path \\ nil) do if Pleroma.Config.get([:configurable_from_database]) do - # TODO: add support for releases - config_file = file_path || "config/#{Pleroma.Config.get(:env)}.secret.exs" + user_config_file = + if Pleroma.Config.get(:release), + do: Pleroma.Config.get(:config_path), + else: "config/#{Pleroma.Config.get(:env)}.secret.exs" + + config_file = file_path || user_config_file do_migrate_to_db(config_file) else - Mix.shell().info( - "migration is not allowed by config. You can change this behavior in instance settings." + shell_info( + "Migration is not allowed by config. You can change this behavior in instance settings." ) end end if Code.ensure_loaded?(Config.Reader) do + defp config_header, do: "import Config\r\n\r\n" defp read_file(config_file), do: Config.Reader.read_imports!(config_file) else + defp config_header, do: "use Mix.Config\r\n\r\n" defp read_file(config_file), do: Mix.Config.eval!(config_file) end @@ -81,7 +98,7 @@ defp do_migrate_to_db(config_file) do |> Keyword.keys() |> Enum.each(&create(&1, custom_config[&1])) else - Logger.warn("to migrate settings, you must define custom settings in #{config_file}") + shell_info("To migrate settings, you must define custom settings in #{config_file}.") end end @@ -94,10 +111,10 @@ defp create(group, settings) do key = inspect(key) {:ok, _} = ConfigDB.update_or_create(%{group: inspect(group), key: key, value: value}) - Mix.shell().info("settings for key #{key} migrated.") + shell_info("Settings for key #{key} migrated.") end) - Mix.shell().info("settings for group :#{group} migrated.") + shell_info("Settings for group :#{group} migrated.") end defp write_to_file_with_deletion(config, file, with_deletion) do @@ -110,7 +127,7 @@ defp write_to_file_with_deletion(config, file, with_deletion) do if with_deletion do {:ok, _} = Repo.delete(config) - Mix.shell().info("#{config.key} deleted from DB.") + shell_info("#{config.key} deleted from DB.") end end end diff --git a/lib/pleroma/config/holder.ex b/lib/pleroma/config/holder.ex index 7436df1a7..0fa7c5bf7 100644 --- a/lib/pleroma/config/holder.ex +++ b/lib/pleroma/config/holder.ex @@ -1,36 +1,5 @@ -defmodule Pleroma.Config.Loader do - # TODO: add support for releases - if Code.ensure_loaded?(Config.Reader) do - @spec load() :: map() - def load do - config = load("config/config.exs") - env_config = load("config/#{Mix.env()}.exs") - - Config.Reader.merge(config, env_config) - end - - @spec load(Path.t()) :: keyword() - def load(path), do: Config.Reader.read!(path) - else - # support for Elixir less than 1.9 - @spec load() :: map() - def load do - config = load("config/config.exs") - env_config = load("config/#{Mix.env()}.exs") - - Mix.Config.merge(config, env_config) - end - - @spec load(Path.t()) :: keyword() - def load(path) do - {config, _paths} = Mix.Config.eval!(path) - config - end - end -end - defmodule Pleroma.Config.Holder do - @config Pleroma.Config.Loader.load() + @config Pleroma.Config.Loader.load_and_merge() @spec config() :: keyword() def config do diff --git a/lib/pleroma/config/loader.ex b/lib/pleroma/config/loader.ex new file mode 100644 index 000000000..b13b6eaf4 --- /dev/null +++ b/lib/pleroma/config/loader.ex @@ -0,0 +1,39 @@ +defmodule Pleroma.Config.Loader do + @paths ["config/config.exs", "config/#{Mix.env()}.exs"] + + if Code.ensure_loaded?(Config.Reader) do + @spec load(Path.t()) :: keyword() + def load(path), do: Config.Reader.read!(path) + + defp do_merge(conf1, conf2), do: Config.Reader.merge(conf1, conf2) + else + # support for Elixir less than 1.9 + @spec load(Path.t()) :: keyword() + def load(path) do + {config, _paths} = Mix.Config.eval!(path) + config + end + + defp do_merge(conf1, conf2), do: Mix.Config.merge(conf1, conf2) + end + + @spec load_and_merge() :: keyword() + def load_and_merge do + all_paths = + if Pleroma.Config.get(:release), + do: @paths ++ ["config/releases.exs"], + else: @paths + + all_paths + |> Enum.map(&load(&1)) + |> merge() + end + + @spec merge([keyword()], keyword()) :: keyword() + def merge(configs, acc \\ []) + def merge([], acc), do: acc + + def merge([config | others], acc) do + merge(others, do_merge(acc, config)) + end +end diff --git a/test/tasks/config_test.exs b/test/tasks/config_test.exs index ff921ecfa..2e56e6cfe 100644 --- a/test/tasks/config_test.exs +++ b/test/tasks/config_test.exs @@ -5,8 +5,6 @@ defmodule Mix.Tasks.Pleroma.ConfigTest do use Pleroma.DataCase - import ExUnit.CaptureLog - alias Pleroma.ConfigDB alias Pleroma.Repo @@ -26,9 +24,14 @@ defmodule Mix.Tasks.Pleroma.ConfigTest do Pleroma.Config.put(:configurable_from_database, true) end - test "warning if file with custom settings doesn't exist" do - assert capture_log(fn -> Mix.Tasks.Pleroma.Config.run(["migrate_to_db"]) end) =~ - "to migrate settings, you must define custom settings in config/test.secret.exs" + test "error if file with custom settings doesn't exist" do + Mix.Tasks.Pleroma.Config.run(["migrate_to_db"]) + + assert_receive {:mix_shell, :info, + [ + "To migrate settings, you must define custom settings in config/test.secret.exs." + ]}, + 15 end test "settings are migrated to db" do @@ -159,8 +162,15 @@ test "load a settings with large values and pass to file", %{temp_file: temp_fil assert File.exists?(temp_file) {:ok, file} = File.read(temp_file) + header = + if Code.ensure_loaded?(Config.Reader) do + "import Config" + else + "use Mix.Config" + end + assert file == - "use Mix.Config\n\nconfig :pleroma, :instance,\n name: \"Pleroma\",\n email: \"example@example.com\",\n notify_email: \"noreply@example.com\",\n description: \"A Pleroma instance, an alternative fediverse server\",\n limit: 5000,\n chat_limit: 5000,\n remote_limit: 100_000,\n upload_limit: 16_000_000,\n avatar_upload_limit: 2_000_000,\n background_upload_limit: 4_000_000,\n banner_upload_limit: 4_000_000,\n poll_limits: %{\n max_expiration: 31_536_000,\n max_option_chars: 200,\n max_options: 20,\n min_expiration: 0\n },\n registrations_open: true,\n federating: true,\n federation_incoming_replies_max_depth: 100,\n federation_reachability_timeout_days: 7,\n federation_publisher_modules: [Pleroma.Web.ActivityPub.Publisher],\n allow_relay: true,\n rewrite_policy: Pleroma.Web.ActivityPub.MRF.NoOpPolicy,\n public: true,\n quarantined_instances: [],\n managed_config: true,\n static_dir: \"instance/static/\",\n allowed_post_formats: [\"text/plain\", \"text/html\", \"text/markdown\", \"text/bbcode\"],\n mrf_transparency: true,\n mrf_transparency_exclusions: [],\n autofollowed_nicknames: [],\n max_pinned_statuses: 1,\n no_attachment_links: true,\n welcome_user_nickname: nil,\n welcome_message: nil,\n max_report_comment_size: 1000,\n safe_dm_mentions: false,\n healthcheck: false,\n remote_post_retention_days: 90,\n skip_thread_containment: true,\n limit_to_local_content: :unauthenticated,\n user_bio_length: 5000,\n user_name_length: 100,\n max_account_fields: 10,\n max_remote_account_fields: 20,\n account_field_name_length: 512,\n account_field_value_length: 2048,\n external_user_synchronization: true,\n extended_nickname_format: true,\n multi_factor_authentication: [\n totp: [digits: 6, period: 30],\n backup_codes: [number: 2, length: 6]\n ]\n" + "#{header}\n\nconfig :pleroma, :instance,\n name: \"Pleroma\",\n email: \"example@example.com\",\n notify_email: \"noreply@example.com\",\n description: \"A Pleroma instance, an alternative fediverse server\",\n limit: 5000,\n chat_limit: 5000,\n remote_limit: 100_000,\n upload_limit: 16_000_000,\n avatar_upload_limit: 2_000_000,\n background_upload_limit: 4_000_000,\n banner_upload_limit: 4_000_000,\n poll_limits: %{\n max_expiration: 31_536_000,\n max_option_chars: 200,\n max_options: 20,\n min_expiration: 0\n },\n registrations_open: true,\n federating: true,\n federation_incoming_replies_max_depth: 100,\n federation_reachability_timeout_days: 7,\n federation_publisher_modules: [Pleroma.Web.ActivityPub.Publisher],\n allow_relay: true,\n rewrite_policy: Pleroma.Web.ActivityPub.MRF.NoOpPolicy,\n public: true,\n quarantined_instances: [],\n managed_config: true,\n static_dir: \"instance/static/\",\n allowed_post_formats: [\"text/plain\", \"text/html\", \"text/markdown\", \"text/bbcode\"],\n mrf_transparency: true,\n mrf_transparency_exclusions: [],\n autofollowed_nicknames: [],\n max_pinned_statuses: 1,\n no_attachment_links: true,\n welcome_user_nickname: nil,\n welcome_message: nil,\n max_report_comment_size: 1000,\n safe_dm_mentions: false,\n healthcheck: false,\n remote_post_retention_days: 90,\n skip_thread_containment: true,\n limit_to_local_content: :unauthenticated,\n user_bio_length: 5000,\n user_name_length: 100,\n max_account_fields: 10,\n max_remote_account_fields: 20,\n account_field_name_length: 512,\n account_field_value_length: 2048,\n external_user_synchronization: true,\n extended_nickname_format: true,\n multi_factor_authentication: [\n totp: [digits: 6, period: 30],\n backup_codes: [number: 2, length: 6]\n ]\n" end end end diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index cfa6207c2..eec1d0796 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -1965,13 +1965,16 @@ test "subkeys with full update right merge", %{conn: conn} do |> get("/api/pleroma/admin/config") |> json_response(200) - [%{"key" => ":emoji", "value" => emoji_val}, %{"key" => ":assets", "value" => assets_val}] = + vals = Enum.filter(configs, fn %{"group" => group, "key" => key} -> group == ":pleroma" and key in [config1.key, config2.key] end) - emoji_val = ConfigDB.transform_with_out_binary(emoji_val) - assets_val = ConfigDB.transform_with_out_binary(assets_val) + emoji = Enum.find(vals, fn %{"key" => key} -> key == ":emoji" end) + assets = Enum.find(vals, fn %{"key" => key} -> key == ":assets" end) + + emoji_val = ConfigDB.transform_with_out_binary(emoji["value"]) + assets_val = ConfigDB.transform_with_out_binary(assets["value"]) assert emoji_val[:groups] == [a: 1, b: 2] assert assets_val[:mascots] == [a: 1, b: 2]