From a2f57bd82b1b495a754516231b56e53ae41c6b69 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 27 May 2020 16:27:07 -0500 Subject: [PATCH 1/6] Permit easy access to vaccum full and analyze via a mix task --- lib/mix/tasks/pleroma/database.ex | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index 778de162f..c4f343f04 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -135,4 +135,30 @@ def run(["fix_likes_collections"]) do end) |> Stream.run() end + + def run(["vacuum", args]) do + start_pleroma() + + case args do + "analyze" -> + Logger.info("Runnning VACUUM ANALYZE.") + Repo.query!( + "vacuum analyze;", + [], + timeout: :infinity + ) + + "full" -> + Logger.info("Runnning VACUUM FULL. This could take a while.") + + Repo.query!( + "vacuum full;", + [], + timeout: :infinity + ) + + _ -> + Logger.error("Error: invalid vacuum argument.") + end + end end From 73ca57e4f1620ddaf167c368f48a0096b2096a96 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 27 May 2020 16:27:29 -0500 Subject: [PATCH 2/6] Make it obvious a full vacuum can take a while --- lib/mix/tasks/pleroma/database.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index c4f343f04..1fdafcc88 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -34,7 +34,7 @@ def run(["remove_embedded_objects" | args]) do ) if Keyword.get(options, :vacuum) do - Logger.info("Runnning VACUUM FULL") + Logger.info("Runnning VACUUM FULL. This could take a while.") Repo.query!( "vacuum full;", @@ -94,7 +94,7 @@ def run(["prune_objects" | args]) do |> Repo.delete_all(timeout: :infinity) if Keyword.get(options, :vacuum) do - Logger.info("Runnning VACUUM FULL") + Logger.info("Runnning VACUUM FULL. This could take a while.") Repo.query!( "vacuum full;", From 0d57e066260234fb582a63870cbae7517e7b6246 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 27 May 2020 16:31:37 -0500 Subject: [PATCH 3/6] Make clearer that this is time and resource consuming --- lib/mix/tasks/pleroma/database.ex | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index 1fdafcc88..2f1f33469 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -34,7 +34,11 @@ def run(["remove_embedded_objects" | args]) do ) if Keyword.get(options, :vacuum) do - Logger.info("Runnning VACUUM FULL. This could take a while.") + Logger.info("Runnning VACUUM FULL.") + + Logger.warn( + "Re-packing your entire database may take a while and will consume extra disk space during the process." + ) Repo.query!( "vacuum full;", @@ -94,7 +98,11 @@ def run(["prune_objects" | args]) do |> Repo.delete_all(timeout: :infinity) if Keyword.get(options, :vacuum) do - Logger.info("Runnning VACUUM FULL. This could take a while.") + Logger.info("Runnning VACUUM FULL.") + + Logger.warn( + "Re-packing your entire database may take a while and will consume extra disk space during the process." + ) Repo.query!( "vacuum full;", @@ -142,6 +150,7 @@ def run(["vacuum", args]) do case args do "analyze" -> Logger.info("Runnning VACUUM ANALYZE.") + Repo.query!( "vacuum analyze;", [], @@ -149,7 +158,11 @@ def run(["vacuum", args]) do ) "full" -> - Logger.info("Runnning VACUUM FULL. This could take a while.") + Logger.info("Runnning VACUUM FULL.") + + Logger.warn( + "Re-packing your entire database may take a while and will consume extra disk space during the process." + ) Repo.query!( "vacuum full;", From 30f96b19c1850d0dd534edbe66ce19a1c8198729 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 27 May 2020 16:40:51 -0500 Subject: [PATCH 4/6] Abstract out the database maintenance. I'd like to use this from AdminFE too. --- lib/mix/tasks/pleroma/database.ex | 52 +++---------------------------- lib/pleroma/maintenance.ex | 37 ++++++++++++++++++++++ 2 files changed, 41 insertions(+), 48 deletions(-) create mode 100644 lib/pleroma/maintenance.ex diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index 2f1f33469..7049293d9 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -5,6 +5,7 @@ defmodule Mix.Tasks.Pleroma.Database do alias Pleroma.Conversation alias Pleroma.Object + alias Pleroma.Maintenance alias Pleroma.Repo alias Pleroma.User require Logger @@ -34,17 +35,7 @@ def run(["remove_embedded_objects" | args]) do ) if Keyword.get(options, :vacuum) do - Logger.info("Runnning VACUUM FULL.") - - Logger.warn( - "Re-packing your entire database may take a while and will consume extra disk space during the process." - ) - - Repo.query!( - "vacuum full;", - [], - timeout: :infinity - ) + Maintenance.vacuum("full") end end @@ -98,17 +89,7 @@ def run(["prune_objects" | args]) do |> Repo.delete_all(timeout: :infinity) if Keyword.get(options, :vacuum) do - Logger.info("Runnning VACUUM FULL.") - - Logger.warn( - "Re-packing your entire database may take a while and will consume extra disk space during the process." - ) - - Repo.query!( - "vacuum full;", - [], - timeout: :infinity - ) + Maintenance.vacuum("full") end end @@ -147,31 +128,6 @@ def run(["fix_likes_collections"]) do def run(["vacuum", args]) do start_pleroma() - case args do - "analyze" -> - Logger.info("Runnning VACUUM ANALYZE.") - - Repo.query!( - "vacuum analyze;", - [], - timeout: :infinity - ) - - "full" -> - Logger.info("Runnning VACUUM FULL.") - - Logger.warn( - "Re-packing your entire database may take a while and will consume extra disk space during the process." - ) - - Repo.query!( - "vacuum full;", - [], - timeout: :infinity - ) - - _ -> - Logger.error("Error: invalid vacuum argument.") - end + Maintenance.vacuum(args) end end diff --git a/lib/pleroma/maintenance.ex b/lib/pleroma/maintenance.ex new file mode 100644 index 000000000..326c17825 --- /dev/null +++ b/lib/pleroma/maintenance.ex @@ -0,0 +1,37 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Maintenance do + alias Pleroma.Repo + require Logger + + def vacuum(args) do + case args do + "analyze" -> + Logger.info("Runnning VACUUM ANALYZE.") + + Repo.query!( + "vacuum analyze;", + [], + timeout: :infinity + ) + + "full" -> + Logger.info("Runnning VACUUM FULL.") + + Logger.warn( + "Re-packing your entire database may take a while and will consume extra disk space during the process." + ) + + Repo.query!( + "vacuum full;", + [], + timeout: :infinity + ) + + _ -> + Logger.error("Error: invalid vacuum argument.") + end + end +end From 92fba24c743a5d2d9ed78df7499fd3123a6ad6ac Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 27 May 2020 17:17:06 -0500 Subject: [PATCH 5/6] Alpha sort --- lib/mix/tasks/pleroma/database.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index 7049293d9..82e2abdcb 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -4,8 +4,8 @@ defmodule Mix.Tasks.Pleroma.Database do alias Pleroma.Conversation - alias Pleroma.Object alias Pleroma.Maintenance + alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User require Logger From 1b586ff3aece21d277e40f95cc5c60fc15818a87 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 29 May 2020 10:17:06 -0500 Subject: [PATCH 6/6] Document new database vacuum tasks --- docs/administration/CLI_tasks/database.md | 29 +++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/docs/administration/CLI_tasks/database.md b/docs/administration/CLI_tasks/database.md index ff400c8ed..647f6f274 100644 --- a/docs/administration/CLI_tasks/database.md +++ b/docs/administration/CLI_tasks/database.md @@ -69,3 +69,32 @@ mix pleroma.database update_users_following_followers_counts ```sh tab="From Source" mix pleroma.database fix_likes_collections ``` + +## Vacuum the database + +### Analyze + +Running an `analyze` vacuum job can improve performance by updating statistics used by the query planner. **It is safe to cancel this.** + +```sh tab="OTP" +./bin/pleroma_ctl database vacuum analyze +``` + +```sh tab="From Source" +mix pleroma.database vacuum analyze +``` + +### Full + +Running a `full` vacuum job rebuilds your entire database by reading all of the data and rewriting it into smaller +and more compact files with an optimized layout. This process will take a long time and use additional disk space as +it builds the files side-by-side the existing database files. It can make your database faster and use less disk space, +but should only be run if necessary. **It is safe to cancel this.** + +```sh tab="OTP" +./bin/pleroma_ctl database vacuum full +``` + +```sh tab="From Source" +mix pleroma.database vacuum full +``` \ No newline at end of file