Merge branch 'frontend-bundles-downloads' into 'develop'

frontend install mix tasks

See merge request pleroma/pleroma!2841

CHANGELOG.md

@@ -55,6 +55,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ### Added
 
+- Frontends: Add mix task to install frontends.
+- Frontends: Add configurable frontends for primary and admin fe.
 - Configuration: Added a blacklist for email servers.
 - Chats: Added `accepts_chat_messages` field to user, exposed in APIs and federation.
 - Chats: Added support for federated chats. For details, see the docs.

config/config.exs

@@ -671,7 +671,50 @@
 # With no frontend configuration, the bundled files from the `static` directory will
 # be used.
 #
-# config :pleroma, :frontends, primary: %{"name" => "pleroma", "ref" => "develop"}
+# config :pleroma, :frontends, 
+# primary: %{"name" => "pleroma-fe", "ref" => "develop"},
+# admin: %{"name" => "admin-fe", "ref" => "stable"},
+# available: %{...}
+
+config :pleroma, :frontends,
+  available: %{
+    "kenoma" => %{
+      "name" => "kenoma",
+      "git" => "https://git.pleroma.social/lambadalambda/kenoma",
+      "build_url" =>
+        "https://git.pleroma.social/lambadalambda/kenoma/-/jobs/artifacts/${ref}/download?job=build",
+      "ref" => "master"
+    },
+    "pleroma-fe" => %{
+      "name" => "pleroma-fe",
+      "git" => "https://git.pleroma.social/pleroma/pleroma-fe",
+      "build_url" =>
+        "https://git.pleroma.social/pleroma/pleroma-fe/-/jobs/artifacts/${ref}/download?job=build",
+      "ref" => "develop"
+    },
+    "fedi-fe" => %{
+      "name" => "fedi-fe",
+      "git" => "https://git.pleroma.social/pleroma/fedi-fe",
+      "build_url" =>
+        "https://git.pleroma.social/pleroma/fedi-fe/-/jobs/artifacts/${ref}/download?job=build",
+      "ref" => "master"
+    },
+    "admin-fe" => %{
+      "name" => "admin-fe",
+      "git" => "https://git.pleroma.social/pleroma/admin-fe",
+      "build_url" =>
+        "https://git.pleroma.social/pleroma/admin-fe/-/jobs/artifacts/${ref}/download?job=build",
+      "ref" => "develop"
+    },
+    "soapbox-fe" => %{
+      "name" => "soapbox-fe",
+      "git" => "https://gitlab.com/soapbox-pub/soapbox-fe",
+      "build_url" =>
+        "https://gitlab.com/soapbox-pub/soapbox-fe/-/jobs/artifacts/${ref}/download?job=build-production",
+      "ref" => "v1.0.0",
+      "build_dir" => "static"
+    }
+  }
 
 config :pleroma, :web_cache_ttl,
   activity_pub: nil,

config/description.exs

@@ -12,6 +12,55 @@
   compress: false
 ]
 
+installed_frontend_options = [
+  %{
+    key: "name",
+    label: "Name",
+    type: :string,
+    description:
+      "Name of the installed frontend. Valid config must include both `Name` and `Reference` values."
+  },
+  %{
+    key: "ref",
+    label: "Reference",
+    type: :string,
+    description:
+      "Reference of the installed frontend to be used. Valid config must include both `Name` and `Reference` values."
+  }
+]
+
+frontend_options = [
+  %{
+    key: "name",
+    label: "Name",
+    type: :string,
+    description: "Name of the frontend."
+  },
+  %{
+    key: "ref",
+    label: "Reference",
+    type: :string,
+    description: "Reference of the frontend to be used."
+  },
+  %{
+    key: "git",
+    type: :string,
+    description: "URL of the git repository of the frontend"
+  },
+  %{
+    key: "build_url",
+    type: :string,
+    description:
+      "Either an url to a zip file containing the frontend or a template to build it by inserting the `ref`. The string `${ref}` will be replaced by the configured `ref`.",
+    example: "https://some.url/builds/${ref}.zip"
+  },
+  %{
+    key: "build_dir",
+    type: :string,
+    description: "The directory inside the zip file "
+  }
+]
+
 config :pleroma, :config_description, [
   %{
     group: :pleroma,
@@ -3553,21 +3602,21 @@
         key: :primary,
         type: :map,
         description: "Primary frontend, the one that is served for all pages by default",
-        children: [
-          %{
-            key: "name",
-            label: "Name",
-            type: :string,
-            description:
-              "Name of the installed primary frontend. Valid config must include both `Name` and `Reference` values."
+        children: installed_frontend_options
       },
       %{
-            key: "ref",
-            label: "Reference",
-            type: :string,
+        key: :admin,
+        type: :map,
+        description: "Admin frontend",
+        children: installed_frontend_options
+      },
+      %{
+        key: :available,
+        type: :map,
         description:
-              "Reference of the installed primary frontend to be used. Valid config must include both `Name` and `Reference` values."
-          }
+          "A map containing available frontends and parameters for their installation.",
+        children: [
+          frontend_options
         ]
       }
     ]

docs/administration/CLI_tasks/frontend.md

@@ -0,0 +1,69 @@
+# Managing frontends
+
+`mix pleroma.frontend install <frontend> [--ref <ref>] [--file <file>] [--build-url <build-url>] [--path <path>] [--build-dir <build-dir>]`
+
+Frontend can be installed either from local zip file, or automatically downloaded from the web.
+
+You can give all the options directly on the command like, but missing information will be filled out by looking at the data configured under `frontends.available` in the config files.
+
+Currently known `<frontend>` values are:
+- [admin-fe](https://git.pleroma.social/pleroma/admin-fe)
+- [kenoma](http://git.pleroma.social/lambadalambda/kenoma)
+- [pleroma-fe](http://git.pleroma.social/pleroma/pleroma-fe)
+- [fedi-fe](https://git.pleroma.social/pleroma/fedi-fe)
+- [soapbox-fe](https://gitlab.com/soapbox-pub/soapbox-fe)
+
+You can still install frontends that are not configured, see below.
+
+## Example installations for a known frontend
+
+For a frontend configured under the `available` key, it's enough to install it by name.
+
+```sh tab="OTP"
+./bin/pleroma_ctl frontend install pleroma
+```
+
+```sh tab="From Source"
+mix pleroma.frontend install pleroma
+```
+
+This will download the latest build for the the pre-configured `ref` and install it. It can then be configured as the one of the served frontends in the config file (see `primary` or `admin`).
+
+You can override any of the details. To install a pleroma build from a different url, you could do this:
+
+```sh tab="OPT"
+./bin/pleroma_ctl frontend install pleroma --ref 2hu_edition --build-url https://example.org/raymoo.zip
+```
+
+```sh tab="From Source"
+mix pleroma.frontend install pleroma --ref 2hu_edition --build-url https://example.org/raymoo.zip
+```
+
+Similarly, you can also install from a local zip file.
+
+```sh tab="OTP"
+./bin/pleroma_ctl frontend install pleroma --ref mybuild --file ~/Downloads/doomfe.zip
+```
+
+```sh tab="From Source"
+mix pleroma.frontend install pleroma --ref mybuild --file ~/Downloads/doomfe.zip
+```
+
+The resulting frontend will always be installed into a folder of this template: `${instance_static}/frontends/${name}/${ref}`
+
+Careful: This folder will be completely replaced on installation
+
+## Example installation for an unknown frontend
+
+The installation process is the same, but you will have to give all the needed options on the commond line. For example:
+
+```sh tab="OTP"
+./bin/pleroma_ctl frontend install gensokyo --ref master --build-url https://gensokyo.2hu/builds/marisa.zip
+```
+
+```sh tab="From Source"
+mix pleroma.frontend install gensokyo --ref master --build-url https://gensokyo.2hu/builds/marisa.zip
+```
+
+If you don't have a zip file but just want to install a frontend from a local path, you can simply copy the files over a folder of this template: `${instance_static}/frontends/${name}/${ref}`
+

docs/configuration/cheatsheet.md

@@ -1070,11 +1070,11 @@ Control favicons for instances.
 
 Frontends in Pleroma are swappable - you can specify which one to use here.
 
-For now, you can set a frontend with the key `primary` and the options of `name` and `ref`. This will then make Pleroma serve the frontend from a folder constructed by concatenating the instance static path, `frontends` and the name and ref.
+You can set a frontends for the key `primary` and `admin` and the options of `name` and `ref`. This will then make Pleroma serve the frontend from a folder constructed by concatenating the instance static path, `frontends` and the name and ref.
 
-The key `primary` refers to the frontend that will be served by default for general requests. In the future, other frontends like the admin frontend will also be configurable here.
+The key `primary` refers to the frontend that will be served by default for general requests. The key `admin` refers to the frontend that will be served at the `/pleroma/admin` path.
 
-If you don't set anything here, the bundled frontend will be used.
+If you don't set anything here, the bundled frontends will be used.
 
 Example:
 
@@ -1083,6 +1083,10 @@ config :pleroma, :frontends,
   primary: %{
     "name" => "pleroma",
     "ref" => "stable"
+  },
+  admin: %{
+    "name" => "admin",
+    "ref" => "develop"
   }
 ```
 

lib/mix/tasks/pleroma/frontend.ex

@@ -0,0 +1,140 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mix.Tasks.Pleroma.Frontend do
+  use Mix.Task
+
+  import Mix.Pleroma
+
+  @shortdoc "Manages bundled Pleroma frontends"
+
+  @moduledoc File.read!("docs/administration/CLI_tasks/frontend.md")
+
+  def run(["install", "none" | _args]) do
+    shell_info("Skipping frontend installation because none was requested")
+    "none"
+  end
+
+  def run(["install", frontend | args]) do
+    log_level = Logger.level()
+    Logger.configure(level: :warn)
+    start_pleroma()
+
+    {options, [], []} =
+      OptionParser.parse(
+        args,
+        strict: [
+          ref: :string,
+          static_dir: :string,
+          build_url: :string,
+          build_dir: :string,
+          file: :string
+        ]
+      )
+
+    instance_static_dir =
+      with nil <- options[:static_dir] do
+        Pleroma.Config.get!([:instance, :static_dir])
+      end
+
+    cmd_frontend_info = %{
+      "name" => frontend,
+      "ref" => options[:ref],
+      "build_url" => options[:build_url],
+      "build_dir" => options[:build_dir]
+    }
+
+    config_frontend_info = Pleroma.Config.get([:frontends, :available, frontend], %{})
+
+    frontend_info =
+      Map.merge(config_frontend_info, cmd_frontend_info, fn _key, config, cmd ->
+        # This only overrides things that are actually set
+        cmd || config
+      end)
+
+    ref = frontend_info["ref"]
+
+    unless ref do
+      raise "No ref given or configured"
+    end
+
+    dest =
+      Path.join([
+        instance_static_dir,
+        "frontends",
+        frontend,
+        ref
+      ])
+
+    fe_label = "#{frontend} (#{ref})"
+
+    tmp_dir = Path.join(dest, "tmp")
+
+    with {_, :ok} <-
+           {:download_or_unzip, download_or_unzip(frontend_info, tmp_dir, options[:file])},
+         shell_info("Installing #{fe_label} to #{dest}"),
+         :ok <- install_frontend(frontend_info, tmp_dir, dest) do
+      File.rm_rf!(tmp_dir)
+      shell_info("Frontend #{fe_label} installed to #{dest}")
+
+      Logger.configure(level: log_level)
+    else
+      {:download_or_unzip, _} ->
+        shell_info("Could not download or unzip the frontend")
+
+      _e ->
+        shell_info("Could not install the frontend")
+    end
+  end
+
+  defp download_or_unzip(frontend_info, temp_dir, file) do
+    if file do
+      with {:ok, zip} <- File.read(Path.expand(file)) do
+        unzip(zip, temp_dir)
+      end
+    else
+      download_build(frontend_info, temp_dir)
+    end
+  end
+
+  def unzip(zip, dest) do
+    with {:ok, unzipped} <- :zip.unzip(zip, [:memory]) do
+      File.rm_rf!(dest)
+      File.mkdir_p!(dest)
+
+      Enum.each(unzipped, fn {filename, data} ->
+        path = filename
+
+        new_file_path = Path.join(dest, path)
+
+        new_file_path
+        |> Path.dirname()
+        |> File.mkdir_p!()
+
+        File.write!(new_file_path, data)
+      end)
+
+      :ok
+    end
+  end
+
+  defp download_build(frontend_info, dest) do
+    shell_info("Downloading pre-built bundle for #{frontend_info["name"]}")
+    url = String.replace(frontend_info["build_url"], "${ref}", frontend_info["ref"])
+
+    with {:ok, %{status: 200, body: zip_body}} <-
+           Pleroma.HTTP.get(url, [], timeout: 120_000, recv_timeout: 120_000) do
+      unzip(zip_body, dest)
+    else
+      e -> {:error, e}
+    end
+  end
+
+  defp install_frontend(frontend_info, source, dest) do
+    from = frontend_info["build_dir"] || "dist"
+    File.mkdir_p!(dest)
+    File.cp_r!(Path.join([source, from]), dest)
+    :ok
+  end
+end

lib/pleroma/plugs/frontend_static.ex

@@ -30,6 +30,7 @@ def init(opts) do
     opts
     |> Keyword.put(:from, "__unconfigured_frontend_static_plug")
     |> Plug.Static.init()
+    |> Map.put(:frontend_type, opts[:frontend_type])
   end
 
   def call(conn, opts) do

lib/pleroma/web/endpoint.ex

@@ -39,6 +39,18 @@ defmodule Pleroma.Web.Endpoint do
     }
   )
 
+  plug(Plug.Static.IndexHtml, at: "/pleroma/admin/")
+
+  plug(Pleroma.Plugs.FrontendStatic,
+    at: "/pleroma/admin",
+    frontend_type: :admin,
+    gzip: true,
+    cache_control_for_etags: @static_cache_control,
+    headers: %{
+      "cache-control" => @static_cache_control
+    }
+  )
+
   # Serve at "/" the static files from "priv/static" directory.
   #
   # You should set gzip to true if you are running phoenix.digest
@@ -56,8 +68,6 @@ defmodule Pleroma.Web.Endpoint do
     }
   )
 
-  plug(Plug.Static.IndexHtml, at: "/pleroma/admin/")
-
   plug(Plug.Static,
     at: "/pleroma/admin/",
     from: {:pleroma, "priv/static/adminfe/"}

test/fixtures/tesla_mock/dist/test.txt

@@ -0,0 +1 @@
+this is a text file

test/plugs/frontend_static_test.exs

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.FrontendStaticPlugTest do
+  alias Pleroma.Plugs.FrontendStatic
   use Pleroma.Web.ConnCase
 
   @dir "test/tmp/instance_static"
@@ -14,6 +15,18 @@ defmodule Pleroma.Web.FrontendStaticPlugTest do
 
   setup do: clear_config([:instance, :static_dir], @dir)
 
+  test "init will give a static plug config + the frontend type" do
+    opts =
+      [
+        at: "/admin",
+        frontend_type: :admin
+      ]
+      |> FrontendStatic.init()
+
+    assert opts[:at] == ["admin"]
+    assert opts[:frontend_type] == :admin
+  end
+
   test "overrides existing static files", %{conn: conn} do
     name = "pelmora"
     ref = "uguu"
@@ -27,4 +40,18 @@ test "overrides existing static files", %{conn: conn} do
     index = get(conn, "/")
     assert html_response(index, 200) == "from frontend plug"
   end
+
+  test "overrides existing static files for the `pleroma/admin` path", %{conn: conn} do
+    name = "pelmora"
+    ref = "uguu"
+
+    clear_config([:frontends, :admin], %{"name" => name, "ref" => ref})
+    path = "#{@dir}/frontends/#{name}/#{ref}"
+
+    File.mkdir_p!(path)
+    File.write!("#{path}/index.html", "from frontend plug")
+
+    index = get(conn, "/pleroma/admin/")
+    assert html_response(index, 200) == "from frontend plug"
+  end
 end

test/tasks/frontend_test.exs

@@ -0,0 +1,78 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.FrontendTest do
+  use Pleroma.DataCase
+  alias Mix.Tasks.Pleroma.Frontend
+
+  import ExUnit.CaptureIO, only: [capture_io: 1]
+
+  @dir "test/frontend_static_test"
+
+  setup do
+    File.mkdir_p!(@dir)
+    clear_config([:instance, :static_dir], @dir)
+
+    on_exit(fn ->
+      File.rm_rf(@dir)
+    end)
+  end
+
+  test "it downloads and unzips a known frontend" do
+    clear_config([:frontends, :available], %{
+      "pleroma" => %{
+        "ref" => "fantasy",
+        "name" => "pleroma",
+        "build_url" => "http://gensokyo.2hu/builds/${ref}"
+      }
+    })
+
+    Tesla.Mock.mock(fn %{url: "http://gensokyo.2hu/builds/fantasy"} ->
+      %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/frontend_dist.zip")}
+    end)
+
+    capture_io(fn ->
+      Frontend.run(["install", "pleroma"])
+    end)
+
+    assert File.exists?(Path.join([@dir, "frontends", "pleroma", "fantasy", "test.txt"]))
+  end
+
+  test "it also works given a file" do
+    clear_config([:frontends, :available], %{
+      "pleroma" => %{
+        "ref" => "fantasy",
+        "name" => "pleroma",
+        "build_dir" => ""
+      }
+    })
+
+    capture_io(fn ->
+      Frontend.run(["install", "pleroma", "--file", "test/fixtures/tesla_mock/frontend.zip"])
+    end)
+
+    assert File.exists?(Path.join([@dir, "frontends", "pleroma", "fantasy", "test.txt"]))
+  end
+
+  test "it downloads and unzips unknown frontends" do
+    Tesla.Mock.mock(fn %{url: "http://gensokyo.2hu/madeup.zip"} ->
+      %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/frontend.zip")}
+    end)
+
+    capture_io(fn ->
+      Frontend.run([
+        "install",
+        "unknown",
+        "--ref",
+        "baka",
+        "--build-url",
+        "http://gensokyo.2hu/madeup.zip",
+        "--build-dir",
+        ""
+      ])
+    end)
+
+    assert File.exists?(Path.join([@dir, "frontends", "unknown", "baka", "test.txt"]))
+  end
+end