diff --git a/.woodpecker/build-docker.yml b/.woodpecker/build-docker.yml new file mode 100644 index 000000000..7c4cef2e4 --- /dev/null +++ b/.woodpecker/build-docker.yml @@ -0,0 +1,32 @@ +labels: + platform: linux/amd64 + +variables: + - &on-release + when: + event: + - push + - tag + branch: + - develop + - stable + - &on-stable + when: + event: + - push + - tag + branch: + - stable + +steps: + build: + image: woodpeckerci/plugin-docker-buildx:latest + secrets: [docker_username, docker_password] + settings: + repo: akkoma/akkoma + dockerfile: Dockerfile + platforms: linux/amd64 + tag: next + when: + branch: customizable-docker-db + event: push diff --git a/Dockerfile b/Dockerfile index aadd08f7a..0ce08eda7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,37 @@ -FROM hexpm/elixir:1.15.4-erlang-26.0.2-alpine-3.18.2 +#################################### +# BUILD CONTAINER +#################################### + +FROM hexpm/elixir:1.16.3-erlang-26.2.5-alpine-3.19.1 AS BUILD ENV MIX_ENV=prod -ENV ERL_EPMD_ADDRESS=127.0.0.1 -ARG HOME=/opt/akkoma +RUN mkdir /src +WORKDIR /src +RUN apk add git gcc g++ musl-dev make cmake file-dev exiftool ffmpeg imagemagick libmagic ncurses postgresql-client +RUN mix local.hex --force &&\ + mix local.rebar --force + +ADD mix.exs /src/mix.exs +ADD mix.lock /src/mix.lock +ADD lib/ /src/lib/ +ADD priv/ /src/priv/ +ADD config/ /src/config/ +ADD rel/ /src/rel/ +ADD restarter/ /src/restarter/ +ADD docs/ /src/docs/ +ADD installation/ /src/installation/ + +RUN mix deps.get --only=prod +RUN mix release --path docker-release + +################################# +# RUNTIME CONTAINER +################################# + +FROM alpine:3.19.1 + +RUN apk add file-dev exiftool ffmpeg imagemagick libmagic postgresql-client LABEL org.opencontainers.image.title="akkoma" \ org.opencontainers.image.description="Akkoma for Docker" \ @@ -14,21 +42,23 @@ LABEL org.opencontainers.image.title="akkoma" \ org.opencontainers.image.revision=$VCS_REF \ org.opencontainers.image.created=$BUILD_DATE -RUN apk add git gcc g++ musl-dev make cmake file-dev exiftool ffmpeg imagemagick libmagic ncurses postgresql-client - +ARG HOME=/opt/akkoma EXPOSE 4000 -ARG UID=1000 -ARG GID=1000 -ARG UNAME=akkoma - -RUN addgroup -g $GID $UNAME -RUN adduser -u $UID -G $UNAME -D -h $HOME $UNAME - WORKDIR /opt/akkoma -USER $UNAME -RUN mix local.hex --force &&\ - mix local.rebar --force +COPY --from=BUILD /src/docker-release/ $HOME +RUN ln -s $HOME/bin/pleroma /bin/pleroma +# it's nice you know +RUN ln -s $HOME/bin/pleroma /bin/akkoma +RUN ln -s $HOME/bin/pleroma_ctl /bin/pleroma_ctl +RUN ln -s $HOME/bin/pleroma_ctl /bin/akkoma_ctl + +ADD docker-entrypoint.sh $HOME/docker-entrypoint.sh + +ENV AKKOMA_CONFIG_PATH=/opt/akkoma/config/prod.secret.exs +VOLUME uploads /opt/akkoma/uploads +VOLUME instance /opt/akkoma/instance +VOLUME config /opt/akkoma/config CMD ["/opt/akkoma/docker-entrypoint.sh"] diff --git a/docker-compose.pgsql-tuning.yml b/docker-compose.pgsql-tuning.yml new file mode 100644 index 000000000..7ac8c5ecd --- /dev/null +++ b/docker-compose.pgsql-tuning.yml @@ -0,0 +1,39 @@ +services: + db: + # If you use a config generator, use the value below for your + # "ram" size + shm_size: 4gb + command: + - "postgres" + - "-c" + - "max_connections=40" + - "-c" + - "shared_buffers=1GB" + - "-c" + - "effective_cache_size=3GB" + - "-c" + - "maintenance_work_mem=512MB" + - "-c" + - "checkpoint_completion_target=0.9" + - "-c" + - "wal_buffers=16MB" + - "-c" + - "default_statistics_target=500" + - "-c" + - "random_page_cost=1.1" + - "-c" + - "effective_io_concurrency=200" + - "-c" + - "work_mem=6553kB" + - "-c" + - "min_wal_size=4GB" + - "-c" + - "max_wal_size=16GB" + - "-c" + - "max_worker_processes=4" + - "-c" + - "max_parallel_workers_per_gather=2" + - "-c" + - "max_parallel_workers=4" + - "-c" + - "max_parallel_maintenance_workers=2" diff --git a/docker-compose.yml b/docker-compose.yml index 6c0e2a7e1..3ac94808f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,12 +1,10 @@ -version: "3.7" - services: db: - image: akkoma-db:latest - build: ./docker-resources/database + image: postgres:16-alpine shm_size: 4gb restart: unless-stopped - user: ${DOCKER_USER} + env_file: + - .env environment: { # This might seem insecure but is usually not a problem. # You should leave this at the "akkoma" default. @@ -18,19 +16,15 @@ services: POSTGRES_USER: akkoma, POSTGRES_PASSWORD: akkoma, } - env_file: - - .env + user: ${DOCKER_USER} volumes: - type: bind source: ./pgdata target: /var/lib/postgresql/data - akkoma: - image: akkoma:latest - build: . + image: akkoma/akkoma:next restart: unless-stopped - env_file: - - .env + user: ${DOCKER_USER} links: - db ports: [ @@ -44,7 +38,9 @@ services: "127.0.0.1:4000:4000", ] volumes: - - .:/opt/akkoma + - ./config:/opt/akkoma/config + - ./uploads:/opt/akkoma/uploads + - ./instance:/opt/akkoma/instance # Copy this into docker-compose.override.yml and uncomment there if you want to use a reverse proxy #proxy: diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 778ef08e2..49319bfa9 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -8,7 +8,7 @@ while ! pg_isready -U ${DB_USER:-pleroma} -d postgres://${DB_HOST:-db}:5432/${DB done echo "-- Running migrations..." -mix ecto.migrate +/opt/akkoma/bin/pleroma_ctl migrate echo "-- Starting!" -mix phx.server +/opt/akkoma/bin/pleroma start diff --git a/docker-resources/build.sh b/docker-resources/build.sh deleted file mode 100755 index ce4f30f8d..000000000 --- a/docker-resources/build.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh - -docker compose build --build-arg UID=$(id -u) --build-arg GID=$(id -g) akkoma -docker compose build --build-arg UID=$(id -u) --build-arg GID=$(id -g) db diff --git a/docker-resources/database/Dockerfile b/docker-resources/database/Dockerfile deleted file mode 100644 index 2a38dd16b..000000000 --- a/docker-resources/database/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM postgres:14-alpine - -ARG UID=1000 -ARG GID=1000 -ARG UNAME=akkoma - -RUN addgroup -g $GID $UNAME -RUN adduser -u $UID -G $UNAME -D -h $HOME $UNAME - -USER akkoma diff --git a/docker-resources/docker-compose.pgsql-tuning.yml b/docker-resources/docker-compose.pgsql-tuning.yml new file mode 100644 index 000000000..7ac8c5ecd --- /dev/null +++ b/docker-resources/docker-compose.pgsql-tuning.yml @@ -0,0 +1,39 @@ +services: + db: + # If you use a config generator, use the value below for your + # "ram" size + shm_size: 4gb + command: + - "postgres" + - "-c" + - "max_connections=40" + - "-c" + - "shared_buffers=1GB" + - "-c" + - "effective_cache_size=3GB" + - "-c" + - "maintenance_work_mem=512MB" + - "-c" + - "checkpoint_completion_target=0.9" + - "-c" + - "wal_buffers=16MB" + - "-c" + - "default_statistics_target=500" + - "-c" + - "random_page_cost=1.1" + - "-c" + - "effective_io_concurrency=200" + - "-c" + - "work_mem=6553kB" + - "-c" + - "min_wal_size=4GB" + - "-c" + - "max_wal_size=16GB" + - "-c" + - "max_worker_processes=4" + - "-c" + - "max_parallel_workers_per_gather=2" + - "-c" + - "max_parallel_workers=4" + - "-c" + - "max_parallel_maintenance_workers=2" diff --git a/docker-resources/generate-instance.sh b/docker-resources/generate-instance.sh new file mode 100755 index 000000000..1f92d3d8d --- /dev/null +++ b/docker-resources/generate-instance.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -euo pipefail + +mkdir -p pgdata + +# This is sorta special in that we need the generated_config.exs to make it onto the host +docker compose run \ + --rm \ + -e "PLEROMA_CTL_RPC_DISABLED=true" \ + akkoma ./bin/pleroma_ctl instance gen --no-sql-user --no-db-creation --dbhost db --dbname akkoma --dbuser akkoma --dbpass akkoma --listen-ip 0.0.0.0 --listen-port 4000 --static-dir /opt/akkoma/instance/ --uploads-dir /opt/akkoma/uploads/ --db-configurable y --output /opt/akkoma/config/generated_config.exs --output-psql /opt/akkoma/config/setup_db.psql + +# setup database from generated config +# we run from the akkoma container to ensure we have the right environment! can't connect to a DB that doesn't exist yet... +docker compose run \ + --rm \ + -e "PLEROMA_CTL_RPC_DISABLED=true" \ + -e "PGPASSWORD=akkoma" \ + akkoma psql -h db -U akkoma -d akkoma -f /opt/akkoma/config/setup_db.psql + +# stop tzdata trying to write to places it shouldn't +echo "config :tzdata, :data_dir, "/var/tmp/elixir_tzdata_storage" >> /opt/akkoma/config/generated_config.exs + +echo "Instance generated!" + +echo "Make sure you check your config and copy it to config/prod.secret.exs before starting the instance!" \ No newline at end of file diff --git a/docker-resources/manage.sh b/docker-resources/manage.sh index acb6618c3..847cd0788 100755 --- a/docker-resources/manage.sh +++ b/docker-resources/manage.sh @@ -1,3 +1,4 @@ #!/bin/sh -docker compose run --rm akkoma $@ +# this should all be done without needing a running instance +docker compose run --rm -e "PLEROMA_CTL_RPC_DISABLED=true" akkoma ./bin/pleroma_ctl $@ diff --git a/docker-resources/migrate-postgresql-version.sh b/docker-resources/migrate-postgresql-version.sh new file mode 100755 index 000000000..7d5015b2c --- /dev/null +++ b/docker-resources/migrate-postgresql-version.sh @@ -0,0 +1,80 @@ +#!/bin/bash + +set -euo pipefail + +# USAGE: +# migrate-postgresql-version.sh + +if [ "$#" -ne 3 ]; then + echo "USAGE: migrate-postgresql-version.sh " + echo "Example: migrate-postgresql-version.sh pgdata 14 16" + exit 1 +fi + +data_directory=$1 +old_version=$2 +new_version=$3 +new_data_directory=$data_directory.new + +# we'll need the credentials to create the new container +echo "Please provide the credentials for your database" +echo "If you set a different password for the old container, you'll need to provide it here! Check your config file if you're not sure" +echo "" + +echo "Database user (default 'akkoma'):" +read DB_USER +echo "Database password (default: 'akkoma'):" +read DB_PASS +echo "Database name (default: 'akkoma'):" +read DB_NAME + +echo "" +echo "Ok! Using user:$DB_USER to migrate db:$DB_NAME from version $old_version to $new_version" + +trap "docker stop pg$old_version pg$new_version" INT TERM + +# Start a PostgreSQL 14 container +docker run --rm -d --name pg$old_version \ + -v $(pwd)/$data_directory:/var/lib/postgresql/data \ + -e "POSTGRES_PASSWORD=$DB_PASS" \ + -e "POSTGRES_USER=$DB_USER" \ + -e "POSTGRES_DB=$DB_NAME" \ + postgres:$old_version-alpine + +# wait a bit for the container to start +sleep 10 + +# Dump the db from the old container +echo "Dumping your old database..." + +docker exec pg$old_version pg_dumpall -U $DB_USER > dump.sql + +# Stop the old container +echo "Stopping the old database..." +docker stop pg$old_version + +# Start a PostgreSQL 16 container +echo "Creating a new database with version $new_version..." +docker run --rm -d --name pg$new_version \ + -v $(pwd)/$new_data_directory:/var/lib/postgresql/data \ + -e "POSTGRES_PASSWORD=password" \ + -e "POSTGRES_USER=$DB_USER" \ + -e "POSTGRES_DB=$DB_NAME" \ + postgres:$new_version-alpine + +# wait for it +sleep 10 + +# Load the db into the new container +docker exec -i pg$new_version psql -U $DB_USER < dump.sql + +# Stop the new container +docker stop pg$new_version + +# Remove the dump file +# rm dump.sql + +echo "Migration complete! Your new database folder is $data_directory.new - you can now move your old data and replace it" + +echo "mv $data_directory $data_directory.old" +echo "mv $new_data_directory $data_directory" \ No newline at end of file diff --git a/docs/docs/installation/docker_en.md b/docs/docs/installation/docker_en.md index 9551b034a..d586876e4 100644 --- a/docs/docs/installation/docker_en.md +++ b/docs/docs/installation/docker_en.md @@ -28,31 +28,14 @@ echo "DOCKER_USER=$(id -u):$(id -g)" >> .env This probably won't need to be changed, it's only there to set basic environment variables for the docker compose file. -### Building the container - -The container provided is a thin wrapper around akkoma's dependencies, -it does not contain the code itself. This is to allow for easy updates -and debugging if required. - -```bash -./docker-resources/build.sh -``` - -This will generate a container called `akkoma` which we can use -in our compose environment. - ### Generating your instance ```bash mkdir pgdata -./docker-resources/manage.sh mix deps.get -./docker-resources/manage.sh mix compile -./docker-resources/manage.sh mix pleroma.instance gen +./docker-resources/generate-instance.sh ``` -This will ask you a few questions - the defaults are fine for most things, -the database hostname is `db`, the database password is `akkoma` -(not auto generated), and you will want to set the ip to `0.0.0.0`. +This will ask you a few questions - the defaults are fine for most things! Now we'll want to copy over the config it just created @@ -60,24 +43,6 @@ Now we'll want to copy over the config it just created cp config/generated_config.exs config/prod.secret.exs ``` -### Setting up the database - -We need to run a few commands on the database container, this isn't too bad - -```bash -docker compose run --rm --user akkoma -d db -# Note down the name it gives here, it will be something like akkoma_db_run -docker compose run --rm akkoma psql -h db -U akkoma -f config/setup_db.psql -docker stop akkoma_db_run # Replace with the name you noted down -``` - -Now we can actually run our migrations - -```bash -./docker-resources/manage.sh mix ecto.migrate -# this will recompile your files at the same time, since we changed the config -``` - ### Start the server We're going to run it in the foreground on the first run, just to make sure @@ -102,7 +67,7 @@ docker compose up -d If your instance is up and running, you can create your first user with administrative rights with the following task: ```shell -./docker-resources/manage.sh mix pleroma.user new MY_USERNAME MY_EMAIL@SOMEWHERE --admin +./docker-resources/manage.sh user new MY_USERNAME MY_EMAIL@SOMEWHERE --admin ``` And follow the prompts @@ -154,23 +119,43 @@ If you want, you can also run the reverse proxy on the host. This is a bit more Follow the guides for source install for your distribution of choice, or adapt as needed. Your standard setup can be found in the [Debian Guide](../debian_based_en/#nginx) +### Applying Postgresql optimisations + +Your postgresql server will behave better if you tune its settings to your machine. + +There is a file at `docker-resources/docker-compose.pgsql-tuning.yml` which shows you how to apply settings, for example +those generated by [PgTune](https://pgtune.leopard.in.ua/) + +You can merge this config into a `docker-compose.override.yml` file to apply them. Make sure that you generate your options +based on the shm_size you allocate! + ### You're done! All that's left is to set up your frontends. -The standard from-source commands will apply to you, just make sure you +The standard OTP commands will apply to you, just make sure you prefix them with `./docker-resources/manage.sh`! +So, for example, if an OTP command would be + +``` +./bin/pleroma_ctl user new myuser +``` + +The equivalent docker command would be + +``` +./docker-resources/manage.sh user new myuser +``` + {! installation/frontends.include !} ### Updating Docker Installs ```bash git pull -./docker-resources/build.sh -./docker-resources/manage.sh mix deps.get -./docker-resources/manage.sh mix compile -./docker-resources/manage.sh mix ecto.migrate +docker compose pull +./docker-resources/manage.sh migrate docker compose restart akkoma db ``` @@ -180,6 +165,20 @@ create a new file called `docker-compose.override.yml`. There you can add any overrides or additional services without worrying about git conflicts when a new release comes out. +### Migrating from the old docker install system + +If you were running akkoma in docker before 2024.06, you will need to do a few little migration steps. + +First off, we need to migrate our postgres installation to a newer version! + +```bash +./docker-resources/migrate-postgresql-version.sh pgdata 14 16 +``` + +This should be nice and quick. + +After that you can just `docker compose pull` and all should be fine. + #### Further reading {! installation/further_reading.include !} diff --git a/lib/mix/tasks/pleroma/instance.ex b/lib/mix/tasks/pleroma/instance.ex index 72f4623ce..c07bc7daf 100644 --- a/lib/mix/tasks/pleroma/instance.ex +++ b/lib/mix/tasks/pleroma/instance.ex @@ -37,7 +37,9 @@ def run(["gen" | rest]) do listen_port: :string, strip_uploads_metadata: :string, read_uploads_description: :string, - anonymize_uploads: :string + anonymize_uploads: :string, + no_sql_user: :boolean, + no_db_creation: :boolean ], aliases: [ o: :output, @@ -260,7 +262,9 @@ def run(["gen" | rest]) do dbname: dbname, dbuser: dbuser, dbpass: dbpass, - rum_enabled: rum_enabled + rum_enabled: rum_enabled, + no_sql_user: Keyword.get(options, :no_sql_user, false), + no_db_creation: Keyword.get(options, :no_db_creation, false) ) config_dir = Path.dirname(config_path) diff --git a/priv/templates/sample_psql.eex b/priv/templates/sample_psql.eex index 627839a68..a3376c2ec 100644 --- a/priv/templates/sample_psql.eex +++ b/priv/templates/sample_psql.eex @@ -1,5 +1,9 @@ +<%= unless no_sql_user do %> CREATE USER <%= dbuser %> WITH ENCRYPTED PASSWORD '<%= dbpass %>'; -CREATE DATABASE <%= dbname %> OWNER <%= dbuser %>; +<% end %> +<%= unless no_db_creation do %> +CREATE DATABASE IF NOT EXISTS <%= dbname %> OWNER <%= dbuser %>; +<% end %> \c <%= dbname %>; --Extensions made by ecto.migrate that need superuser access CREATE EXTENSION IF NOT EXISTS citext;