Use EXIF data of image to prefill image description

During attachment upload Pleroma returns a "description" field.

* This MR allows Pleroma to read the EXIF data during upload and return the description to the FE using this field.
    * If a description is already present (e.g. because a previous module added it), it will use that
    * Otherwise it will read from the EXIF data. First it will check -ImageDescription, if that's empty, it will check -iptc:Caption-Abstract
    * If no description is found, it will simply return nil, which is the default value
* When people set up a new instance, they will be asked if they want to read metadata and this module will be activated if so

There was an Exiftool module, which has now been renamed to Exiftool.StripLocation

CHANGELOG.md

@@ -9,10 +9,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ## Added
 - Officially supported docker release
 - Ability to remove followers unilaterally without a block
+- Uploadfilter `Pleroma.Upload.Filter.Exiftool.ReadDescription` returns description values to the FE so they can pre fill the image description field
 
 ## Changes
 - Follows no longer override domain blocks, a domain block is final
 - Deletes are now the lowest priority to publish and will be handled after creates
+- Uploadfilter `Pleroma.Upload.Filter.Exiftool` has been renamed to `Pleroma.Upload.Filter.Exiftool.StripLocation`
 
 ## 2022.10
 

docs/docs/administration/CLI_tasks/instance.md

@@ -37,7 +37,8 @@ If any of the options are left unspecified, you will be prompted interactively.
 - `--static-dir <path>` - the directory custom public files should be read from (custom emojis, frontend bundle overrides, robots.txt, etc.)
 - `--listen-ip <ip>` - the ip the app should listen to, defaults to 127.0.0.1
 - `--listen-port <port>` - the port the app should listen to, defaults to 4000
-- `--strip-uploads <Y|N>` - use ExifTool to strip uploads of sensitive location data
+- `--strip-uploads-location <Y|N>` - use ExifTool to strip uploads of sensitive location data
+- `--read-uploads-description <Y|N>` - use ExifTool to read image descriptions from uploads
 - `--anonymize-uploads <Y|N>` - randomize uploaded filenames
 - `--dedupe-uploads <Y|N>` - store files based on their hash to reduce data storage requirements if duplicates are uploaded with different filenames
 - `--skip-release-env` - skip generation the release environment file

docs/docs/configuration/cheatsheet.md

@@ -657,12 +657,18 @@ This filter replaces the filename (not the path) of an upload. For complete obfu
 
 No specific configuration.
 
-#### Pleroma.Upload.Filter.Exiftool
+#### Pleroma.Upload.Filter.Exiftool.StripLocation
 
 This filter only strips the GPS and location metadata with Exiftool leaving color profiles and attributes intact.
 
 No specific configuration.
 
+#### Pleroma.Upload.Filter.Exiftool.ReadDescription
+
+This filter reads the ImageDescription and iptc:Caption-Abstract fields with Exiftool so clients can prefill the media description field.
+
+No specific configuration.
+
 #### Pleroma.Upload.Filter.Mogrify
 
 * `args`: List of actions for the `mogrify` command like `"strip"` or `["strip", "auto-orient", {"implode", "1"}]`.

docs/docs/installation/optional/media_graphics_packages.md

@@ -29,4 +29,5 @@ It is required for the following Pleroma features:
 `exiftool` is media files metadata reader/writer.
 
 It is required for the following Pleroma features:
-  * `Pleroma.Upload.Filters.Exiftool` upload filter (related config: `Plaroma.Upload/filters` in `config/config.exs`)
+    * `Pleroma.Upload.Filters.Exiftool.StripLocation` upload filter (related config: `Plaroma.Upload/filters` in `config/config.exs`)
+    * `Pleroma.Upload.Filters.Exiftool.ReadDescription` upload filter (related config: `Plaroma.Upload/filters` in `config/config.exs`)

lib/mix/tasks/pleroma/instance.ex

@@ -34,7 +34,8 @@ def run(["gen" | rest]) do
           static_dir: :string,
           listen_ip: :string,
           listen_port: :string,
-          strip_uploads: :string,
+          strip_uploads_location: :string,
+          read_uploads_description: :string,
           anonymize_uploads: :string,
           dedupe_uploads: :string
         ],
@@ -161,7 +162,7 @@ def run(["gen" | rest]) do
         )
         |> Path.expand()
 
-      {strip_uploads_message, strip_uploads_default} =
+      {strip_uploads_location_message, strip_uploads_location_default} =
         if Pleroma.Utils.command_available?("exiftool") do
           {"Do you want to strip location (GPS) data from uploaded images? This requires exiftool, it was detected as installed. (y/n)",
            "y"}
@@ -170,12 +171,29 @@ def run(["gen" | rest]) do
            "n"}
         end
 
-      strip_uploads =
+      strip_uploads_location =
         get_option(
           options,
-          :strip_uploads,
+          :strip_uploads_location,
-          strip_uploads_message,
+          strip_uploads_location_message,
-          strip_uploads_default
+          strip_uploads_location_default
+        ) === "y"
+
+      {read_uploads_description_message, read_uploads_description_default} =
+        if Pleroma.Utils.command_available?("exiftool") do
+          {"Do you want to read data from uploaded files so clients can use it to prefill fields like image description? This requires exiftool, it was detected as installed. (y/n)",
+           "y"}
+        else
+          {"Do you want to read data from uploaded files so clients can use it to prefill fields like image description? This requires exiftool, it was detected as not installed, please install it if you answer yes. (y/n)",
+           "n"}
+        end
+
+      read_uploads_description =
+        get_option(
+          options,
+          :read_uploads_description,
+          read_uploads_description_message,
+          read_uploads_description_default
         ) === "y"
 
       anonymize_uploads =
@@ -229,7 +247,8 @@ def run(["gen" | rest]) do
           listen_port: listen_port,
           upload_filters:
             upload_filters(%{
-              strip: strip_uploads,
+              strip_location: strip_uploads_location,
+              read_description: read_uploads_description,
               anonymize: anonymize_uploads,
               dedupe: dedupe_uploads
             })
@@ -297,12 +316,19 @@ defp write_robots_txt(static_dir, indexable, template_dir) do
 
   defp upload_filters(filters) when is_map(filters) do
     enabled_filters =
-      if filters.strip do
+      if filters.strip_location do
-        [Pleroma.Upload.Filter.Exiftool]
+        [Pleroma.Upload.Filter.Exiftool.StripLocation]
       else
         []
       end
 
+    enabled_filters =
+      if filters.read_description do
+        enabled_filters ++ [Pleroma.Upload.Filter.Exiftool.ReadDescription]
+      else
+        enabled_filters
+      end
+
     enabled_filters =
       if filters.anonymize do
         enabled_filters ++ [Pleroma.Upload.Filter.AnonymizeFilename]

lib/pleroma/application_requirements.ex

@@ -164,7 +164,8 @@ defp do_check_rum!(setting, migrate) do
 
   defp check_system_commands!(:ok) do
     filter_commands_statuses = [
-      check_filter(Pleroma.Upload.Filter.Exiftool, "exiftool"),
+      check_filter(Pleroma.Upload.Filter.Exiftool.StripLocation, "exiftool"),
+      check_filter(Pleroma.Upload.Filter.Exiftool.ReadDescription, "exiftool"),
       check_filter(Pleroma.Upload.Filter.Mogrify, "mogrify"),
       check_filter(Pleroma.Upload.Filter.Mogrifun, "mogrify"),
       check_filter(Pleroma.Upload.Filter.AnalyzeMetadata, "mogrify"),

lib/pleroma/config/deprecation_warnings.ex

@@ -22,6 +22,43 @@ defmodule Pleroma.Config.DeprecationWarnings do
      "\n* `config :pleroma, :instance, :quarantined_instances` is now covered by `:pleroma, :mrf_simple, :reject`"}
   ]
 
+  def check_exiftool_filter do
+    filters = Config.get([Pleroma.Upload]) |> Keyword.get(:filters, [])
+
+    if Pleroma.Upload.Filter.Exiftool in filters do
+      Logger.warn("""
+      !!!DEPRECATION WARNING!!!
+      Your config is using Exiftool as a filter instead of Exiftool.StripLocation. This should work for now, but you are advised to change to the new configuration to prevent possible issues later:
+
+      ```
+      config :pleroma, Pleroma.Upload,
+        filters: [Pleroma.Upload.Filter.Exiftool]
+      ```
+
+      Is now
+
+
+      ```
+      config :pleroma, Pleroma.Upload,
+        filters: [Pleroma.Upload.Filter.Exiftool.StripLocation]
+      ```
+      """)
+
+      new_config =
+        filters
+        |> Enum.map(fn
+          Pleroma.Upload.Filter.Exiftool -> Pleroma.Upload.Filter.Exiftool.StripLocation
+          filter -> filter
+        end)
+
+      Config.put([Pleroma.Upload, :filters], new_config)
+
+      :error
+    else
+      :ok
+    end
+  end
+
   def check_simple_policy_tuples do
     has_strings =
       Config.get([:mrf_simple])
@@ -180,7 +217,8 @@ def warn do
       check_uploders_s3_public_endpoint(),
       check_quarantined_instances_tuples(),
       check_transparency_exclusions_tuples(),
-      check_simple_policy_tuples()
+      check_simple_policy_tuples(),
+      check_exiftool_filter()
     ]
     |> Enum.reduce(:ok, fn
       :ok, :ok -> :ok

lib/pleroma/upload.ex

@@ -61,12 +61,23 @@ defmodule Pleroma.Upload do
           width: integer(),
           height: integer(),
           blurhash: String.t(),
+          description: String.t(),
           path: String.t()
         }
-  defstruct [:id, :name, :tempfile, :content_type, :width, :height, :blurhash, :path]
+  defstruct [
+    :id,
+    :name,
+    :tempfile,
+    :content_type,
+    :width,
+    :height,
+    :blurhash,
+    :description,
+    :path
+  ]
 
-  defp get_description(opts, upload) do
+  defp get_description(upload) do
-    case {opts[:description], Pleroma.Config.get([Pleroma.Upload, :default_description])} do
+    case {upload.description, Pleroma.Config.get([Pleroma.Upload, :default_description])} do
       {description, _} when is_binary(description) -> description
       {_, :filename} -> upload.name
       {_, str} when is_binary(str) -> str
@@ -82,7 +93,7 @@ def store(upload, opts \\ []) do
     with {:ok, upload} <- prepare_upload(upload, opts),
          upload = %__MODULE__{upload | path: upload.path || "#{upload.id}/#{upload.name}"},
          {:ok, upload} <- Pleroma.Upload.Filter.filter(opts.filters, upload),
-         description = get_description(opts, upload),
+         description = get_description(upload),
          {_, true} <-
            {:description_limit,
             String.length(description) <= Pleroma.Config.get([:instance, :description_limit])},
@@ -154,7 +165,8 @@ defp prepare_upload(%Plug.Upload{} = file, opts) do
          id: UUID.generate(),
          name: file.filename,
          tempfile: file.path,
-         content_type: file.content_type
+         content_type: file.content_type,
+         description: opts.description
        }}
     end
   end
@@ -174,7 +186,8 @@ defp prepare_upload(%{img: "data:image/" <> image_data}, opts) do
          id: UUID.generate(),
          name: hash <> "." <> ext,
          tempfile: tmp_path,
-         content_type: content_type
+         content_type: content_type,
+         description: opts.description
        }}
     end
   end

lib/pleroma/upload/filter/exiftool/read_description.ex

@@ -0,0 +1,47 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Upload.Filter.Exiftool.ReadDescription do
+  @moduledoc """
+  Gets the description from the related EXIF tags and provides them in the response if no description is provided yet.
+  It will first check ImageDescription, when that's too long or empty, it will check iptc:Caption-Abstract.
+  When the description is too long (see `:instance, :description_limit`), an empty string is returned.
+  """
+  @behaviour Pleroma.Upload.Filter
+
+  @spec filter(Pleroma.Upload.t()) :: {:ok, any()} | {:error, String.t()}
+
+  def filter(%Pleroma.Upload{description: description})
+      when is_binary(description),
+      do: {:ok, :noop}
+
+  def filter(%Pleroma.Upload{tempfile: file} = upload),
+    do: {:ok, :filtered, upload |> Map.put(:description, read_description_from_exif_data(file))}
+
+  def filter(_, _), do: {:ok, :noop}
+
+  defp read_description_from_exif_data(file) do
+    nil
+    |> read_when_empty(file, "-ImageDescription")
+    |> read_when_empty(file, "-iptc:Caption-Abstract")
+  end
+
+  defp read_when_empty(current_description, _, _) when is_binary(current_description),
+    do: current_description
+
+  defp read_when_empty(_, file, tag) do
+    try do
+      {tag_content, 0} =
+        System.cmd("exiftool", ["-b", "-s3", tag, file], stderr_to_stdout: true, parallelism: true)
+
+      if tag_content != "" and
+           String.length(tag_content) <=
+             Pleroma.Config.get([:instance, :description_limit]),
+         do: tag_content,
+         else: nil
+    rescue
+      _ in ErlangError -> nil
+    end
+  end
+end

lib/pleroma/upload/filter/exiftool/strip_location.ex

@@ -2,7 +2,7 @@
 # Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.Upload.Filter.Exiftool do
+defmodule Pleroma.Upload.Filter.Exiftool.StripLocation do
   @moduledoc """
   Strips GPS related EXIF tags and overwrites the file in place.
   Also strips or replaces filesystem metadata e.g., timestamps.

priv/repo/migrations/20220220135625_upload_filter_exiftool_to_exiftool_strip_location.exs

@@ -0,0 +1,37 @@
+defmodule Pleroma.Repo.Migrations.UploadFilterExiftoolToExiftoolStripLocation do
+  use Ecto.Migration
+
+  alias Pleroma.ConfigDB
+
+  def up,
+    do:
+      ConfigDB.get_by_params(%{group: :pleroma, key: Pleroma.Upload})
+      |> update_filtername(
+        Pleroma.Upload.Filter.Exiftool,
+        Pleroma.Upload.Filter.Exiftool.StripLocation
+      )
+
+  def down,
+    do:
+      ConfigDB.get_by_params(%{group: :pleroma, key: Pleroma.Upload})
+      |> update_filtername(
+        Pleroma.Upload.Filter.Exiftool.StripLocation,
+        Pleroma.Upload.Filter.Exiftool
+      )
+
+  defp update_filtername(%{value: value}, from_filtername, to_filtername) do
+    new_value =
+      value
+      |> Keyword.update(:filters, [], fn filters ->
+        filters
+        |> Enum.map(fn
+          ^from_filtername -> to_filtername
+          filter -> filter
+        end)
+      end)
+
+    ConfigDB.update_or_create(%{group: :pleroma, key: Pleroma.Upload, value: new_value})
+  end
+
+  defp update_filtername(_, _, _), do: nil
+end

test/mix/tasks/pleroma/instance_test.exs

@@ -67,7 +67,9 @@ test "running gen" do
         "test/uploads",
         "--static-dir",
         "./test/../test/instance/static/",
-        "--strip-uploads",
+        "--strip-uploads-location",
+        "y",
+        "--read-uploads-description",
         "y",
         "--dedupe-uploads",
         "n",
@@ -91,7 +93,10 @@ test "running gen" do
     assert generated_config =~ "password: \"dbpass\""
     assert generated_config =~ "configurable_from_database: true"
     assert generated_config =~ "http: [ip: {127, 0, 0, 1}, port: 4000]"
-    assert generated_config =~ "filters: [Pleroma.Upload.Filter.Exiftool]"
+
+    assert generated_config =~
+             "filters: [Pleroma.Upload.Filter.Exiftool.StripLocation, Pleroma.Upload.Filter.Exiftool.ReadDescription]"
+
     assert File.read!(tmp_path() <> "setup.psql") == generated_setup_psql()
     assert File.exists?(Path.expand("./test/instance/static/robots.txt"))
   end

test/pleroma/config/deprecation_warnings_test.exs

@@ -11,6 +11,62 @@ defmodule Pleroma.Config.DeprecationWarningsTest do
   alias Pleroma.Config
   alias Pleroma.Config.DeprecationWarnings
 
+  describe "filter exiftool" do
+    test "gives warning when still used" do
+      clear_config(
+        [Pleroma.Upload, :filters],
+        [Pleroma.Upload.Filter.Exiftool]
+      )
+
+      assert capture_log(fn -> DeprecationWarnings.check_exiftool_filter() end) =~
+               """
+               !!!DEPRECATION WARNING!!!
+               Your config is using Exiftool as a filter instead of Exiftool.StripLocation. This should work for now, but you are advised to change to the new configuration to prevent possible issues later:
+
+               ```
+               config :pleroma, Pleroma.Upload,
+                 filters: [Pleroma.Upload.Filter.Exiftool]
+               ```
+
+               Is now
+
+
+               ```
+               config :pleroma, Pleroma.Upload,
+                 filters: [Pleroma.Upload.Filter.Exiftool.StripLocation]
+               ```
+               """
+    end
+
+    test "changes setting to exiftool strip location" do
+      clear_config(
+        [Pleroma.Upload, :filters],
+        [Pleroma.Upload.Filter.Exiftool, Pleroma.Upload.Filter.Exiftool.ReadDescription]
+      )
+
+      expected_config = [
+        Pleroma.Upload.Filter.Exiftool.StripLocation,
+        Pleroma.Upload.Filter.Exiftool.ReadDescription
+      ]
+
+      capture_log(fn -> DeprecationWarnings.warn() end)
+
+      assert Config.get([Pleroma.Upload]) |> Keyword.get(:filters, []) == expected_config
+    end
+
+    test "doesn't give a warning with correct config" do
+      clear_config(
+        [Pleroma.Upload, :filters],
+        [
+          Pleroma.Upload.Filter.Exiftool.StripLocation,
+          Pleroma.Upload.Filter.Exiftool.ReadDescription
+        ]
+      )
+
+      assert capture_log(fn -> DeprecationWarnings.check_exiftool_filter() end) == ""
+    end
+  end
+
   describe "simple policy tuples" do
     test "gives warning when there are still strings" do
       clear_config([:mrf_simple],

test/pleroma/upload/filter/exiftool/read_description_test.exs

@@ -0,0 +1,98 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Upload.Filter.Exiftool.ReadDescriptionTest do
+  use Pleroma.DataCase, async: true
+  alias Pleroma.Upload.Filter
+
+  @uploads %Pleroma.Upload{
+    name: "image_with_imagedescription_and_caption-abstract.jpg",
+    content_type: "image/jpeg",
+    path: Path.absname("test/fixtures/image_with_imagedescription_and_caption-abstract.jpg"),
+    tempfile: Path.absname("test/fixtures/image_with_imagedescription_and_caption-abstract.jpg"),
+    description: nil
+  }
+
+  test "keeps description when not empty" do
+    uploads = %Pleroma.Upload{
+      name: "image_with_imagedescription_and_caption-abstract.jpg",
+      content_type: "image/jpeg",
+      path: Path.absname("test/fixtures/image_with_imagedescription_and_caption-abstract.jpg"),
+      tempfile:
+        Path.absname("test/fixtures/image_with_imagedescription_and_caption-abstract.jpg"),
+      description: "Some description"
+    }
+
+    assert Filter.Exiftool.ReadDescription.filter(uploads) ==
+             {:ok, :noop}
+  end
+
+  test "otherwise returns ImageDescription when present" do
+    uploads_after = %Pleroma.Upload{
+      name: "image_with_imagedescription_and_caption-abstract.jpg",
+      content_type: "image/jpeg",
+      path: Path.absname("test/fixtures/image_with_imagedescription_and_caption-abstract.jpg"),
+      tempfile:
+        Path.absname("test/fixtures/image_with_imagedescription_and_caption-abstract.jpg"),
+      description: "a descriptive white pixel"
+    }
+
+    assert Filter.Exiftool.ReadDescription.filter(@uploads) ==
+             {:ok, :filtered, uploads_after}
+  end
+
+  test "otherwise returns iptc:Caption-Abstract when present" do
+    upload = %Pleroma.Upload{
+      name: "image_with_caption-abstract.jpg",
+      content_type: "image/jpeg",
+      path: Path.absname("test/fixtures/image_with_caption-abstract.jpg"),
+      tempfile: Path.absname("test/fixtures/image_with_caption-abstract.jpg"),
+      description: nil
+    }
+
+    upload_after = %Pleroma.Upload{
+      name: "image_with_caption-abstract.jpg",
+      content_type: "image/jpeg",
+      path: Path.absname("test/fixtures/image_with_caption-abstract.jpg"),
+      tempfile: Path.absname("test/fixtures/image_with_caption-abstract.jpg"),
+      description: "an abstract white pixel"
+    }
+
+    assert Filter.Exiftool.ReadDescription.filter(upload) ==
+             {:ok, :filtered, upload_after}
+  end
+
+  test "otherwise returns nil" do
+    uploads = %Pleroma.Upload{
+      name: "image_with_no_description.jpg",
+      content_type: "image/jpeg",
+      path: Path.absname("test/fixtures/image_with_no_description.jpg"),
+      tempfile: Path.absname("test/fixtures/image_with_no_description.jpg"),
+      description: nil
+    }
+
+    assert Filter.Exiftool.ReadDescription.filter(uploads) ==
+             {:ok, :filtered, uploads}
+  end
+
+  test "Return nil when image description from EXIF data exceeds the maximum length" do
+    clear_config([:instance, :description_limit], 5)
+
+    assert Filter.Exiftool.ReadDescription.filter(@uploads) ==
+             {:ok, :filtered, @uploads}
+  end
+
+  test "Return nil when image description from EXIF data can't be read" do
+    uploads = %Pleroma.Upload{
+      name: "non-existant.jpg",
+      content_type: "image/jpeg",
+      path: Path.absname("test/fixtures/non-existant.jpg"),
+      tempfile: Path.absname("test/fixtures/non-existant_tmp.jpg"),
+      description: nil
+    }
+
+    assert Filter.Exiftool.ReadDescription.filter(uploads) ==
+             {:ok, :filtered, uploads}
+  end
+end

test/pleroma/upload/filter/exiftool/strip_location_test.exs

@@ -2,7 +2,7 @@
 # Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.Upload.Filter.ExiftoolTest do
+defmodule Pleroma.Upload.Filter.Exiftool.StripLocationTest do
   use Pleroma.DataCase, async: true
   alias Pleroma.Upload.Filter
 
@@ -21,7 +21,7 @@ test "apply exiftool filter" do
       tempfile: Path.absname("test/fixtures/DSCN0010_tmp.jpg")
     }
 
-    assert Filter.Exiftool.filter(upload) == {:ok, :filtered}
+    assert Filter.Exiftool.StripLocation.filter(upload) == {:ok, :filtered}
 
     {exif_original, 0} = System.cmd("exiftool", ["test/fixtures/DSCN0010.jpg"])
     {exif_filtered, 0} = System.cmd("exiftool", ["test/fixtures/DSCN0010_tmp.jpg"])
@@ -37,6 +37,6 @@ test "verify webp files are skipped" do
       content_type: "image/webp"
     }
 
-    assert Filter.Exiftool.filter(upload) == {:ok, :noop}
+    assert Filter.Exiftool.StripLocation.filter(upload) == {:ok, :noop}
   end
 end