Merge branch 'feature/770-add-emoji-tags' into 'develop'

Feature/770 add emoji tags

See merge request pleroma/pleroma!998
This commit is contained in:
lambda 2019-04-08 09:50:00 +00:00
commit 23067908de
15 changed files with 270 additions and 28 deletions

3
.gitignore vendored
View file

@ -35,3 +35,6 @@ erl_crash.dump
# Editor config # Editor config
/.vscode/ /.vscode/
# Prevent committing docs files
/priv/static/doc/*

View file

@ -58,7 +58,13 @@ config :pleroma, Pleroma.Uploaders.MDII,
cgi: "https://mdii.sakura.ne.jp/mdii-post.cgi", cgi: "https://mdii.sakura.ne.jp/mdii-post.cgi",
files: "https://mdii.sakura.ne.jp" files: "https://mdii.sakura.ne.jp"
config :pleroma, :emoji, shortcode_globs: ["/emoji/custom/**/*.png"] config :pleroma, :emoji,
shortcode_globs: ["/emoji/custom/**/*.png"],
groups: [
# Put groups that have higher priority than defaults here. Example in `docs/config/custom_emoji.md`
Finmoji: "/finmoji/128px/*-128.png",
Custom: ["/emoji/*.png", "/emoji/custom/*.png"]
]
config :pleroma, :uri_schemes, config :pleroma, :uri_schemes,
valid_schemes: [ valid_schemes: [

View file

@ -1,5 +1,5 @@
firefox, /emoji/Firefox.gif firefox, /emoji/Firefox.gif, Gif,Fun
blank, /emoji/blank.png blank, /emoji/blank.png, Fun
f_00b, /emoji/f_00b.png f_00b, /emoji/f_00b.png
f_00b11b, /emoji/f_00b11b.png f_00b11b, /emoji/f_00b11b.png
f_00b33b, /emoji/f_00b33b.png f_00b33b, /emoji/f_00b33b.png
@ -28,4 +28,3 @@ f_33b00b, /emoji/f_33b00b.png
f_33b22b, /emoji/f_33b22b.png f_33b22b, /emoji/f_33b22b.png
f_33h, /emoji/f_33h.png f_33h, /emoji/f_33h.png
f_33t, /emoji/f_33t.png f_33t, /emoji/f_33t.png

View file

@ -10,7 +10,7 @@ Request parameters can be passed via [query strings](https://en.wikipedia.org/wi
* Authentication: not required * Authentication: not required
* Params: none * Params: none
* Response: JSON * Response: JSON
* Example response: `{"kalsarikannit_f":"/finmoji/128px/kalsarikannit_f-128.png","perkele":"/finmoji/128px/perkele-128.png","blobdab":"/emoji/blobdab.png","happiness":"/finmoji/128px/happiness-128.png"}` * Example response: `[{"kalsarikannit_f":{"tags":["Finmoji"],"image_url":"/finmoji/128px/kalsarikannit_f-128.png"}},{"perkele":{"tags":["Finmoji"],"image_url":"/finmoji/128px/perkele-128.png"}},{"blobdab":{"tags":["SomeTag"],"image_url":"/emoji/blobdab.png"}},"happiness":{"tags":["Finmoji"],"image_url":"/finmoji/128px/happiness-128.png"}}]`
* Note: Same data as Mastodon APIs `/api/v1/custom_emojis` but in a different format * Note: Same data as Mastodon APIs `/api/v1/custom_emojis` but in a different format
## `/api/pleroma/follow_import` ## `/api/pleroma/follow_import`
@ -27,14 +27,14 @@ Request parameters can be passed via [query strings](https://en.wikipedia.org/wi
* Method: `GET` * Method: `GET`
* Authentication: not required * Authentication: not required
* Params: none * Params: none
* Response: Provider specific JSON, the only guaranteed parameter is `type` * Response: Provider specific JSON, the only guaranteed parameter is `type`
* Example response: `{"type": "kocaptcha", "token": "whatever", "url": "https://captcha.kotobank.ch/endpoint"}` * Example response: `{"type": "kocaptcha", "token": "whatever", "url": "https://captcha.kotobank.ch/endpoint"}`
## `/api/pleroma/delete_account` ## `/api/pleroma/delete_account`
### Delete an account ### Delete an account
* Method `POST` * Method `POST`
* Authentication: required * Authentication: required
* Params: * Params:
* `password`: user's password * `password`: user's password
* Response: JSON. Returns `{"status": "success"}` if the deletion was successful, `{"error": "[error message]"}` otherwise * Response: JSON. Returns `{"status": "success"}` if the deletion was successful, `{"error": "[error message]"}` otherwise
* Example response: `{"error": "Invalid password."}` * Example response: `{"error": "Invalid password."}`

View file

@ -11,8 +11,43 @@ image files (in `/priv/static/emoji/custom`): `happy.png` and `sad.png`
content of `config/custom_emoji.txt`: content of `config/custom_emoji.txt`:
``` ```
happy, /emoji/custom/happy.png happy, /emoji/custom/happy.png, Tag1,Tag2
sad, /emoji/custom/sad.png sad, /emoji/custom/sad.png, Tag1
foo, /emoji/custom/foo.png
``` ```
The files should be PNG (APNG is okay with `.png` for `image/png` Content-type) and under 50kb for compatibility with mastodon. The files should be PNG (APNG is okay with `.png` for `image/png` Content-type) and under 50kb for compatibility with mastodon.
## Emoji tags (groups)
Default tags are set in `config.exs`.
```elixir
config :pleroma, :emoji,
shortcode_globs: ["/emoji/custom/**/*.png"],
groups: [
Finmoji: "/finmoji/128px/*-128.png",
Custom: ["/emoji/*.png", "/emoji/custom/*.png"]
]
```
Order of the `groups` matters, so to override default tags just put your group on top of the list. E.g:
```elixir
config :pleroma, :emoji,
shortcode_globs: ["/emoji/custom/**/*.png"],
groups: [
"Finmoji special": "/finmoji/128px/a_trusted_friend-128.png", # special file
"Cirno": "/emoji/custom/cirno*.png", # png files in /emoji/custom/ which start with `cirno`
"Special group": "/emoji/custom/special_folder/*.png", # png files in /emoji/custom/special_folder/
"Another group": "/emoji/custom/special_folder/*/.png", # png files in /emoji/custom/special_folder/ subfolders
Finmoji: "/finmoji/128px/*-128.png",
Custom: ["/emoji/*.png", "/emoji/custom/*.png"]
]
```
Priority of tags assigns in emoji.txt and custom.txt:
`tag in file > special group setting in config.exs > default setting in config.exs`
Priority for globs:
`special group setting in config.exs > default setting in config.exs`

View file

@ -8,13 +8,19 @@ defmodule Pleroma.Emoji do
* the built-in Finmojis (if enabled in configuration), * the built-in Finmojis (if enabled in configuration),
* the files: `config/emoji.txt` and `config/custom_emoji.txt` * the files: `config/emoji.txt` and `config/custom_emoji.txt`
* glob paths * glob paths, nested folder is used as tag name for grouping e.g. priv/static/emoji/custom/nested_folder
This GenServer stores in an ETS table the list of the loaded emojis, and also allows to reload the list at runtime. This GenServer stores in an ETS table the list of the loaded emojis, and also allows to reload the list at runtime.
""" """
use GenServer use GenServer
@type pattern :: Regex.t() | module() | String.t()
@type patterns :: pattern() | [pattern()]
@type group_patterns :: keyword(patterns())
@ets __MODULE__.Ets @ets __MODULE__.Ets
@ets_options [:ordered_set, :protected, :named_table, {:read_concurrency, true}] @ets_options [:ordered_set, :protected, :named_table, {:read_concurrency, true}]
@groups Application.get_env(:pleroma, :emoji)[:groups]
@doc false @doc false
def start_link do def start_link do
@ -73,13 +79,14 @@ defmodule Pleroma.Emoji do
end end
defp load do defp load do
finmoji_enabled = Keyword.get(Application.get_env(:pleroma, :instance), :finmoji_enabled)
shortcode_globs = Application.get_env(:pleroma, :emoji)[:shortcode_globs] || []
emojis = emojis =
(load_finmoji(Keyword.get(Application.get_env(:pleroma, :instance), :finmoji_enabled)) ++ (load_finmoji(finmoji_enabled) ++
load_from_file("config/emoji.txt") ++ load_from_file("config/emoji.txt") ++
load_from_file("config/custom_emoji.txt") ++ load_from_file("config/custom_emoji.txt") ++
load_from_globs( load_from_globs(shortcode_globs))
Keyword.get(Application.get_env(:pleroma, :emoji, []), :shortcode_globs, [])
))
|> Enum.reject(fn value -> value == nil end) |> Enum.reject(fn value -> value == nil end)
true = :ets.insert(@ets, emojis) true = :ets.insert(@ets, emojis)
@ -151,9 +158,12 @@ defmodule Pleroma.Emoji do
"white_nights", "white_nights",
"woollysocks" "woollysocks"
] ]
defp load_finmoji(true) do defp load_finmoji(true) do
Enum.map(@finmoji, fn finmoji -> Enum.map(@finmoji, fn finmoji ->
{finmoji, "/finmoji/128px/#{finmoji}-128.png"} file_name = "/finmoji/128px/#{finmoji}-128.png"
group = match_extra(@groups, file_name)
{finmoji, file_name, to_string(group)}
end) end)
end end
@ -172,8 +182,14 @@ defmodule Pleroma.Emoji do
|> Stream.map(&String.trim/1) |> Stream.map(&String.trim/1)
|> Stream.map(fn line -> |> Stream.map(fn line ->
case String.split(line, ~r/,\s*/) do case String.split(line, ~r/,\s*/) do
[name, file] -> {name, file} [name, file, tags] ->
_ -> nil {name, file, tags}
[name, file] ->
{name, file, to_string(match_extra(@groups, file))}
_ ->
nil
end end
end) end)
|> Enum.to_list() |> Enum.to_list()
@ -190,9 +206,40 @@ defmodule Pleroma.Emoji do
|> Enum.concat() |> Enum.concat()
Enum.map(paths, fn path -> Enum.map(paths, fn path ->
tag = match_extra(@groups, Path.join("/", Path.relative_to(path, static_path)))
shortcode = Path.basename(path, Path.extname(path)) shortcode = Path.basename(path, Path.extname(path))
external_path = Path.join("/", Path.relative_to(path, static_path)) external_path = Path.join("/", Path.relative_to(path, static_path))
{shortcode, external_path} {shortcode, external_path, to_string(tag)}
end)
end
@doc """
Finds a matching group for the given emoji filename
"""
@spec match_extra(group_patterns(), String.t()) :: atom() | nil
def match_extra(group_patterns, filename) do
match_group_patterns(group_patterns, fn pattern ->
case pattern do
%Regex{} = regex -> Regex.match?(regex, filename)
string when is_binary(string) -> filename == string
end
end)
end
defp match_group_patterns(group_patterns, matcher) do
Enum.find_value(group_patterns, fn {group, patterns} ->
patterns =
patterns
|> List.wrap()
|> Enum.map(fn pattern ->
if String.contains?(pattern, "*") do
~r(#{String.replace(pattern, "*", ".*")})
else
pattern
end
end)
Enum.any?(patterns, matcher) && group
end) end)
end end
end end

View file

@ -77,9 +77,9 @@ defmodule Pleroma.Formatter do
def emojify(text, nil), do: text def emojify(text, nil), do: text
def emojify(text, emoji, strip \\ false) do def emojify(text, emoji, strip \\ false) do
Enum.reduce(emoji, text, fn {emoji, file}, text -> Enum.reduce(emoji, text, fn emoji_data, text ->
emoji = HTML.strip_tags(emoji) emoji = HTML.strip_tags(elem(emoji_data, 0))
file = HTML.strip_tags(file) file = HTML.strip_tags(elem(emoji_data, 1))
html = html =
if not strip do if not strip do
@ -101,7 +101,7 @@ defmodule Pleroma.Formatter do
def demojify(text, nil), do: text def demojify(text, nil), do: text
def get_emoji(text) when is_binary(text) do def get_emoji(text) when is_binary(text) do
Enum.filter(Emoji.get_all(), fn {emoji, _} -> String.contains?(text, ":#{emoji}:") end) Enum.filter(Emoji.get_all(), fn {emoji, _, _} -> String.contains?(text, ":#{emoji}:") end)
end end
def get_emoji(_), do: [] def get_emoji(_), do: []

View file

@ -167,7 +167,7 @@ defmodule Pleroma.Web.CommonAPI do
object, object,
"emoji", "emoji",
(Formatter.get_emoji(status) ++ Formatter.get_emoji(data["spoiler_text"])) (Formatter.get_emoji(status) ++ Formatter.get_emoji(data["spoiler_text"]))
|> Enum.reduce(%{}, fn {name, file}, acc -> |> Enum.reduce(%{}, fn {name, file, _}, acc ->
Map.put(acc, name, "#{Pleroma.Web.Endpoint.static_url()}#{file}") Map.put(acc, name, "#{Pleroma.Web.Endpoint.static_url()}#{file}")
end) end)
) do ) do

View file

@ -293,7 +293,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do
def emoji_from_profile(%{info: _info} = user) do def emoji_from_profile(%{info: _info} = user) do
(Formatter.get_emoji(user.bio) ++ Formatter.get_emoji(user.name)) (Formatter.get_emoji(user.bio) ++ Formatter.get_emoji(user.name))
|> Enum.map(fn {shortcode, url} -> |> Enum.map(fn {shortcode, url, _} ->
%{ %{
"type" => "Emoji", "type" => "Emoji",
"icon" => %{"type" => "Image", "url" => "#{Endpoint.url()}#{url}"}, "icon" => %{"type" => "Image", "url" => "#{Endpoint.url()}#{url}"},

View file

@ -181,14 +181,15 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
defp mastodonized_emoji do defp mastodonized_emoji do
Pleroma.Emoji.get_all() Pleroma.Emoji.get_all()
|> Enum.map(fn {shortcode, relative_url} -> |> Enum.map(fn {shortcode, relative_url, tags} ->
url = to_string(URI.merge(Web.base_url(), relative_url)) url = to_string(URI.merge(Web.base_url(), relative_url))
%{ %{
"shortcode" => shortcode, "shortcode" => shortcode,
"static_url" => url, "static_url" => url,
"visible_in_picker" => true, "visible_in_picker" => true,
"url" => url "url" => url,
"tags" => String.split(tags, ",")
} }
end) end)
end end

View file

@ -283,7 +283,13 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do
end end
def emoji(conn, _params) do def emoji(conn, _params) do
json(conn, Enum.into(Emoji.get_all(), %{})) emoji =
Emoji.get_all()
|> Enum.map(fn {short_code, path, tags} ->
%{short_code => %{image_url: path, tags: String.split(tags, ",")}}
end)
json(conn, emoji)
end end
def follow_import(conn, %{"list" => %Plug.Upload{} = listfile}) do def follow_import(conn, %{"list" => %Plug.Upload{} = listfile}) do

106
test/emoji_test.exs Normal file
View file

@ -0,0 +1,106 @@
defmodule Pleroma.EmojiTest do
use ExUnit.Case, async: true
alias Pleroma.Emoji
describe "get_all/0" do
setup do
emoji_list = Emoji.get_all()
{:ok, emoji_list: emoji_list}
end
test "first emoji", %{emoji_list: emoji_list} do
[emoji | _others] = emoji_list
{code, path, tags} = emoji
assert tuple_size(emoji) == 3
assert is_binary(code)
assert is_binary(path)
assert is_binary(tags)
end
test "random emoji", %{emoji_list: emoji_list} do
emoji = Enum.random(emoji_list)
{code, path, tags} = emoji
assert tuple_size(emoji) == 3
assert is_binary(code)
assert is_binary(path)
assert is_binary(tags)
end
end
describe "match_extra/2" do
setup do
groups = [
"list of files": ["/emoji/custom/first_file.png", "/emoji/custom/second_file.png"],
"wildcard folder": "/emoji/custom/*/file.png",
"wildcard files": "/emoji/custom/folder/*.png",
"special file": "/emoji/custom/special.png"
]
{:ok, groups: groups}
end
test "config for list of files", %{groups: groups} do
group =
groups
|> Emoji.match_extra("/emoji/custom/first_file.png")
|> to_string()
assert group == "list of files"
end
test "config with wildcard folder", %{groups: groups} do
group =
groups
|> Emoji.match_extra("/emoji/custom/some_folder/file.png")
|> to_string()
assert group == "wildcard folder"
end
test "config with wildcard folder and subfolders", %{groups: groups} do
group =
groups
|> Emoji.match_extra("/emoji/custom/some_folder/another_folder/file.png")
|> to_string()
assert group == "wildcard folder"
end
test "config with wildcard files", %{groups: groups} do
group =
groups
|> Emoji.match_extra("/emoji/custom/folder/some_file.png")
|> to_string()
assert group == "wildcard files"
end
test "config with wildcard files and subfolders", %{groups: groups} do
group =
groups
|> Emoji.match_extra("/emoji/custom/folder/another_folder/some_file.png")
|> to_string()
assert group == "wildcard files"
end
test "config for special file", %{groups: groups} do
group =
groups
|> Emoji.match_extra("/emoji/custom/special.png")
|> to_string()
assert group == "special file"
end
test "no mathing returns nil", %{groups: groups} do
group =
groups
|> Emoji.match_extra("/emoji/some_undefined.png")
refute group
end
end
end

View file

@ -271,7 +271,9 @@ defmodule Pleroma.FormatterTest do
test "it returns the emoji used in the text" do test "it returns the emoji used in the text" do
text = "I love :moominmamma:" text = "I love :moominmamma:"
assert Formatter.get_emoji(text) == [{"moominmamma", "/finmoji/128px/moominmamma-128.png"}] assert Formatter.get_emoji(text) == [
{"moominmamma", "/finmoji/128px/moominmamma-128.png", "Finmoji"}
]
end end
test "it returns a nice empty result when no emojis are present" do test "it returns a nice empty result when no emojis are present" do

View file

@ -2342,6 +2342,22 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do
assert acc_two == acc_three assert acc_two == acc_three
end end
describe "custom emoji" do
test "with tags", %{conn: conn} do
[emoji | _body] =
conn
|> get("/api/v1/custom_emojis")
|> json_response(200)
assert Map.has_key?(emoji, "shortcode")
assert Map.has_key?(emoji, "static_url")
assert Map.has_key?(emoji, "tags")
assert is_list(emoji["tags"])
assert Map.has_key?(emoji, "url")
assert Map.has_key?(emoji, "visible_in_picker")
end
end
describe "index/2 redirections" do describe "index/2 redirections" do
setup %{conn: conn} do setup %{conn: conn} do
session_opts = [ session_opts = [

View file

@ -170,6 +170,27 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do
end end
end end
describe "/api/pleroma/emoji" do
test "returns json with custom emoji with tags", %{conn: conn} do
[emoji | _body] =
conn
|> get("/api/pleroma/emoji")
|> json_response(200)
[key] = Map.keys(emoji)
%{
^key => %{
"image_url" => url,
"tags" => tags
}
} = emoji
assert is_binary(url)
assert is_list(tags)
end
end
describe "GET /ostatus_subscribe?acct=...." do describe "GET /ostatus_subscribe?acct=...." do
test "adds status to pleroma instance if the `acct` is a status", %{conn: conn} do test "adds status to pleroma instance if the `acct` is a status", %{conn: conn} do
conn = conn =