Merge branch 'feature/770-add-emoji-tags' into 'develop'

Feature/770 add emoji tags

See merge request pleroma/pleroma!998
This commit is contained in:
lambda 2019-04-08 09:50:00 +00:00
commit 23067908de
15 changed files with 270 additions and 28 deletions

3
.gitignore vendored
View file

@ -35,3 +35,6 @@ erl_crash.dump
# Editor config
/.vscode/
# Prevent committing docs files
/priv/static/doc/*

View file

@ -58,7 +58,13 @@ config :pleroma, Pleroma.Uploaders.MDII,
cgi: "https://mdii.sakura.ne.jp/mdii-post.cgi",
files: "https://mdii.sakura.ne.jp"
config :pleroma, :emoji, shortcode_globs: ["/emoji/custom/**/*.png"]
config :pleroma, :emoji,
shortcode_globs: ["/emoji/custom/**/*.png"],
groups: [
# Put groups that have higher priority than defaults here. Example in `docs/config/custom_emoji.md`
Finmoji: "/finmoji/128px/*-128.png",
Custom: ["/emoji/*.png", "/emoji/custom/*.png"]
]
config :pleroma, :uri_schemes,
valid_schemes: [

View file

@ -1,5 +1,5 @@
firefox, /emoji/Firefox.gif
blank, /emoji/blank.png
firefox, /emoji/Firefox.gif, Gif,Fun
blank, /emoji/blank.png, Fun
f_00b, /emoji/f_00b.png
f_00b11b, /emoji/f_00b11b.png
f_00b33b, /emoji/f_00b33b.png
@ -28,4 +28,3 @@ f_33b00b, /emoji/f_33b00b.png
f_33b22b, /emoji/f_33b22b.png
f_33h, /emoji/f_33h.png
f_33t, /emoji/f_33t.png

View file

@ -10,7 +10,7 @@ Request parameters can be passed via [query strings](https://en.wikipedia.org/wi
* Authentication: not required
* Params: none
* Response: JSON
* Example response: `{"kalsarikannit_f":"/finmoji/128px/kalsarikannit_f-128.png","perkele":"/finmoji/128px/perkele-128.png","blobdab":"/emoji/blobdab.png","happiness":"/finmoji/128px/happiness-128.png"}`
* Example response: `[{"kalsarikannit_f":{"tags":["Finmoji"],"image_url":"/finmoji/128px/kalsarikannit_f-128.png"}},{"perkele":{"tags":["Finmoji"],"image_url":"/finmoji/128px/perkele-128.png"}},{"blobdab":{"tags":["SomeTag"],"image_url":"/emoji/blobdab.png"}},"happiness":{"tags":["Finmoji"],"image_url":"/finmoji/128px/happiness-128.png"}}]`
* Note: Same data as Mastodon APIs `/api/v1/custom_emojis` but in a different format
## `/api/pleroma/follow_import`

View file

@ -11,8 +11,43 @@ image files (in `/priv/static/emoji/custom`): `happy.png` and `sad.png`
content of `config/custom_emoji.txt`:
```
happy, /emoji/custom/happy.png
sad, /emoji/custom/sad.png
happy, /emoji/custom/happy.png, Tag1,Tag2
sad, /emoji/custom/sad.png, Tag1
foo, /emoji/custom/foo.png
```
The files should be PNG (APNG is okay with `.png` for `image/png` Content-type) and under 50kb for compatibility with mastodon.
## Emoji tags (groups)
Default tags are set in `config.exs`.
```elixir
config :pleroma, :emoji,
shortcode_globs: ["/emoji/custom/**/*.png"],
groups: [
Finmoji: "/finmoji/128px/*-128.png",
Custom: ["/emoji/*.png", "/emoji/custom/*.png"]
]
```
Order of the `groups` matters, so to override default tags just put your group on top of the list. E.g:
```elixir
config :pleroma, :emoji,
shortcode_globs: ["/emoji/custom/**/*.png"],
groups: [
"Finmoji special": "/finmoji/128px/a_trusted_friend-128.png", # special file
"Cirno": "/emoji/custom/cirno*.png", # png files in /emoji/custom/ which start with `cirno`
"Special group": "/emoji/custom/special_folder/*.png", # png files in /emoji/custom/special_folder/
"Another group": "/emoji/custom/special_folder/*/.png", # png files in /emoji/custom/special_folder/ subfolders
Finmoji: "/finmoji/128px/*-128.png",
Custom: ["/emoji/*.png", "/emoji/custom/*.png"]
]
```
Priority of tags assigns in emoji.txt and custom.txt:
`tag in file > special group setting in config.exs > default setting in config.exs`
Priority for globs:
`special group setting in config.exs > default setting in config.exs`

View file

@ -8,13 +8,19 @@ defmodule Pleroma.Emoji do
* the built-in Finmojis (if enabled in configuration),
* the files: `config/emoji.txt` and `config/custom_emoji.txt`
* glob paths
* glob paths, nested folder is used as tag name for grouping e.g. priv/static/emoji/custom/nested_folder
This GenServer stores in an ETS table the list of the loaded emojis, and also allows to reload the list at runtime.
"""
use GenServer
@type pattern :: Regex.t() | module() | String.t()
@type patterns :: pattern() | [pattern()]
@type group_patterns :: keyword(patterns())
@ets __MODULE__.Ets
@ets_options [:ordered_set, :protected, :named_table, {:read_concurrency, true}]
@groups Application.get_env(:pleroma, :emoji)[:groups]
@doc false
def start_link do
@ -73,13 +79,14 @@ defmodule Pleroma.Emoji do
end
defp load do
finmoji_enabled = Keyword.get(Application.get_env(:pleroma, :instance), :finmoji_enabled)
shortcode_globs = Application.get_env(:pleroma, :emoji)[:shortcode_globs] || []
emojis =
(load_finmoji(Keyword.get(Application.get_env(:pleroma, :instance), :finmoji_enabled)) ++
(load_finmoji(finmoji_enabled) ++
load_from_file("config/emoji.txt") ++
load_from_file("config/custom_emoji.txt") ++
load_from_globs(
Keyword.get(Application.get_env(:pleroma, :emoji, []), :shortcode_globs, [])
))
load_from_globs(shortcode_globs))
|> Enum.reject(fn value -> value == nil end)
true = :ets.insert(@ets, emojis)
@ -151,9 +158,12 @@ defmodule Pleroma.Emoji do
"white_nights",
"woollysocks"
]
defp load_finmoji(true) do
Enum.map(@finmoji, fn finmoji ->
{finmoji, "/finmoji/128px/#{finmoji}-128.png"}
file_name = "/finmoji/128px/#{finmoji}-128.png"
group = match_extra(@groups, file_name)
{finmoji, file_name, to_string(group)}
end)
end
@ -172,8 +182,14 @@ defmodule Pleroma.Emoji do
|> Stream.map(&String.trim/1)
|> Stream.map(fn line ->
case String.split(line, ~r/,\s*/) do
[name, file] -> {name, file}
_ -> nil
[name, file, tags] ->
{name, file, tags}
[name, file] ->
{name, file, to_string(match_extra(@groups, file))}
_ ->
nil
end
end)
|> Enum.to_list()
@ -190,9 +206,40 @@ defmodule Pleroma.Emoji do
|> Enum.concat()
Enum.map(paths, fn path ->
tag = match_extra(@groups, Path.join("/", Path.relative_to(path, static_path)))
shortcode = Path.basename(path, Path.extname(path))
external_path = Path.join("/", Path.relative_to(path, static_path))
{shortcode, external_path}
{shortcode, external_path, to_string(tag)}
end)
end
@doc """
Finds a matching group for the given emoji filename
"""
@spec match_extra(group_patterns(), String.t()) :: atom() | nil
def match_extra(group_patterns, filename) do
match_group_patterns(group_patterns, fn pattern ->
case pattern do
%Regex{} = regex -> Regex.match?(regex, filename)
string when is_binary(string) -> filename == string
end
end)
end
defp match_group_patterns(group_patterns, matcher) do
Enum.find_value(group_patterns, fn {group, patterns} ->
patterns =
patterns
|> List.wrap()
|> Enum.map(fn pattern ->
if String.contains?(pattern, "*") do
~r(#{String.replace(pattern, "*", ".*")})
else
pattern
end
end)
Enum.any?(patterns, matcher) && group
end)
end
end

View file

@ -77,9 +77,9 @@ defmodule Pleroma.Formatter do
def emojify(text, nil), do: text
def emojify(text, emoji, strip \\ false) do
Enum.reduce(emoji, text, fn {emoji, file}, text ->
emoji = HTML.strip_tags(emoji)
file = HTML.strip_tags(file)
Enum.reduce(emoji, text, fn emoji_data, text ->
emoji = HTML.strip_tags(elem(emoji_data, 0))
file = HTML.strip_tags(elem(emoji_data, 1))
html =
if not strip do
@ -101,7 +101,7 @@ defmodule Pleroma.Formatter do
def demojify(text, nil), do: text
def get_emoji(text) when is_binary(text) do
Enum.filter(Emoji.get_all(), fn {emoji, _} -> String.contains?(text, ":#{emoji}:") end)
Enum.filter(Emoji.get_all(), fn {emoji, _, _} -> String.contains?(text, ":#{emoji}:") end)
end
def get_emoji(_), do: []

View file

@ -167,7 +167,7 @@ defmodule Pleroma.Web.CommonAPI do
object,
"emoji",
(Formatter.get_emoji(status) ++ Formatter.get_emoji(data["spoiler_text"]))
|> Enum.reduce(%{}, fn {name, file}, acc ->
|> Enum.reduce(%{}, fn {name, file, _}, acc ->
Map.put(acc, name, "#{Pleroma.Web.Endpoint.static_url()}#{file}")
end)
) do

View file

@ -293,7 +293,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do
def emoji_from_profile(%{info: _info} = user) do
(Formatter.get_emoji(user.bio) ++ Formatter.get_emoji(user.name))
|> Enum.map(fn {shortcode, url} ->
|> Enum.map(fn {shortcode, url, _} ->
%{
"type" => "Emoji",
"icon" => %{"type" => "Image", "url" => "#{Endpoint.url()}#{url}"},

View file

@ -181,14 +181,15 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
defp mastodonized_emoji do
Pleroma.Emoji.get_all()
|> Enum.map(fn {shortcode, relative_url} ->
|> Enum.map(fn {shortcode, relative_url, tags} ->
url = to_string(URI.merge(Web.base_url(), relative_url))
%{
"shortcode" => shortcode,
"static_url" => url,
"visible_in_picker" => true,
"url" => url
"url" => url,
"tags" => String.split(tags, ",")
}
end)
end

View file

@ -283,7 +283,13 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do
end
def emoji(conn, _params) do
json(conn, Enum.into(Emoji.get_all(), %{}))
emoji =
Emoji.get_all()
|> Enum.map(fn {short_code, path, tags} ->
%{short_code => %{image_url: path, tags: String.split(tags, ",")}}
end)
json(conn, emoji)
end
def follow_import(conn, %{"list" => %Plug.Upload{} = listfile}) do

106
test/emoji_test.exs Normal file
View file

@ -0,0 +1,106 @@
defmodule Pleroma.EmojiTest do
use ExUnit.Case, async: true
alias Pleroma.Emoji
describe "get_all/0" do
setup do
emoji_list = Emoji.get_all()
{:ok, emoji_list: emoji_list}
end
test "first emoji", %{emoji_list: emoji_list} do
[emoji | _others] = emoji_list
{code, path, tags} = emoji
assert tuple_size(emoji) == 3
assert is_binary(code)
assert is_binary(path)
assert is_binary(tags)
end
test "random emoji", %{emoji_list: emoji_list} do
emoji = Enum.random(emoji_list)
{code, path, tags} = emoji
assert tuple_size(emoji) == 3
assert is_binary(code)
assert is_binary(path)
assert is_binary(tags)
end
end
describe "match_extra/2" do
setup do
groups = [
"list of files": ["/emoji/custom/first_file.png", "/emoji/custom/second_file.png"],
"wildcard folder": "/emoji/custom/*/file.png",
"wildcard files": "/emoji/custom/folder/*.png",
"special file": "/emoji/custom/special.png"
]
{:ok, groups: groups}
end
test "config for list of files", %{groups: groups} do
group =
groups
|> Emoji.match_extra("/emoji/custom/first_file.png")
|> to_string()
assert group == "list of files"
end
test "config with wildcard folder", %{groups: groups} do
group =
groups
|> Emoji.match_extra("/emoji/custom/some_folder/file.png")
|> to_string()
assert group == "wildcard folder"
end
test "config with wildcard folder and subfolders", %{groups: groups} do
group =
groups
|> Emoji.match_extra("/emoji/custom/some_folder/another_folder/file.png")
|> to_string()
assert group == "wildcard folder"
end
test "config with wildcard files", %{groups: groups} do
group =
groups
|> Emoji.match_extra("/emoji/custom/folder/some_file.png")
|> to_string()
assert group == "wildcard files"
end
test "config with wildcard files and subfolders", %{groups: groups} do
group =
groups
|> Emoji.match_extra("/emoji/custom/folder/another_folder/some_file.png")
|> to_string()
assert group == "wildcard files"
end
test "config for special file", %{groups: groups} do
group =
groups
|> Emoji.match_extra("/emoji/custom/special.png")
|> to_string()
assert group == "special file"
end
test "no mathing returns nil", %{groups: groups} do
group =
groups
|> Emoji.match_extra("/emoji/some_undefined.png")
refute group
end
end
end

View file

@ -271,7 +271,9 @@ defmodule Pleroma.FormatterTest do
test "it returns the emoji used in the text" do
text = "I love :moominmamma:"
assert Formatter.get_emoji(text) == [{"moominmamma", "/finmoji/128px/moominmamma-128.png"}]
assert Formatter.get_emoji(text) == [
{"moominmamma", "/finmoji/128px/moominmamma-128.png", "Finmoji"}
]
end
test "it returns a nice empty result when no emojis are present" do

View file

@ -2342,6 +2342,22 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do
assert acc_two == acc_three
end
describe "custom emoji" do
test "with tags", %{conn: conn} do
[emoji | _body] =
conn
|> get("/api/v1/custom_emojis")
|> json_response(200)
assert Map.has_key?(emoji, "shortcode")
assert Map.has_key?(emoji, "static_url")
assert Map.has_key?(emoji, "tags")
assert is_list(emoji["tags"])
assert Map.has_key?(emoji, "url")
assert Map.has_key?(emoji, "visible_in_picker")
end
end
describe "index/2 redirections" do
setup %{conn: conn} do
session_opts = [

View file

@ -170,6 +170,27 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do
end
end
describe "/api/pleroma/emoji" do
test "returns json with custom emoji with tags", %{conn: conn} do
[emoji | _body] =
conn
|> get("/api/pleroma/emoji")
|> json_response(200)
[key] = Map.keys(emoji)
%{
^key => %{
"image_url" => url,
"tags" => tags
}
} = emoji
assert is_binary(url)
assert is_list(tags)
end
end
describe "GET /ostatus_subscribe?acct=...." do
test "adds status to pleroma instance if the `acct` is a status", %{conn: conn} do
conn =