forked from AkkomaGang/akkoma
Pipeline Ingestion: Article
This commit is contained in:
parent
2132b24a9d
commit
1b3d5956b1
9 changed files with 143 additions and 28 deletions
|
@ -84,7 +84,7 @@ defp increase_replies_count_if_reply(%{
|
||||||
|
|
||||||
defp increase_replies_count_if_reply(_create_data), do: :noop
|
defp increase_replies_count_if_reply(_create_data), do: :noop
|
||||||
|
|
||||||
@object_types ~w[ChatMessage Question Answer Audio Video Event]
|
@object_types ~w[ChatMessage Question Answer Audio Video Event Article]
|
||||||
@spec persist(map(), keyword()) :: {:ok, Activity.t() | Object.t()}
|
@spec persist(map(), keyword()) :: {:ok, Activity.t() | Object.t()}
|
||||||
def persist(%{"type" => type} = object, meta) when type in @object_types do
|
def persist(%{"type" => type} = object, meta) when type in @object_types do
|
||||||
with {:ok, object} <- Object.create(object) do
|
with {:ok, object} <- Object.create(object) do
|
||||||
|
|
|
@ -17,6 +17,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
|
||||||
alias Pleroma.Web.ActivityPub.ObjectValidators.AcceptRejectValidator
|
alias Pleroma.Web.ActivityPub.ObjectValidators.AcceptRejectValidator
|
||||||
alias Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator
|
alias Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator
|
||||||
alias Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator
|
alias Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator
|
||||||
|
alias Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator
|
||||||
alias Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator
|
alias Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator
|
||||||
alias Pleroma.Web.ActivityPub.ObjectValidators.BlockValidator
|
alias Pleroma.Web.ActivityPub.ObjectValidators.BlockValidator
|
||||||
alias Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator
|
alias Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator
|
||||||
|
@ -160,6 +161,16 @@ def validate(%{"type" => type} = object, meta) when type in ~w[Audio Video] do
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def validate(%{"type" => "Article"} = object, meta) do
|
||||||
|
with {:ok, object} <-
|
||||||
|
object
|
||||||
|
|> ArticleNoteValidator.cast_and_validate()
|
||||||
|
|> Ecto.Changeset.apply_action(:insert) do
|
||||||
|
object = stringify_keys(object)
|
||||||
|
{:ok, object, meta}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
def validate(%{"type" => "Answer"} = object, meta) do
|
def validate(%{"type" => "Answer"} = object, meta) do
|
||||||
with {:ok, object} <-
|
with {:ok, object} <-
|
||||||
object
|
object
|
||||||
|
@ -199,7 +210,7 @@ def validate(
|
||||||
%{"type" => "Create", "object" => %{"type" => objtype} = object} = create_activity,
|
%{"type" => "Create", "object" => %{"type" => objtype} = object} = create_activity,
|
||||||
meta
|
meta
|
||||||
)
|
)
|
||||||
when objtype in ~w[Question Answer Audio Video Event] do
|
when objtype in ~w[Question Answer Audio Video Event Article] do
|
||||||
with {:ok, object_data} <- cast_and_apply(object),
|
with {:ok, object_data} <- cast_and_apply(object),
|
||||||
meta = Keyword.put(meta, :object_data, object_data |> stringify_keys),
|
meta = Keyword.put(meta, :object_data, object_data |> stringify_keys),
|
||||||
{:ok, create_activity} <-
|
{:ok, create_activity} <-
|
||||||
|
@ -241,6 +252,10 @@ def cast_and_apply(%{"type" => "Event"} = object) do
|
||||||
EventValidator.cast_and_apply(object)
|
EventValidator.cast_and_apply(object)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def cast_and_apply(%{"type" => "Article"} = object) do
|
||||||
|
ArticleNoteValidator.cast_and_apply(object)
|
||||||
|
end
|
||||||
|
|
||||||
def cast_and_apply(o), do: {:error, {:validator_not_set, o}}
|
def cast_and_apply(o), do: {:error, {:validator_not_set, o}}
|
||||||
|
|
||||||
# is_struct/1 isn't present in Elixir 1.8.x
|
# is_struct/1 isn't present in Elixir 1.8.x
|
||||||
|
|
|
@ -2,15 +2,19 @@
|
||||||
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||||
# SPDX-License-Identifier: AGPL-3.0-only
|
# SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
|
||||||
defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do
|
defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator do
|
||||||
use Ecto.Schema
|
use Ecto.Schema
|
||||||
|
|
||||||
alias Pleroma.EctoType.ActivityPub.ObjectValidators
|
alias Pleroma.EctoType.ActivityPub.ObjectValidators
|
||||||
|
alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator
|
||||||
|
alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
|
||||||
|
alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
|
||||||
alias Pleroma.Web.ActivityPub.Transmogrifier
|
alias Pleroma.Web.ActivityPub.Transmogrifier
|
||||||
|
|
||||||
import Ecto.Changeset
|
import Ecto.Changeset
|
||||||
|
|
||||||
@primary_key false
|
@primary_key false
|
||||||
|
@derive Jason.Encoder
|
||||||
|
|
||||||
embedded_schema do
|
embedded_schema do
|
||||||
field(:id, ObjectValidators.ObjectID, primary_key: true)
|
field(:id, ObjectValidators.ObjectID, primary_key: true)
|
||||||
|
@ -30,13 +34,14 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do
|
||||||
# short identifier for PleromaFE to group statuses by context
|
# short identifier for PleromaFE to group statuses by context
|
||||||
field(:context_id, :integer)
|
field(:context_id, :integer)
|
||||||
|
|
||||||
|
# TODO: Remove actor on objects
|
||||||
field(:actor, ObjectValidators.ObjectID)
|
field(:actor, ObjectValidators.ObjectID)
|
||||||
|
|
||||||
field(:attributedTo, ObjectValidators.ObjectID)
|
field(:attributedTo, ObjectValidators.ObjectID)
|
||||||
field(:published, ObjectValidators.DateTime)
|
field(:published, ObjectValidators.DateTime)
|
||||||
field(:emoji, ObjectValidators.Emoji, default: %{})
|
field(:emoji, ObjectValidators.Emoji, default: %{})
|
||||||
field(:sensitive, :boolean, default: false)
|
field(:sensitive, :boolean, default: false)
|
||||||
# TODO: Write type
|
embeds_many(:attachment, AttachmentValidator)
|
||||||
field(:attachment, {:array, :map}, default: [])
|
|
||||||
field(:replies_count, :integer, default: 0)
|
field(:replies_count, :integer, default: 0)
|
||||||
field(:like_count, :integer, default: 0)
|
field(:like_count, :integer, default: 0)
|
||||||
field(:announcement_count, :integer, default: 0)
|
field(:announcement_count, :integer, default: 0)
|
||||||
|
@ -47,27 +52,55 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do
|
||||||
field(:announcements, {:array, ObjectValidators.ObjectID}, default: [])
|
field(:announcements, {:array, ObjectValidators.ObjectID}, default: [])
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def cast_and_apply(data) do
|
||||||
|
data
|
||||||
|
|> cast_data
|
||||||
|
|> apply_action(:insert)
|
||||||
|
end
|
||||||
|
|
||||||
def cast_and_validate(data) do
|
def cast_and_validate(data) do
|
||||||
data
|
data
|
||||||
|> cast_data()
|
|> cast_data()
|
||||||
|> validate_data()
|
|> validate_data()
|
||||||
end
|
end
|
||||||
|
|
||||||
defp fix(data) do
|
|
||||||
data
|
|
||||||
|> Transmogrifier.fix_emoji()
|
|
||||||
end
|
|
||||||
|
|
||||||
def cast_data(data) do
|
def cast_data(data) do
|
||||||
data = fix(data)
|
data = fix(data)
|
||||||
|
|
||||||
%__MODULE__{}
|
%__MODULE__{}
|
||||||
|> cast(data, __schema__(:fields))
|
|> changeset(data)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp fix_url(%{"url" => url} = data) when is_map(url) do
|
||||||
|
Map.put(data, "url", url["href"])
|
||||||
|
end
|
||||||
|
|
||||||
|
defp fix_url(data), do: data
|
||||||
|
|
||||||
|
defp fix(data) do
|
||||||
|
data
|
||||||
|
|> CommonFixes.fix_defaults()
|
||||||
|
|> CommonFixes.fix_attribution()
|
||||||
|
|> CommonFixes.fix_actor()
|
||||||
|
|> fix_url()
|
||||||
|
|> Transmogrifier.fix_emoji()
|
||||||
|
end
|
||||||
|
|
||||||
|
def changeset(struct, data) do
|
||||||
|
data = fix(data)
|
||||||
|
|
||||||
|
struct
|
||||||
|
|> cast(data, __schema__(:fields) -- [:attachment])
|
||||||
|
|> cast_embed(:attachment)
|
||||||
end
|
end
|
||||||
|
|
||||||
def validate_data(data_cng) do
|
def validate_data(data_cng) do
|
||||||
data_cng
|
data_cng
|
||||||
|> validate_inclusion(:type, ["Note"])
|
|> validate_inclusion(:type, ["Article", "Note"])
|
||||||
|> validate_required([:id, :actor, :to, :cc, :type, :content, :context])
|
|> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id])
|
||||||
|
|> CommonValidations.validate_any_presence([:cc, :to])
|
||||||
|
|> CommonValidations.validate_fields_match([:actor, :attributedTo])
|
||||||
|
|> CommonValidations.validate_actor_presence()
|
||||||
|
|> CommonValidations.validate_host_match()
|
||||||
end
|
end
|
||||||
end
|
end
|
|
@ -336,7 +336,7 @@ def handle_object_creation(%{"type" => "Answer"} = object_map, meta) do
|
||||||
end
|
end
|
||||||
|
|
||||||
def handle_object_creation(%{"type" => objtype} = object, meta)
|
def handle_object_creation(%{"type" => objtype} = object, meta)
|
||||||
when objtype in ~w[Audio Video Question Event] do
|
when objtype in ~w[Audio Video Question Event Article] do
|
||||||
with {:ok, object, meta} <- Pipeline.common_pipeline(object, meta) do
|
with {:ok, object, meta} <- Pipeline.common_pipeline(object, meta) do
|
||||||
{:ok, object, meta}
|
{:ok, object, meta}
|
||||||
end
|
end
|
||||||
|
|
|
@ -424,7 +424,7 @@ def handle_incoming(
|
||||||
%{"type" => "Create", "object" => %{"type" => objtype} = object} = data,
|
%{"type" => "Create", "object" => %{"type" => objtype} = object} = data,
|
||||||
options
|
options
|
||||||
)
|
)
|
||||||
when objtype in ~w{Article Note Page} do
|
when objtype in ~w{Note Page} do
|
||||||
actor = Containment.get_actor(data)
|
actor = Containment.get_actor(data)
|
||||||
|
|
||||||
with nil <- Activity.get_create_by_object_ap_id(object["id"]),
|
with nil <- Activity.get_create_by_object_ap_id(object["id"]),
|
||||||
|
@ -518,7 +518,7 @@ def handle_incoming(
|
||||||
%{"type" => "Create", "object" => %{"type" => objtype}} = data,
|
%{"type" => "Create", "object" => %{"type" => objtype}} = data,
|
||||||
_options
|
_options
|
||||||
)
|
)
|
||||||
when objtype in ~w{Question Answer ChatMessage Audio Video Event} do
|
when objtype in ~w{Question Answer ChatMessage Audio Video Event Article} do
|
||||||
data = Map.put(data, "object", strip_internal_fields(data["object"]))
|
data = Map.put(data, "object", strip_internal_fields(data["object"]))
|
||||||
|
|
||||||
with {:ok, %User{}} <- ObjectValidator.fetch_actor(data),
|
with {:ok, %User{}} <- ObjectValidator.fetch_actor(data),
|
||||||
|
|
1
test/fixtures/tesla_mock/wedistribute-create-article.json
vendored
Normal file
1
test/fixtures/tesla_mock/wedistribute-create-article.json
vendored
Normal file
|
@ -0,0 +1 @@
|
||||||
|
{"@context":["https:\/\/www.w3.org\/ns\/activitystreams"],"type":"Create","actor":"https:\/\/wedistribute.org\/wp-json\/pterotype\/v1\/actor\/-blog","object":{"@context":["https:\/\/www.w3.org\/ns\/activitystreams"],"type":"Article","name":"The end is near: Mastodon plans to drop OStatus support","content":"<!-- wp:paragraph {\"dropCap\":true} -->\n<p class=\"has-drop-cap\">The days of OStatus are numbered. The venerable protocol has served as a glue between many different types of servers since the early days of the Fediverse, connecting StatusNet (now GNU Social) to Friendica, Hubzilla, Mastodon, and Pleroma.<\/p>\n<!-- \/wp:paragraph -->\n\n<!-- wp:paragraph -->\n<p>Now that many fediverse platforms support ActivityPub as a successor protocol, Mastodon appears to be drawing a line in the sand. In <a href=\"https:\/\/www.patreon.com\/posts\/mastodon-2-9-and-28121681\">a Patreon update<\/a>, Eugen Rochko writes:<\/p>\n<!-- \/wp:paragraph -->\n\n<!-- wp:quote -->\n<blockquote class=\"wp-block-quote\"><p>...OStatus...has overstayed its welcome in the code...and now that most of the network uses ActivityPub, it's time for it to go. <\/p><cite>Eugen Rochko, Mastodon creator<\/cite><\/blockquote>\n<!-- \/wp:quote -->\n\n<!-- wp:paragraph -->\n<p>The <a href=\"https:\/\/github.com\/tootsuite\/mastodon\/pull\/11205\">pull request<\/a> to remove Pubsubhubbub and Salmon, two of the main components of OStatus, has already been merged into Mastodon's master branch.<\/p>\n<!-- \/wp:paragraph -->\n\n<!-- wp:paragraph -->\n<p>Some projects will be left in the dark as a side effect of this. GNU Social and PostActiv, for example, both only communicate using OStatus. While <a href=\"https:\/\/mastodon.social\/@dansup\/102076573310057902\">some discussion<\/a> exists regarding adopting ActivityPub for GNU Social, and <a href=\"https:\/\/notabug.org\/diogo\/gnu-social\/src\/activitypub\/plugins\/ActivityPub\">a plugin is in development<\/a>, it hasn't been formally adopted yet. We just hope that the <a href=\"https:\/\/status.fsf.org\/main\/public\">Free Software Foundation's instance<\/a> gets updated in time!<\/p>\n<!-- \/wp:paragraph -->","summary":"One of the largest platforms in the federated social web is dropping the protocol that it started with.","attributedTo":"https:\/\/wedistribute.org\/wp-json\/pterotype\/v1\/actor\/-blog","url":"https:\/\/wedistribute.org\/2019\/07\/mastodon-drops-ostatus\/","to":["https:\/\/www.w3.org\/ns\/activitystreams#Public","https:\/\/wedistribute.org\/wp-json\/pterotype\/v1\/actor\/-blog\/followers"],"id":"https:\/\/wedistribute.org\/wp-json\/pterotype\/v1\/object\/85810","likes":"https:\/\/wedistribute.org\/wp-json\/pterotype\/v1\/object\/85810\/likes","shares":"https:\/\/wedistribute.org\/wp-json\/pterotype\/v1\/object\/85810\/shares"},"to":["https:\/\/www.w3.org\/ns\/activitystreams#Public","https:\/\/wedistribute.org\/wp-json\/pterotype\/v1\/actor\/-blog\/followers"],"id":"https:\/\/wedistribute.org\/wp-json\/pterotype\/v1\/object\/85809"}
|
|
@ -2,10 +2,10 @@
|
||||||
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||||
# SPDX-License-Identifier: AGPL-3.0-only
|
# SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
|
||||||
defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidatorTest do
|
defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidatorTest do
|
||||||
use Pleroma.DataCase
|
use Pleroma.DataCase
|
||||||
|
|
||||||
alias Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator
|
alias Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator
|
||||||
alias Pleroma.Web.ActivityPub.Utils
|
alias Pleroma.Web.ActivityPub.Utils
|
||||||
|
|
||||||
import Pleroma.Factory
|
import Pleroma.Factory
|
||||||
|
@ -29,7 +29,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidatorTest do
|
||||||
end
|
end
|
||||||
|
|
||||||
test "a basic note validates", %{note: note} do
|
test "a basic note validates", %{note: note} do
|
||||||
%{valid?: true} = NoteValidator.cast_and_validate(note)
|
%{valid?: true} = ArticleNoteValidator.cast_and_validate(note)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
|
@ -0,0 +1,75 @@
|
||||||
|
# Pleroma: A lightweight social networking server
|
||||||
|
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||||
|
# SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
|
||||||
|
defmodule Pleroma.Web.ActivityPub.Transmogrifier.ArticleHandlingTest do
|
||||||
|
use Oban.Testing, repo: Pleroma.Repo
|
||||||
|
use Pleroma.DataCase
|
||||||
|
|
||||||
|
alias Pleroma.Activity
|
||||||
|
alias Pleroma.Object
|
||||||
|
alias Pleroma.Object.Fetcher
|
||||||
|
alias Pleroma.Web.ActivityPub.Transmogrifier
|
||||||
|
|
||||||
|
test "Pterotype (Wordpress Plugin) Article" do
|
||||||
|
Tesla.Mock.mock(fn %{url: "https://wedistribute.org/wp-json/pterotype/v1/actor/-blog"} ->
|
||||||
|
%Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/wedistribute-user.json")}
|
||||||
|
end)
|
||||||
|
|
||||||
|
data =
|
||||||
|
File.read!("test/fixtures/tesla_mock/wedistribute-create-article.json") |> Jason.decode!()
|
||||||
|
|
||||||
|
{:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
|
||||||
|
|
||||||
|
object = Object.normalize(data["object"])
|
||||||
|
|
||||||
|
assert object.data["name"] == "The end is near: Mastodon plans to drop OStatus support"
|
||||||
|
|
||||||
|
assert object.data["summary"] ==
|
||||||
|
"One of the largest platforms in the federated social web is dropping the protocol that it started with."
|
||||||
|
|
||||||
|
assert object.data["url"] == "https://wedistribute.org/2019/07/mastodon-drops-ostatus/"
|
||||||
|
end
|
||||||
|
|
||||||
|
test "Plume Article" do
|
||||||
|
Tesla.Mock.mock(fn
|
||||||
|
%{url: "https://baptiste.gelez.xyz/~/PlumeDevelopment/this-month-in-plume-june-2018/"} ->
|
||||||
|
%Tesla.Env{
|
||||||
|
status: 200,
|
||||||
|
body: File.read!("test/fixtures/tesla_mock/baptiste.gelex.xyz-article.json")
|
||||||
|
}
|
||||||
|
|
||||||
|
%{url: "https://baptiste.gelez.xyz/@/BaptisteGelez"} ->
|
||||||
|
%Tesla.Env{
|
||||||
|
status: 200,
|
||||||
|
body: File.read!("test/fixtures/tesla_mock/baptiste.gelex.xyz-user.json")
|
||||||
|
}
|
||||||
|
end)
|
||||||
|
|
||||||
|
{:ok, object} =
|
||||||
|
Fetcher.fetch_object_from_id(
|
||||||
|
"https://baptiste.gelez.xyz/~/PlumeDevelopment/this-month-in-plume-june-2018/"
|
||||||
|
)
|
||||||
|
|
||||||
|
assert object.data["name"] == "This Month in Plume: June 2018"
|
||||||
|
|
||||||
|
assert object.data["url"] ==
|
||||||
|
"https://baptiste.gelez.xyz/~/PlumeDevelopment/this-month-in-plume-june-2018/"
|
||||||
|
end
|
||||||
|
|
||||||
|
test "Prismo Article" do
|
||||||
|
Tesla.Mock.mock(fn %{url: "https://prismo.news/@mxb"} ->
|
||||||
|
%Tesla.Env{
|
||||||
|
status: 200,
|
||||||
|
body: File.read!("test/fixtures/tesla_mock/https___prismo.news__mxb.json")
|
||||||
|
}
|
||||||
|
end)
|
||||||
|
|
||||||
|
data = File.read!("test/fixtures/prismo-url-map.json") |> Jason.decode!()
|
||||||
|
|
||||||
|
{:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
|
||||||
|
object = Object.normalize(data["object"])
|
||||||
|
|
||||||
|
assert object.data["url"] == "https://prismo.news/posts/83"
|
||||||
|
end
|
||||||
|
end
|
|
@ -44,15 +44,6 @@ test "it works for incoming notices with tag not being an array (kroeg)" do
|
||||||
assert "test" in object.data["tag"]
|
assert "test" in object.data["tag"]
|
||||||
end
|
end
|
||||||
|
|
||||||
test "it works for incoming notices with url not being a string (prismo)" do
|
|
||||||
data = File.read!("test/fixtures/prismo-url-map.json") |> Poison.decode!()
|
|
||||||
|
|
||||||
{:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
|
|
||||||
object = Object.normalize(data["object"])
|
|
||||||
|
|
||||||
assert object.data["url"] == "https://prismo.news/posts/83"
|
|
||||||
end
|
|
||||||
|
|
||||||
test "it cleans up incoming notices which are not really DMs" do
|
test "it cleans up incoming notices which are not really DMs" do
|
||||||
user = insert(:user)
|
user = insert(:user)
|
||||||
other_user = insert(:user)
|
other_user = insert(:user)
|
||||||
|
|
Loading…
Reference in a new issue