From b71b7d00a1adaf7901337292fa3263f37b81f3c9 Mon Sep 17 00:00:00 2001 From: Marcel Otto Date: Mon, 20 Dec 2021 02:22:51 +0100 Subject: [PATCH] Add generation of EARL implementation reports --- mix.exs | 23 +- test/acceptance/nquads_w3c_test.exs | 3 +- test/acceptance/ntriples_star_w3c_test.exs | 1 + test/acceptance/ntriples_w3c_test.exs | 3 +- test/acceptance/turtle_star_w3c_eval_test.exs | 1 + .../turtle_star_w3c_syntax_test.exs | 1 + test/acceptance/turtle_w3c_test.exs | 10 +- test/support/earl_formatter.ex | 223 ++++++++++++++++++ 8 files changed, 261 insertions(+), 4 deletions(-) create mode 100644 test/support/earl_formatter.ex diff --git a/mix.exs b/mix.exs index 0cb9ef7..da914db 100644 --- a/mix.exs +++ b/mix.exs @@ -15,6 +15,7 @@ defmodule RDF.Mixfile do deps: deps(), elixirc_paths: elixirc_paths(Mix.env()), compilers: Mix.compilers() ++ [:protocol_ex], + aliases: aliases(), # Dialyzer dialyzer: dialyzer(), @@ -38,7 +39,8 @@ defmodule RDF.Mixfile do coveralls: :test, "coveralls.detail": :test, "coveralls.post": :test, - "coveralls.html": :test + "coveralls.html": :test, + earl_reports: :test ] ] end @@ -86,6 +88,25 @@ defmodule RDF.Mixfile do ] end + defp aliases do + [ + earl_reports: &earl_reports/1 + ] + end + + defp earl_reports(_) do + files = [ + "test/acceptance/ntriples_w3c_test.exs", + "test/acceptance/ntriples_star_w3c_test.exs", + "test/acceptance/nquads_w3c_test.exs", + "test/acceptance/turtle_w3c_test.exs", + "test/acceptance/turtle_star_w3c_syntax_test.exs", + "test/acceptance/turtle_star_w3c_eval_test.exs" + ] + + Mix.Task.run("test", ["--formatter", "EarlFormatter", "--seed", "0"] ++ files) + end + defp elixirc_paths(:test), do: ["lib", "test/support"] defp elixirc_paths(_), do: ["lib"] end diff --git a/test/acceptance/nquads_w3c_test.exs b/test/acceptance/nquads_w3c_test.exs index 1c32155..851c2de 100644 --- a/test/acceptance/nquads_w3c_test.exs +++ b/test/acceptance/nquads_w3c_test.exs @@ -6,12 +6,13 @@ defmodule RDF.NQuads.W3C.TestSuite do """ use ExUnit.Case, async: false + use EarlFormatter, test_suite: :nquads alias RDF.{TestSuite, NQuads} alias TestSuite.NS.RDFT @path RDF.TestData.path("N-QUADS-TESTS") - @base "http://example/base/" + @base "https://www.w3.org/2013/N-QuadsTests/" @manifest TestSuite.manifest_path(@path) |> TestSuite.manifest_graph(base: @base) @manifest diff --git a/test/acceptance/ntriples_star_w3c_test.exs b/test/acceptance/ntriples_star_w3c_test.exs index 746f21c..c840815 100644 --- a/test/acceptance/ntriples_star_w3c_test.exs +++ b/test/acceptance/ntriples_star_w3c_test.exs @@ -6,6 +6,7 @@ defmodule RDF.Star.NTriples.W3C.TestSuite do """ use ExUnit.Case, async: false + use EarlFormatter, test_suite: :ntriples_star alias RDF.{TestSuite, NTriples} alias TestSuite.NS.RDFT diff --git a/test/acceptance/ntriples_w3c_test.exs b/test/acceptance/ntriples_w3c_test.exs index f45c3dc..51ebe1b 100644 --- a/test/acceptance/ntriples_w3c_test.exs +++ b/test/acceptance/ntriples_w3c_test.exs @@ -6,12 +6,13 @@ defmodule RDF.NTriples.W3C.TestSuite do """ use ExUnit.Case, async: false + use EarlFormatter, test_suite: :ntriples alias RDF.{TestSuite, NTriples} alias TestSuite.NS.RDFT @path RDF.TestData.path("N-TRIPLES-TESTS") - @base "http://example/base/" + @base "https://www.w3.org/2013/N-TriplesTests/" @manifest TestSuite.manifest_path(@path) |> TestSuite.manifest_graph(base: @base) @manifest diff --git a/test/acceptance/turtle_star_w3c_eval_test.exs b/test/acceptance/turtle_star_w3c_eval_test.exs index b30c38e..baacdbd 100644 --- a/test/acceptance/turtle_star_w3c_eval_test.exs +++ b/test/acceptance/turtle_star_w3c_eval_test.exs @@ -6,6 +6,7 @@ defmodule RDF.Star.Turtle.W3C.EvalTest do """ use ExUnit.Case, async: false + use EarlFormatter, test_suite: :turtle_star alias RDF.{Turtle, TestSuite, NTriples} alias TestSuite.NS.RDFT diff --git a/test/acceptance/turtle_star_w3c_syntax_test.exs b/test/acceptance/turtle_star_w3c_syntax_test.exs index d6b6d88..80efa4e 100644 --- a/test/acceptance/turtle_star_w3c_syntax_test.exs +++ b/test/acceptance/turtle_star_w3c_syntax_test.exs @@ -6,6 +6,7 @@ defmodule RDF.Star.Turtle.W3C.SyntaxTest do """ use ExUnit.Case, async: false + use EarlFormatter, test_suite: :turtle_star alias RDF.{Turtle, TestSuite} alias TestSuite.NS.RDFT diff --git a/test/acceptance/turtle_w3c_test.exs b/test/acceptance/turtle_w3c_test.exs index 41db75d..3b4c283 100644 --- a/test/acceptance/turtle_w3c_test.exs +++ b/test/acceptance/turtle_w3c_test.exs @@ -8,6 +8,7 @@ defmodule RDF.Turtle.W3C.Test do """ use ExUnit.Case, async: false + use EarlFormatter, test_suite: :turtle alias RDF.{Turtle, TestSuite, NTriples} alias TestSuite.NS.RDFT @@ -46,6 +47,8 @@ defmodule RDF.Turtle.W3C.Test do turtle-subm-10 turtle-subm-14 ] do + @tag earl_result: :passed + @tag earl_mode: :semi_auto @tag skip: """ The produced graphs are correct, but have different blank node labels than the result graph. TODO: Implement a graph isomorphism algorithm. @@ -90,7 +93,12 @@ defmodule RDF.Turtle.W3C.Test do TestSuite.test_cases(@manifest, RDFT.TestTurtleNegativeEval) |> Enum.each(fn test_case -> - if TestSuite.test_name(test_case) in ~w[turtle-eval-bad-01 turtle-eval-bad-02 turtle-eval-bad-03] do + if TestSuite.test_name(test_case) in ~w[ + turtle-eval-bad-01 + turtle-eval-bad-02 + turtle-eval-bad-03 + ] do + @tag earl_result: :failed @tag skip: "TODO: IRI validation" end diff --git a/test/support/earl_formatter.ex b/test/support/earl_formatter.ex new file mode 100644 index 0000000..24c4893 --- /dev/null +++ b/test/support/earl_formatter.ex @@ -0,0 +1,223 @@ +defmodule EarlFormatter do + @moduledoc """ + An `ExUnit.Formatter` implementation that generates EARL reports. + + see + """ + use GenServer + + defmodule NS do + use RDF.Vocabulary.Namespace + + defvocab EARL, base_iri: "http://www.w3.org/ns/earl#", terms: [], strict: false + defvocab DC, base_iri: "http://purl.org/dc/terms/", terms: [], strict: false + defvocab FOAF, base_iri: "http://xmlns.com/foaf/0.1/", terms: [], strict: false + end + + @compile {:no_warn_undefined, EarlFormatter.NS.EARL} + @compile {:no_warn_undefined, EarlFormatter.NS.DC} + @compile {:no_warn_undefined, EarlFormatter.NS.FOAF} + + alias EarlFormatter.NS.{EARL, DC, FOAF} + alias RDF.{Graph, Turtle} + + import RDF.Sigils + + @output_path "earl_reports" + @doap_file "doap.ttl" + + @marcel ~I + @rdf_ex ~I + + @prefixes RDF.prefix_map( + xsd: RDF.NS.XSD, + rdf: RDF, + rdfs: RDF.NS.RDFS, + mf: RDF.TestSuite.NS.MF, + earl: EARL, + dc: DC, + foaf: FOAF, + doap: "http://usefulinc.com/ns/doap#" + ) + + @impl true + def init(_opts) do + {:ok, {%{}, %{time: RDF.XSD.DateTime.now()}}} + end + + @impl true + def handle_cast({:suite_finished, %{async: _, load: _, run: _}}, {results, config} = state) do + finish(results, config) + {:noreply, state} + end + + def handle_cast({:suite_finished, _run_us, _load_us}, {results, config} = state) do + finish(results, config) + {:noreply, state} + end + + def handle_cast({:test_finished, %ExUnit.Test{state: nil} = test}, {results, config}) do + print_success("PASSED: #{test.name}") + + {:noreply, + {add_result(results, test, assertion(test.tags.test_case, :passed, config)), config}} + end + + def handle_cast({:test_finished, %ExUnit.Test{state: {:skipped, _}} = test}, {results, config}) do + result = test.tags[:earl_result] || :failed + mode = test.tags[:earl_mode] + print_warn("SKIPPED (#{mode} #{result}): #{test.name}") + + {:noreply, + {add_result(results, test, assertion(test.tags.test_case, result, mode, config)), config}} + end + + def handle_cast({:test_finished, %ExUnit.Test{state: {:excluded, _}} = test}, {results, config}) do + print_warn("EXCLUDED: #{test.name}") + + {:noreply, + {add_result(results, test, assertion(test.tags.test_case, :untested, config)), config}} + end + + def handle_cast( + {:test_finished, %ExUnit.Test{state: {:failed, _failed}} = test}, + {results, config} + ) do + print_failed("FAILED: #{test.name}") + + {:noreply, + {add_result(results, test, assertion(test.tags.test_case, :failed, config)), config}} + end + + def handle_cast( + {:test_finished, %ExUnit.Test{state: {:invalid, _module}} = test}, + {results, config} + ) do + print_failed("INVALID: #{test.name}") + + {:noreply, + {add_result(results, test, assertion(test.tags.test_case, :failed, config)), config}} + end + + def handle_cast(_event, state), do: {:noreply, state} + + defp add_result(results, test, assertion) do + Map.update( + results, + test_suite(test), + RDF.graph(prefixes: @prefixes), + &Graph.add(&1, assertion) + ) + end + + defp finish(results, config) do + project_metadata = project_metadata() + + IO.puts("---------------------------------") + + Enum.each(results, fn {test_suite, results} -> + IO.puts("Writing report for #{test_suite}") + path = Path.join(@output_path, "#{test_suite}.ttl") + + results + |> Graph.add(project_metadata) + |> Turtle.write_file!(path, force: true, base_description: document_description(config)) + end) + end + + defp project_metadata do + doap = Turtle.read_file!(@doap_file) + + # ensure the URIs we use here are consistent we the ones in the DOAP file + %RDF.Description{} = doap[@rdf_ex] + %RDF.Description{} = doap[@marcel] + + doap + |> Graph.add(@rdf_ex |> RDF.type([EARL.TestSubject, EARL.Software])) + |> Graph.add(@marcel |> RDF.type(EARL.Assertor)) + end + + defp document_description(config) do + %{ + FOAF.primaryTopic() => @rdf_ex, + FOAF.maker() => @marcel, + DC.issued() => config.time + } + end + + defp base_assertion(test_case) do + RDF.bnode() + |> RDF.type(EARL.Assertion) + |> EARL.assertedBy(@marcel) + |> EARL.subject(@rdf_ex) + |> EARL.test(test_case.subject) + end + + defp assertion(test_case, outcome, mode \\ nil, config) + + defp assertion(test_case, outcome, nil, config), + do: assertion(test_case, outcome, :automatic, config) + + defp assertion(test_case, outcome, mode, config) do + result = result(outcome, config) + + assertion = + test_case + |> base_assertion() + |> EARL.result(result.subject) + |> EARL.mode(mode(mode)) + + [assertion, result] + end + + defp base_result(config) do + RDF.bnode() + |> RDF.type(EARL.TestResult) + |> DC.date(config.time) + end + + defp result(outcome, config) do + base_result(config) + |> EARL.outcome(outcome(outcome)) + end + + # earl:passed := the subject passed the test. + defp outcome(:passed), do: EARL.passed() + # earl:failed := the subject failed the test. + defp outcome(:failed), do: EARL.failed() + # earl:cantTell := it is unclear if the subject passed or failed the test. + defp outcome(:cant_tell), do: EARL.cantTell() + # earl:inapplicable := the test is not applicable to the subject. + defp outcome(:inapplicable), do: EARL.inapplicable() + # earl:untested := the test has not been carried out. + defp outcome(:untested), do: EARL.untested() + + # earl:automatic := where the test was carried out automatically by the software tool and without any human intervention. + defp mode(:automatic), do: EARL.automatic() + + # earl:manual := where the test was carried out by human evaluators. This includes the case where the evaluators are aided by instructions or guidance provided by software tools, but where the evaluators carried out the actual test procedure. + defp mode(:manual), do: EARL.manual() + + # earl:semiAuto := where the test was partially carried out by software tools, but where human input or judgment was still required to decide or help decide the outcome of the test. + defp mode(:semi_auto), do: EARL.semiAuto() + + # earl:undisclosed := where the exact testing process is undisclosed. + defp mode(:undisclosed), do: EARL.undisclosed() + + # earl:unknownMode := where the testing process is unknown or undetermined. + defp mode(:unknown_mode), do: EARL.unknownMode() + + defmacro __using__(opts) do + earl_test_suite = Keyword.fetch!(opts, :test_suite) + + quote do + def earl_test_suite(), do: unquote(earl_test_suite) + end + end + + defp test_suite(test), do: test.module.earl_test_suite() + + defp print_success(msg), do: IO.puts(IO.ANSI.format([:green, msg])) + defp print_failed(msg), do: IO.puts(IO.ANSI.format([:red, msg])) + defp print_warn(msg), do: IO.puts(IO.ANSI.format([:yellow, msg])) +end